From 5e0baa0c28dc769c324216ba062363a27b3bfcc0 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 22 Jan 2025 08:11:04 +0300 Subject: [PATCH] Replace cpp sdk to v3 (#7796) --- .github/config/muted_ya.txt | 4 +- ydb/apps/pgwire/pg_ydb_connection.cpp | 12 +- ydb/apps/pgwire/pg_ydb_connection.h | 4 +- ydb/apps/pgwire/pg_ydb_proxy.cpp | 2 +- ydb/apps/pgwire/pg_ydb_proxy.h | 2 +- ydb/apps/pgwire/ya.make | 8 +- ydb/apps/ydb/commands/ya.make | 2 +- ydb/apps/ydb/commands/ydb_cloud_root.cpp | 6 +- ydb/apps/ydb/main.cpp | 2 +- ydb/apps/ydb/ut/workload-topic.cpp | 2 +- .../ut/workload-transfer-topic-to-table.cpp | 4 +- ydb/apps/ydb/ut/ya.make | 4 +- ydb/apps/ydb/ut/ydb-dump.cpp | 4 +- ydb/apps/ydbd/ya.make | 2 +- ydb/core/base/appdata_fwd.h | 2 +- ydb/core/client/scheme_cache_lib/ya.make | 2 +- ydb/core/client/server/msgbus_securereq.h | 6 +- .../server/msgbus_server_pq_metacache.h | 4 +- .../cms/console/console_tenants_manager.cpp | 2 +- .../cms/console/console_tenants_manager.h | 3 +- ydb/core/cms/console/ya.make | 3 +- ydb/core/config/init/init.cpp | 22 +- ydb/core/config/init/init_impl.h | 4 +- ydb/core/config/init/ya.make | 4 +- ydb/core/driver_lib/base_utils/ya.make | 2 +- ydb/core/driver_lib/cli_base/cli_cmds_db.cpp | 12 +- ydb/core/driver_lib/cli_base/cli_grpc.h | 10 +- ydb/core/driver_lib/cli_base/cli_kicli.cpp | 4 +- ydb/core/driver_lib/cli_base/ya.make | 8 +- .../driver_lib/cli_utils/cli_cmds_tablet.cpp | 2 +- .../driver_lib/cli_utils/cli_cmds_tenant.cpp | 4 +- .../cli_persqueue_cluster_discovery.cpp | 2 +- ydb/core/driver_lib/cli_utils/ya.make | 4 +- ydb/core/driver_lib/run/ya.make | 6 +- ydb/core/external_sources/object_storage.cpp | 5 +- .../external_sources/object_storage/events.h | 5 +- .../external_sources/object_storage/ya.make | 1 + .../s3/ut/s3_aws_credentials_ut.cpp | 24 +- ydb/core/external_sources/s3/ut/ya.make | 2 +- ydb/core/external_sources/ya.make | 5 +- ydb/core/fq/libs/actors/database_resolver.cpp | 2 +- ydb/core/fq/libs/actors/nodes_manager.cpp | 4 +- ydb/core/fq/libs/actors/pending_fetcher.cpp | 8 +- ydb/core/fq/libs/actors/result_writer.cpp | 2 +- ydb/core/fq/libs/actors/run_actor.cpp | 2 +- ydb/core/fq/libs/actors/ya.make | 6 +- .../fq/libs/checkpoint_storage/ut/gc_ut.cpp | 2 +- .../fq/libs/checkpoint_storage/ut/ya.make | 2 +- ydb/core/fq/libs/checkpoint_storage/ya.make | 5 +- .../ydb_checkpoint_storage.cpp | 20 +- .../checkpoint_storage/ydb_state_storage.cpp | 33 +- ydb/core/fq/libs/common/debug_info.cpp | 2 +- ydb/core/fq/libs/common/debug_info.h | 2 +- .../fq/libs/common/rows_proto_splitter_ut.cpp | 2 +- ydb/core/fq/libs/common/util.cpp | 2 +- ydb/core/fq/libs/common/util.h | 2 +- ydb/core/fq/libs/compute/common/pinger.cpp | 2 +- ydb/core/fq/libs/compute/common/retry_actor.h | 2 +- .../libs/compute/common/run_actor_params.cpp | 2 +- ydb/core/fq/libs/compute/common/utils.cpp | 2 +- ydb/core/fq/libs/compute/common/utils.h | 4 +- ydb/core/fq/libs/compute/ydb/actors_factory.h | 2 +- .../fq/libs/compute/ydb/base_compute_actor.h | 2 +- ...compute_database_control_plane_service.cpp | 4 +- .../fq/libs/compute/ydb/control_plane/ya.make | 2 +- ydb/core/fq/libs/compute/ydb/events/events.h | 6 +- ydb/core/fq/libs/compute/ydb/events/ya.make | 4 +- .../fq/libs/compute/ydb/executer_actor.cpp | 14 +- ydb/core/fq/libs/compute/ydb/executer_actor.h | 2 +- .../fq/libs/compute/ydb/finalizer_actor.cpp | 4 +- .../fq/libs/compute/ydb/finalizer_actor.h | 2 +- .../fq/libs/compute/ydb/initializer_actor.cpp | 4 +- .../fq/libs/compute/ydb/initializer_actor.h | 2 +- .../compute/ydb/resources_cleaner_actor.cpp | 14 +- .../libs/compute/ydb/result_writer_actor.cpp | 20 +- .../libs/compute/ydb/status_tracker_actor.cpp | 16 +- .../fq/libs/compute/ydb/stopper_actor.cpp | 14 +- .../synchronization_service.cpp | 11 +- .../ydb/synchronization_service/ya.make | 3 +- ydb/core/fq/libs/compute/ydb/ya.make | 3 +- .../libs/compute/ydb/ydb_connector_actor.cpp | 23 +- .../fq/libs/compute/ydb/ydb_run_actor.cpp | 16 +- .../control_plane_config.cpp | 2 +- ydb/core/fq/libs/control_plane_config/ya.make | 4 +- .../libs/control_plane_proxy/actors/ya.make | 1 + .../actors/ydb_schema_query_actor.cpp | 19 +- .../libs/control_plane_proxy/events/events.h | 2 +- .../control_plane_storage/events/events.h | 4 +- .../events/internal_events.h | 4 +- .../internal/nodes_health_check.cpp | 4 +- .../internal/rate_limiter_resources.cpp | 4 +- .../internal/task_get.cpp | 20 +- .../internal/task_ping.cpp | 12 +- .../control_plane_storage/internal/utils.h | 2 +- .../control_plane_storage/internal/ya.make | 4 +- .../control_plane_storage/request_actor.h | 8 +- .../libs/control_plane_storage/validators.cpp | 24 +- .../libs/control_plane_storage/validators.h | 8 +- .../fq/libs/control_plane_storage/ya.make | 5 +- .../ydb_control_plane_storage.cpp | 58 +- .../ydb_control_plane_storage_bindings.cpp | 4 +- ...control_plane_storage_compute_database.cpp | 4 +- .../ydb_control_plane_storage_connections.cpp | 2 +- .../ydb_control_plane_storage_impl.h | 17 +- .../ydb_control_plane_storage_queries.cpp | 20 +- .../ydb_control_plane_storage_quotas.cpp | 8 +- ydb/core/fq/libs/db_schema/db_schema.cpp | 2 +- ydb/core/fq/libs/db_schema/db_schema.h | 4 +- ydb/core/fq/libs/db_schema/ya.make | 4 +- ydb/core/fq/libs/events/events.h | 2 +- ydb/core/fq/libs/events/ya.make | 2 +- ydb/core/fq/libs/health/health.cpp | 2 +- ydb/core/fq/libs/health/ya.make | 2 +- ydb/core/fq/libs/mock/ya.make | 2 +- ydb/core/fq/libs/private_client/events.h | 10 +- .../libs/private_client/internal_service.cpp | 2 +- .../fq/libs/private_client/internal_service.h | 2 +- .../fq/libs/private_client/private_client.cpp | 2 +- .../fq/libs/private_client/private_client.h | 2 +- ydb/core/fq/libs/private_client/ya.make | 6 +- .../fq/libs/quota_manager/quota_manager.cpp | 6 +- .../fq/libs/quota_manager/quota_proxy.cpp | 2 +- ydb/core/fq/libs/quota_manager/ya.make | 2 +- .../rate_limiter_control_plane_service.cpp | 11 +- .../update_limit_actor.cpp | 2 +- .../control_plane_service/ya.make | 1 + .../fq/libs/read_rule/read_rule_creator.cpp | 5 +- .../fq/libs/read_rule/read_rule_creator.h | 2 +- .../fq/libs/read_rule/read_rule_deleter.cpp | 5 +- .../fq/libs/read_rule/read_rule_deleter.h | 2 +- ydb/core/fq/libs/read_rule/ya.make | 5 +- .../result_formatter/result_formatter.cpp | 6 +- .../libs/result_formatter/result_formatter.h | 4 +- ydb/core/fq/libs/result_formatter/ya.make | 6 +- .../fq/libs/row_dispatcher/actors_factory.h | 2 +- .../format_handler/filters/filters_set.h | 2 +- .../format_handler/format_handler.cpp | 6 +- .../format_handler/format_handler.h | 4 +- .../format_handler/parsers/json_parser.cpp | 2 +- .../format_handler/parsers/parser_abstract.h | 4 +- .../format_handler/parsers/raw_parser.cpp | 2 +- .../format_handler/parsers/ya.make | 2 +- .../format_handler/ut/format_handler_ut.cpp | 4 +- .../format_handler/ut/topic_filter_ut.cpp | 4 +- .../libs/row_dispatcher/leader_election.cpp | 2 +- .../fq/libs/row_dispatcher/topic_session.cpp | 26 +- .../row_dispatcher/ut/topic_session_ut.cpp | 3 +- ydb/core/fq/libs/row_dispatcher/ya.make | 5 +- ydb/core/fq/libs/shared_resources/db_exec.h | 20 +- .../shared_resources/shared_resources.cpp | 6 +- .../libs/shared_resources/shared_resources.h | 2 +- ydb/core/fq/libs/shared_resources/ya.make | 8 +- .../test_connection/test_object_storage.cpp | 2 +- ydb/core/fq/libs/ydb/schema.cpp | 17 +- ydb/core/fq/libs/ydb/schema.h | 2 +- ydb/core/fq/libs/ydb/ut/ydb_ut.cpp | 4 +- ydb/core/fq/libs/ydb/util.h | 2 +- ydb/core/fq/libs/ydb/ya.make | 9 +- ydb/core/fq/libs/ydb/ydb.cpp | 13 +- ydb/core/fq/libs/ydb/ydb.h | 11 +- ydb/core/graph/service/ya.make | 1 + ydb/core/grpc_caching/cache_item.h | 2 +- .../grpc_caching/cached_grpc_request_actor.h | 2 +- ydb/core/grpc_caching/ya.make | 1 + ydb/core/grpc_services/base/base.h | 2 +- ydb/core/grpc_services/base/ya.make | 2 +- ydb/core/grpc_services/operation_helpers.cpp | 2 +- ydb/core/grpc_services/operation_helpers.h | 3 +- .../query/rpc_execute_script.cpp | 2 +- ydb/core/grpc_services/rpc_bsconfig_base.h | 4 +- ydb/core/grpc_services/rpc_calls.h | 2 +- .../grpc_services/rpc_cancel_operation.cpp | 4 +- .../rpc_common/rpc_common_kqp_session.cpp | 2 +- ydb/core/grpc_services/rpc_deferrable.h | 2 +- .../grpc_services/rpc_execute_data_query.cpp | 2 +- ydb/core/grpc_services/rpc_export_base.h | 3 +- .../grpc_services/rpc_forget_operation.cpp | 4 +- ydb/core/grpc_services/rpc_get_operation.cpp | 2 +- ydb/core/grpc_services/rpc_import_base.h | 3 +- ydb/core/grpc_services/rpc_kqp_base.h | 4 +- .../grpc_services/rpc_list_operations.cpp | 2 +- ydb/core/grpc_services/rpc_object_storage.cpp | 2 +- .../grpc_services/rpc_prepare_data_query.cpp | 2 +- ydb/core/grpc_services/rpc_read_rows.cpp | 2 +- ydb/core/grpc_services/rpc_whoami.cpp | 2 +- ydb/core/grpc_services/ya.make | 8 +- ydb/core/grpc_streaming/ut/ya.make | 2 +- ydb/core/health_check/health_check.cpp | 4 +- ydb/core/health_check/ya.make | 1 + ydb/core/http_proxy/auth_factory.cpp | 2 +- ydb/core/http_proxy/events.h | 4 +- ydb/core/http_proxy/grpc_service.cpp | 6 +- ydb/core/http_proxy/http_req.cpp | 14 +- ydb/core/http_proxy/http_req.h | 2 +- ydb/core/http_proxy/http_service.h | 2 +- ydb/core/http_proxy/json_proto_conversion.h | 3 +- ydb/core/http_proxy/ut/datastreams_fixture.h | 10 +- ydb/core/http_proxy/ut/inside_ydb_ut/ya.make | 4 +- ydb/core/http_proxy/ut/ya.make | 4 +- ydb/core/http_proxy/ya.make | 13 +- .../kafka_proxy/actors/control_plane_common.h | 8 +- .../actors/kafka_alter_configs_actor.cpp | 2 +- .../actors/kafka_create_partitions_actor.cpp | 4 +- .../actors/kafka_create_topics_actor.cpp | 4 +- .../actors/kafka_sasl_auth_actor.cpp | 2 +- ydb/core/kafka_proxy/ut/metarequest_ut.cpp | 2 +- ydb/core/kafka_proxy/ut/port_discovery_ut.cpp | 2 +- ydb/core/kafka_proxy/ut/ut_protocol.cpp | 27 +- ydb/core/kafka_proxy/ut/ya.make | 6 +- .../kqp/common/events/script_executions.h | 2 +- ydb/core/kqp/common/events/ya.make | 2 +- ydb/core/kqp/common/kqp_script_executions.cpp | 6 +- ydb/core/kqp/common/kqp_script_executions.h | 2 +- ydb/core/kqp/common/ya.make | 4 +- ydb/core/kqp/counters/ya.make | 1 + .../kqp/executer_actor/ut/kqp_executer_ut.cpp | 2 +- ydb/core/kqp/executer_actor/ut/ya.make | 2 +- .../resource_pool_classifier/checker.cpp | 6 +- ydb/core/kqp/gateway/kqp_ic_gateway.cpp | 2 +- ydb/core/kqp/host/kqp_gateway_proxy.cpp | 10 +- ydb/core/kqp/host/ya.make | 2 +- .../kqp/provider/read_attributes_utils.cpp | 2 +- ydb/core/kqp/provider/ya.make | 2 +- .../kqp/provider/yql_kikimr_datasource.cpp | 2 +- ydb/core/kqp/provider/yql_kikimr_exec.cpp | 2 +- ydb/core/kqp/provider/yql_kikimr_gateway.h | 2 +- .../kqp/proxy_service/kqp_proxy_service.cpp | 4 +- ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp | 10 +- .../proxy_service/kqp_script_executions.cpp | 116 +- .../kqp/proxy_service/kqp_script_executions.h | 2 +- .../kqp_script_executions_ut.cpp | 18 +- ydb/core/kqp/proxy_service/ut/ya.make | 4 +- ydb/core/kqp/proxy_service/ya.make | 4 +- ydb/core/kqp/query_data/kqp_query_data.cpp | 4 +- ydb/core/kqp/query_data/ya.make | 2 +- .../run_script_actor/kqp_run_script_actor.cpp | 2 +- .../kqp/session_actor/kqp_session_actor.cpp | 4 +- ydb/core/kqp/session_actor/ya.make | 2 +- .../kqp/tests/kikimr_tpch/kqp_tpch_ut.cpp | 2 +- ydb/core/kqp/tests/kikimr_tpch/ya.make | 2 +- ydb/core/kqp/tests/tpch/cmd_run_bench.cpp | 2 +- ydb/core/kqp/tests/tpch/lib/tpch_runner.cpp | 4 +- ydb/core/kqp/tests/tpch/lib/tpch_runner.h | 4 +- ydb/core/kqp/tests/tpch/lib/tpch_tables.h | 3 +- ydb/core/kqp/tests/tpch/lib/ya.make | 4 +- ydb/core/kqp/topics/ya.make | 1 + .../kqp/ut/arrow/kqp_arrow_in_channels_ut.cpp | 38 +- ydb/core/kqp/ut/arrow/kqp_types_arrow_ut.cpp | 38 +- ydb/core/kqp/ut/common/columnshard.h | 4 +- ydb/core/kqp/ut/common/kqp_ut_common.cpp | 27 +- ydb/core/kqp/ut/common/kqp_ut_common.h | 11 +- ydb/core/kqp/ut/common/ya.make | 12 +- ydb/core/kqp/ut/cost/kqp_cost_ut.cpp | 2 +- ydb/core/kqp/ut/data/kqp_read_null_ut.cpp | 2 +- ydb/core/kqp/ut/data/ya.make | 2 +- ydb/core/kqp/ut/effects/kqp_effects_ut.cpp | 26 +- .../ut/effects/kqp_immediate_effects_ut.cpp | 74 +- .../kqp/ut/effects/kqp_inplace_update_ut.cpp | 2 +- ydb/core/kqp/ut/effects/kqp_write_ut.cpp | 2 +- .../kqp/ut/federated_query/common/common.h | 4 +- .../kqp/ut/federated_query/common/ya.make | 4 +- .../generic_ut/kqp_generic_provider_ut.cpp | 8 +- .../s3/kqp_federated_query_ut.cpp | 48 +- .../s3/kqp_federated_scheme_ut.cpp | 2 +- .../ut/federated_query/s3/kqp_s3_plan_ut.cpp | 10 +- ydb/core/kqp/ut/federated_query/s3/ya.make | 2 +- ydb/core/kqp/ut/idx_test/ya.make | 4 +- ydb/core/kqp/ut/idx_test/ydb_index_ut.cpp | 12 +- .../ut/indexes/kqp_indexes_multishard_ut.cpp | 9 +- ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp | 194 +- ydb/core/kqp/ut/indexes/ya.make | 1 + .../kqp/ut/join/kqp_index_lookup_join_ut.cpp | 2 +- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 16 +- ydb/core/kqp/ut/join/kqp_join_ut.cpp | 36 +- ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp | 4 +- ydb/core/kqp/ut/olap/datatime64_ut.cpp | 8 +- ydb/core/kqp/ut/olap/decimal_ut.cpp | 8 +- ydb/core/kqp/ut/olap/helpers/get_value.cpp | 2 +- ydb/core/kqp/ut/olap/helpers/get_value.h | 2 +- .../kqp/ut/olap/helpers/query_executor.cpp | 4 +- ydb/core/kqp/ut/olap/helpers/query_executor.h | 4 +- ydb/core/kqp/ut/olap/helpers/typed_local.h | 2 +- ydb/core/kqp/ut/olap/indexes_ut.cpp | 2 +- ydb/core/kqp/ut/olap/ya.make | 2 +- .../opt/kqp_extract_predicate_unpack_ut.cpp | 4 +- ydb/core/kqp/ut/opt/kqp_kv_ut.cpp | 18 +- ydb/core/kqp/ut/opt/kqp_ne_ut.cpp | 8 +- ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp | 4 +- ydb/core/kqp/ut/opt/kqp_ranges_ut.cpp | 4 +- ydb/core/kqp/ut/opt/kqp_returning_ut.cpp | 2 +- ydb/core/kqp/ut/opt/kqp_sort_ut.cpp | 30 +- ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp | 28 +- ydb/core/kqp/ut/perf/kqp_query_perf_ut.cpp | 2 +- ydb/core/kqp/ut/perf/kqp_workload_ut.cpp | 2 +- ydb/core/kqp/ut/pg/kqp_pg_ut.cpp | 48 +- ydb/core/kqp/ut/pg/pg_catalog_ut.cpp | 8 +- ydb/core/kqp/ut/query/kqp_analyze_ut.cpp | 4 +- ydb/core/kqp/ut/query/kqp_explain_ut.cpp | 4 +- ydb/core/kqp/ut/query/kqp_limits_ut.cpp | 30 +- ydb/core/kqp/ut/query/kqp_params_ut.cpp | 8 +- ydb/core/kqp/ut/query/kqp_query_ut.cpp | 46 +- ydb/core/kqp/ut/query/kqp_stats_ut.cpp | 8 +- ydb/core/kqp/ut/query/kqp_types_ut.cpp | 2 +- ydb/core/kqp/ut/query/ya.make | 2 +- ydb/core/kqp/ut/scan/kqp_scan_ut.cpp | 20 +- ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp | 28 +- ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp | 6 +- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 94 +- .../kqp/ut/service/kqp_document_api_ut.cpp | 2 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 96 +- ydb/core/kqp/ut/service/kqp_qs_scripts_ut.cpp | 36 +- ydb/core/kqp/ut/service/kqp_service_ut.cpp | 10 +- ydb/core/kqp/ut/service/ya.make | 4 +- ydb/core/kqp/ut/spilling/ya.make | 2 +- ydb/core/kqp/ut/sysview/kqp_sys_view_ut.cpp | 2 +- ydb/core/kqp/ut/sysview/ya.make | 4 + ydb/core/kqp/ut/tx/kqp_locks_tricky_ut.cpp | 6 +- ydb/core/kqp/ut/tx/kqp_locks_ut.cpp | 20 +- ydb/core/kqp/ut/tx/kqp_mvcc_ut.cpp | 6 +- ydb/core/kqp/ut/tx/kqp_sink_locks_ut.cpp | 16 +- ydb/core/kqp/ut/tx/kqp_sink_mvcc_ut.cpp | 4 +- ydb/core/kqp/ut/tx/kqp_tx_ut.cpp | 2 +- ydb/core/kqp/ut/view/view_ut.cpp | 8 +- ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp | 2 +- ydb/core/kqp/ut/yql/kqp_scripting_ut.cpp | 14 +- ydb/core/kqp/ut/yql/kqp_yql_ut.cpp | 14 +- .../actors/cpu_load_actors.cpp | 4 +- .../actors/pool_handlers_acors.cpp | 4 +- .../common/cpu_quota_manager.h | 2 +- ydb/core/kqp/workload_service/common/ya.make | 2 +- .../workload_service/tables/table_queries.cpp | 14 +- .../workload_service/tables/table_queries.h | 2 +- .../common/kqp_workload_service_ut_common.h | 4 +- .../ut/kqp_workload_service_tables_ut.cpp | 2 +- ydb/core/kqp/ya.make | 2 +- ydb/core/load_test/aggregated_result.cpp | 14 +- ydb/core/load_test/kqp.cpp | 2 +- ydb/core/load_test/ut_ycsb/ya.make | 2 +- ydb/core/load_test/ya.make | 4 +- ydb/core/load_test/ycsb/kqp_select.cpp | 8 +- ydb/core/load_test/ycsb/kqp_upsert.cpp | 8 +- .../local_pgwire/local_pgwire_connection.cpp | 8 +- ydb/core/local_pgwire/local_pgwire_util.cpp | 22 +- ydb/core/local_pgwire/local_pgwire_util.h | 10 +- ydb/core/local_pgwire/pgwire_kqp_proxy.cpp | 9 +- ydb/core/local_pgwire/ya.make | 7 +- ydb/core/log_backend/ya.make | 1 + ydb/core/mon/mon.cpp | 4 +- ydb/core/mon/mon.h | 2 +- ydb/core/mon/ya.make | 3 +- ydb/core/persqueue/account_read_quoter.cpp | 2 +- .../persqueue/actor_persqueue_client_iface.h | 4 +- ydb/core/persqueue/cluster_tracker.cpp | 2 +- ydb/core/persqueue/codecs/pqv1.h | 4 +- ydb/core/persqueue/codecs/ya.make | 2 +- .../persqueue/dread_cache_service/ut/ya.make | 2 +- ydb/core/persqueue/events/internal.h | 2 +- ydb/core/persqueue/mirrorer.cpp | 22 +- ydb/core/persqueue/mirrorer.h | 6 +- .../ut/common/autoscaling_ut_common.cpp | 30 +- .../ut/common/autoscaling_ut_common.h | 8 +- ydb/core/persqueue/ut/common/ya.make | 4 + ydb/core/persqueue/ut/fetch_request_ut.cpp | 2 +- .../persqueue/ut/partition_chooser_ut.cpp | 6 +- ydb/core/persqueue/ut/pqrb_describes_ut.cpp | 2 +- .../persqueue/ut/slow/autopartitioning_ut.cpp | 2 +- ydb/core/persqueue/ut/slow/ya.make | 9 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 36 +- .../persqueue/ut/ut_with_sdk/balancing_ut.cpp | 14 +- .../persqueue/ut/ut_with_sdk/mirrorer_ut.cpp | 13 +- ydb/core/persqueue/ut/ut_with_sdk/ya.make | 10 +- ydb/core/persqueue/ut/ya.make | 8 +- ydb/core/persqueue/write_meta.cpp | 16 +- ydb/core/persqueue/write_meta.h | 4 +- ...partition_chooser_impl__partition_helper.h | 2 +- .../partition_chooser_impl__table_helper.h | 12 +- .../persqueue/writer/source_id_encoding.h | 2 +- ydb/core/persqueue/writer/ya.make | 2 +- ydb/core/persqueue/ya.make | 2 +- ydb/core/pgproxy/pg_proxy_events.h | 2 +- ydb/core/public_http/http_req.cpp | 2 +- ydb/core/public_http/ya.make | 3 +- ydb/core/security/ticket_parser_impl.h | 12 +- ydb/core/security/ticket_parser_ut.cpp | 4 +- .../statistics/database/ut/ut_database.cpp | 2 +- ydb/core/statistics/service/service_impl.cpp | 2 +- ydb/core/sys_view/ut/ya.make | 2 +- ydb/core/sys_view/ut_common.h | 6 +- ydb/core/sys_view/ut_kqp.cpp | 6 +- ydb/core/sys_view/ut_large.cpp | 2 +- .../tablet/tablet_counters_aggregator.cpp | 2 +- ydb/core/testlib/common_helper.cpp | 4 +- ydb/core/testlib/common_helper.h | 2 +- ydb/core/testlib/test_client.h | 2 +- ydb/core/testlib/test_pq_client.h | 32 +- ydb/core/testlib/ya.make | 10 +- ydb/core/tracing/ya.make | 1 + .../columnshard/bg_tasks/abstract/session.cpp | 4 +- .../tx/columnshard/bg_tasks/abstract/ya.make | 2 +- .../tx/columnshard/hooks/abstract/ya.make | 1 + .../test_helper/columnshard_ut_common.h | 2 +- .../ut_schema/ut_columnshard_schema.cpp | 2 +- .../datashard_ut_background_compaction.cpp | 2 +- .../datashard_ut_change_exchange.cpp | 25 +- .../tx/datashard/datashard_ut_common_kqp.h | 2 +- .../tx/datashard/datashard_ut_common_pq.cpp | 6 +- ...tashard_ut_ext_blobs_multiple_channels.cpp | 2 +- .../datashard_ut_incremental_backup.cpp | 6 +- ydb/core/tx/datashard/datashard_ut_kqp.cpp | 2 +- .../datashard/datashard_ut_read_iterator.cpp | 2 +- .../datashard_ut_read_iterator_ext_blobs.cpp | 2 +- .../tx/datashard/datashard_ut_read_table.h | 2 +- ydb/core/tx/datashard/type_serialization.cpp | 2 +- .../ut_background_compaction/ya.make | 2 +- ydb/core/tx/datashard/ut_build_index/ya.make | 2 +- .../tx/datashard/ut_change_collector/ya.make | 2 +- .../tx/datashard/ut_change_exchange/ya.make | 8 +- .../ut_common/datashard_ut_common.cpp | 4 +- ydb/core/tx/datashard/ut_compaction/ya.make | 2 +- ydb/core/tx/datashard/ut_erase_rows/ya.make | 2 +- ydb/core/tx/datashard/ut_followers/ya.make | 2 +- .../datashard/ut_incremental_backup/ya.make | 8 +- .../ut_incremental_restore_scan/ya.make | 8 +- ydb/core/tx/datashard/ut_init/ya.make | 2 +- ydb/core/tx/datashard/ut_keys/ya.make | 2 +- ydb/core/tx/datashard/ut_kqp/ya.make | 2 +- ydb/core/tx/datashard/ut_kqp_errors/ya.make | 2 +- ydb/core/tx/datashard/ut_kqp_scan/ya.make | 2 +- ydb/core/tx/datashard/ut_local_kmeans/ya.make | 2 +- ydb/core/tx/datashard/ut_locks/ya.make | 2 +- ydb/core/tx/datashard/ut_minikql/ya.make | 2 +- ydb/core/tx/datashard/ut_minstep/ya.make | 2 +- .../ut_object_storage_listing/ya.make | 2 +- ydb/core/tx/datashard/ut_order/ya.make | 2 +- ydb/core/tx/datashard/ut_range_ops/ya.make | 2 +- .../tx/datashard/ut_read_iterator/ya.make | 2 +- ydb/core/tx/datashard/ut_read_table/ya.make | 2 +- ydb/core/tx/datashard/ut_reassign/ya.make | 2 +- ydb/core/tx/datashard/ut_replication/ya.make | 2 +- .../tx/datashard/ut_reshuffle_kmeans/ya.make | 2 +- ydb/core/tx/datashard/ut_rs/ya.make | 2 +- ydb/core/tx/datashard/ut_sample_k/ya.make | 2 +- ydb/core/tx/datashard/ut_sequence/ya.make | 2 +- ydb/core/tx/datashard/ut_snapshot/ya.make | 2 +- ydb/core/tx/datashard/ut_stats/ya.make | 2 +- ydb/core/tx/datashard/ut_trace/ya.make | 2 +- ydb/core/tx/datashard/ut_upload_rows/ya.make | 2 +- ydb/core/tx/datashard/ut_volatile/ya.make | 2 +- ydb/core/tx/datashard/ut_write/ya.make | 2 +- ydb/core/tx/locks/ut_range_treap/ya.make | 2 +- .../replication/controller/private_events.h | 2 +- .../replication/controller/stream_creator.cpp | 6 +- .../controller/target_discoverer.cpp | 14 +- .../controller/tx_describe_replication.cpp | 2 +- ydb/core/tx/replication/controller/util.h | 2 +- .../replication/service/topic_reader_ut.cpp | 2 +- .../service/ut_topic_reader/ya.make | 2 +- .../tx/replication/service/ut_worker/ya.make | 2 +- ydb/core/tx/replication/service/worker_ut.cpp | 2 +- .../tx/replication/ut_helpers/write_topic.h | 2 +- ydb/core/tx/replication/ut_helpers/ya.make | 2 +- ydb/core/tx/replication/ydb_proxy/ut/ya.make | 2 +- ydb/core/tx/replication/ydb_proxy/ya.make | 12 +- .../tx/replication/ydb_proxy/ydb_proxy.cpp | 6 +- ydb/core/tx/replication/ydb_proxy/ydb_proxy.h | 6 +- .../tx/replication/ydb_proxy/ydb_proxy_ut.cpp | 4 +- ydb/core/tx/scheme_cache/ya.make | 1 + .../olap/operations/alter/abstract/ya.make | 1 + .../olap/operations/alter/common/ya.make | 1 + .../alter/in_store/transfer/ya.make | 1 + ydb/core/tx/schemeshard/olap/ttl/ya.make | 2 + .../tx/schemeshard/ut_helpers/helpers.cpp | 2 +- ydb/core/tx/schemeshard/ut_helpers/test_env.h | 2 +- ydb/core/tx/schemeshard/ut_helpers/ya.make | 4 +- .../schemeshard/ut_index/ut_async_index.cpp | 2 +- .../ut_index_build/ut_index_build.cpp | 2 +- .../ut_index_build/ut_vector_index_build.cpp | 2 +- .../tx/schemeshard/ut_index_build/ya.make | 2 +- .../ut_index_build_reboots/ya.make | 2 +- ydb/core/tx/tiering/manager.h | 2 +- ydb/core/tx/tiering/ut/ut_tiers.cpp | 2 +- .../tx/tx_proxy/upload_rows_common_impl.h | 2 +- ydb/core/tx/tx_proxy/ya.make | 4 + ydb/core/util/ya.make | 1 + ydb/core/viewer/json_local_rpc.h | 5 +- ydb/core/viewer/operation_list.h | 1 + ydb/core/viewer/viewer.h | 2 +- ydb/core/viewer/viewer_labeled_counters.h | 2 +- ydb/core/viewer/viewer_query.h | 14 +- ydb/core/viewer/viewer_query_old.h | 2 +- ydb/core/viewer/ya.make | 3 +- ydb/core/viewer/yaml/yaml.h | 1 + ydb/core/ydb_convert/ydb_convert.cpp | 2 +- ydb/core/ydb_convert/ydb_convert.h | 2 +- ydb/core/ymq/actor/auth_factory.h | 2 +- ydb/core/ymq/actor/auth_mocks.cpp | 2 +- ydb/core/ymq/actor/auth_multi_factory.cpp | 4 +- ydb/core/ymq/actor/auth_multi_factory.h | 2 +- ydb/core/ymq/actor/cleanup_queue_data.h | 2 +- ydb/core/ymq/actor/index_events_processor.cpp | 2 +- ydb/core/ymq/actor/index_events_processor.h | 12 +- ydb/core/ymq/actor/service.cpp | 2 +- ydb/core/ymq/actor/service.h | 2 +- ydb/core/ymq/actor/ya.make | 4 +- ydb/core/ymq/base/run_query.h | 2 +- ydb/core/yql_testlib/ya.make | 2 +- .../result_set_parquet_printer.cpp | 4 +- .../result_set_parquet_printer.h | 5 +- ydb/library/arrow_parquet/ya.make | 2 +- ydb/library/backup/backup.cpp | 30 +- ydb/library/backup/backup.h | 4 +- ydb/library/backup/db_iterator.h | 6 +- ydb/library/backup/query_builder.cpp | 4 +- ydb/library/backup/query_builder.h | 14 +- ydb/library/backup/query_uploader.h | 2 +- ydb/library/backup/ut/ut.cpp | 14 +- ydb/library/backup/ya.make | 14 +- ydb/library/db_pool/db_pool.cpp | 5 +- ydb/library/db_pool/db_pool.h | 2 +- ydb/library/db_pool/events.h | 6 +- ydb/library/db_pool/ya.make | 4 +- ydb/library/folder_service/events.h | 2 +- ydb/library/folder_service/mock/ya.make | 1 + ydb/library/formats/arrow/csv/table/table.cpp | 4 +- ydb/library/formats/arrow/csv/table/table.h | 4 +- ydb/library/formats/arrow/csv/table/ya.make | 2 +- .../grpc/actor_client/grpc_service_client.h | 2 +- ydb/library/grpc/actor_client/ya.make | 4 +- ydb/library/grpc/client/grpc_client_low.h | 1435 +---- ydb/library/grpc/client/grpc_common.h | 84 +- ydb/library/grpc/client/ut/ya.make | 7 - ydb/library/grpc/client/ya.make | 10 +- ydb/library/grpc/common/constants.h | 8 +- ydb/library/grpc/common/ya.make | 2 +- ydb/library/grpc/server/grpc_server.h | 2 +- ydb/library/ncloud/api/events.h | 2 +- ydb/library/ncloud/api/ya.make | 2 +- ydb/library/persqueue/obfuscate/obfuscate.h | 10 +- ydb/library/persqueue/obfuscate/ya.make | 5 +- .../persqueue/topic_parser/topic_parser.h | 2 +- ydb/library/persqueue/topic_parser/ya.make | 2 +- .../topic_parser_public/topic_parser.h | 34 +- .../persqueue/topic_parser_public/ya.make | 5 +- ydb/library/persqueue/ya.make | 1 - ydb/library/query_actor/query_actor.h | 8 +- ydb/library/query_actor/query_actor_ut.cpp | 6 +- ydb/library/query_actor/ya.make | 6 +- ydb/library/security/ya.make | 4 +- .../ydb_credentials_provider_factory.cpp | 2 +- .../ydb_credentials_provider_factory.h | 4 +- .../table_creator/table_creator_ut.cpp | 4 +- ydb/library/table_creator/ut/ya.make | 2 +- .../abstract/workload_query_generator.h | 8 +- ydb/library/workload/abstract/ya.make | 8 +- .../workload/clickbench/data_generator.cpp | 2 + ydb/library/workload/kv/kv.h | 4 +- ydb/library/ycloud/api/events.h | 2 +- ydb/library/ycloud/api/ya.make | 2 +- .../client/bearer_credentials_provider.cpp | 4 +- .../client/bearer_credentials_provider.h | 2 +- .../common/token_accessor/client/factory.h | 2 +- .../client/token_accessor_client.cpp | 4 +- .../client/token_accessor_client.h | 4 +- .../client/token_accessor_client_factory.h | 4 +- .../common/token_accessor/client/ya.make | 4 +- .../yql/providers/dq/actors/yt/ya.make | 2 +- .../global_worker_manager.cpp | 2 +- .../service_node_resolver.cpp | 2 +- .../service_node_resolver.h | 2 +- .../yql/providers/dq/provider/exec/ya.make | 2 +- ydb/library/yql/providers/dq/provider/ya.make | 4 +- .../providers/dq/provider/yql_dq_control.cpp | 2 +- .../providers/dq/provider/yql_dq_gateway.cpp | 4 +- .../providers/dq/provider/yql_dq_gateway.h | 2 +- .../yql/providers/generic/actors/ya.make | 2 +- .../generic/actors/yql_generic_read_actor.cpp | 2 +- .../actors/yql_generic_token_provider.cpp | 4 +- .../generic/connector/libcpp/client.h | 2 +- .../generic/connector/libcpp/error.cpp | 2 +- .../generic/connector/libcpp/error.h | 2 +- .../libcpp/ut_helpers/connector_client_mock.h | 4 +- .../connector/libcpp/ut_helpers/test_creds.h | 4 +- .../generic/connector/libcpp/ya.make | 2 +- .../yql/providers/generic/provider/ya.make | 2 +- .../generic/provider/yql_generic_state.h | 2 +- .../pq/async_io/dq_pq_meta_extractor.cpp | 6 +- .../pq/async_io/dq_pq_meta_extractor.h | 2 +- .../pq/async_io/dq_pq_rd_read_actor.cpp | 4 +- .../pq/async_io/dq_pq_rd_read_actor.h | 2 +- .../pq/async_io/dq_pq_read_actor.cpp | 20 +- .../providers/pq/async_io/dq_pq_read_actor.h | 2 +- .../pq/async_io/dq_pq_write_actor.cpp | 4 +- .../providers/pq/async_io/dq_pq_write_actor.h | 2 +- ydb/library/yql/providers/pq/async_io/ya.make | 7 +- .../yql/providers/pq/cm_client/client.h | 2 +- .../yql/providers/pq/cm_client/ya.make | 2 +- .../pq/gateway/dummy/yql_pq_blocking_queue.h | 4 +- .../dummy/yql_pq_file_topic_client.cpp | 54 +- .../yql/providers/pq/gateway/native/ya.make | 6 +- .../pq/gateway/native/yql_pq_gateway.cpp | 2 +- .../pq/gateway/native/yql_pq_gateway.h | 2 +- .../pq/gateway/native/yql_pq_session.h | 4 +- .../yql/providers/pq/provider/ut/ya.make | 4 +- .../providers/pq/provider/ut/yql_pq_ut.cpp | 4 +- ydb/library/yql/providers/pq/provider/ya.make | 2 +- .../providers/pq/provider/yql_pq_gateway.h | 2 +- .../pq/provider/yql_pq_topic_client.h | 2 +- .../providers/s3/credentials/credentials.h | 2 +- .../yql/providers/s3/credentials/ya.make | 2 +- ydb/library/yql/providers/ydb/actors/ya.make | 3 +- .../ydb/actors/yql_ydb_read_actor.cpp | 3 +- .../providers/ydb/actors/yql_ydb_read_actor.h | 2 +- .../ydb/actors/yql_ydb_source_factory.h | 2 +- .../yql/providers/ydb/comp_nodes/ya.make | 3 +- .../providers/ydb/comp_nodes/yql_kik_scan.cpp | 3 +- .../providers/ydb/comp_nodes/yql_kik_scan.h | 2 +- .../ydb/comp_nodes/yql_ydb_factory.h | 2 +- .../yql/providers/ydb/provider/ya.make | 5 +- .../ydb/provider/yql_ydb_load_meta.cpp | 5 +- .../providers/ydb/provider/yql_ydb_provider.h | 2 +- .../ydb/provider/yql_ydb_provider_impl.h | 2 +- ydb/library/yql/tools/dq/dq_cli/main.cpp | 8 +- ydb/library/yql/tools/dq/dq_cli/ya.make | 2 +- ydb/library/yql/tools/dq/worker_node/main.cpp | 2 +- ydb/library/yql/tools/dq/worker_node/ya.make | 2 +- ydb/library/yql/tools/dqrun/dqrun.cpp | 2 +- ydb/library/yql/tools/dqrun/ya.make | 2 +- ydb/library/yql/tools/mrrun/mrrun.cpp | 6 +- ydb/library/yql/tools/mrrun/ya.make | 4 +- ydb/mvp/core/core_ydb.h | 20 +- ydb/mvp/core/core_ydb_impl.h | 42 +- ydb/mvp/core/grpc_log.h | 2 +- ydb/mvp/core/mvp_tokens.h | 2 +- ydb/mvp/core/ya.make | 14 +- ydb/mvp/meta/meta.cpp | 4 +- ydb/mvp/meta/meta_cloud.h | 6 +- ydb/mvp/meta/meta_cluster.h | 6 +- ydb/mvp/meta/meta_clusters.h | 12 +- ydb/mvp/meta/meta_cp_databases.h | 6 +- ydb/mvp/meta/meta_db_clusters.h | 6 +- ydb/mvp/meta/meta_versions.cpp | 6 +- ydb/mvp/meta/ya.make | 1 + .../oidc_proxy/oidc_session_create_yandex.cpp | 2 +- ydb/mvp/oidc_proxy/openid_connect.h | 2 +- ydb/mvp/oidc_proxy/ya.make | 1 + .../lib/deprecated/client/grpc_client.h | 2 +- ydb/public/lib/deprecated/client/ya.make | 2 +- .../lib/deprecated/json_value/ut/ya.make | 18 + ydb/public/lib/deprecated/json_value/ya.make | 20 + .../deprecated/json_value/ydb_json_value.cpp | 1105 ++++ .../deprecated/json_value/ydb_json_value.h | 42 + .../json_value/ydb_json_value_ut.cpp | 697 +++ ydb/public/lib/deprecated/kicli/kikimr.cpp | 4 +- ydb/public/lib/deprecated/kicli/ya.make | 1 + ydb/public/lib/deprecated/ya.make | 2 + ydb/public/lib/deprecated/yson_value/ya.make | 15 + .../deprecated/yson_value/ydb_yson_value.cpp | 277 + .../deprecated/yson_value/ydb_yson_value.h | 21 + ydb/public/lib/experimental/ya.make | 8 +- .../experimental/ydb_clickhouse_internal.cpp | 6 +- .../experimental/ydb_clickhouse_internal.h | 2 +- ydb/public/lib/experimental/ydb_logstore.cpp | 10 +- ydb/public/lib/experimental/ydb_logstore.h | 8 +- .../lib/experimental/ydb_object_storage.cpp | 10 +- .../lib/experimental/ydb_object_storage.h | 4 +- ydb/public/lib/fq/fq.cpp | 4 +- ydb/public/lib/fq/fq.h | 2 +- ydb/public/lib/fq/ya.make | 6 +- ydb/public/lib/idx_test/idx_test.h | 2 +- ydb/public/lib/idx_test/idx_test_checker.cpp | 26 +- ydb/public/lib/idx_test/idx_test_common.cpp | 6 +- ydb/public/lib/idx_test/idx_test_common.h | 16 +- .../lib/idx_test/idx_test_data_provider.cpp | 2 +- .../lib/idx_test/idx_test_data_provider.h | 6 +- ydb/public/lib/idx_test/idx_test_loader.cpp | 64 +- ydb/public/lib/idx_test/idx_test_upload.cpp | 8 +- ydb/public/lib/idx_test/ya.make | 2 +- ydb/public/lib/json_value/ut/ya.make | 4 +- ydb/public/lib/json_value/ya.make | 4 +- ydb/public/lib/json_value/ydb_json_value.cpp | 10 +- ydb/public/lib/json_value/ydb_json_value.h | 10 +- .../lib/json_value/ydb_json_value_ut.cpp | 4 +- ydb/public/lib/jwt/jwt.h | 22 +- ydb/public/lib/jwt/ya.make | 5 +- ydb/public/lib/operation_id/operation_id.cpp | 203 - ydb/public/lib/operation_id/operation_id.h | 33 +- ydb/public/lib/operation_id/ya.make | 11 +- .../lib/ut_helpers/ut_helpers_query.cpp | 4 +- ydb/public/lib/ut_helpers/ut_helpers_query.h | 2 +- ydb/public/lib/ut_helpers/ya.make | 2 +- ydb/public/lib/value/value.cpp | 2 +- ydb/public/lib/value/ya.make | 2 +- .../lib/ydb_cli/commands/benchmark_utils.cpp | 10 +- .../lib/ydb_cli/commands/benchmark_utils.h | 5 +- .../lib/ydb_cli/commands/command_base/ya.make | 4 +- .../commands/interactive/interactive_cli.cpp | 2 +- .../lib/ydb_cli/commands/query_workload.cpp | 6 +- .../ydb_cli/commands/sdk_core_access/ya.make | 8 +- .../commands/topic_operations_scenario.cpp | 16 +- .../commands/topic_operations_scenario.h | 4 +- .../topic_workload/topic_workload_clean.cpp | 2 +- .../topic_workload/topic_workload_describe.h | 2 +- .../topic_workload/topic_workload_init.cpp | 2 +- .../topic_workload/topic_workload_params.cpp | 4 +- .../topic_workload/topic_workload_reader.cpp | 16 +- .../topic_workload/topic_workload_reader.h | 8 +- ...opic_workload_reader_transaction_support.h | 2 +- .../topic_workload/topic_workload_writer.h | 2 +- .../topic_workload_writer_producer.cpp | 4 +- .../topic_workload_writer_producer.h | 2 +- .../ut/topic_workload_writer_producer_ut.cpp | 16 +- .../ydb_cli/commands/topic_workload/ya.make | 17 +- .../transfer_workload_topic_to_table_run.cpp | 2 +- .../commands/transfer_workload/ya.make | 22 +- ydb/public/lib/ydb_cli/commands/ya.make | 35 +- .../lib/ydb_cli/commands/ydb_cluster.cpp | 4 +- .../lib/ydb_cli/commands/ydb_command.cpp | 2 +- ydb/public/lib/ydb_cli/commands/ydb_command.h | 6 +- ydb/public/lib/ydb_cli/commands/ydb_common.h | 36 +- .../ydb_cli/commands/ydb_discovery/ya.make | 2 +- .../ydb_cli/commands/ydb_dynamic_config.cpp | 52 +- .../lib/ydb_cli/commands/ydb_dynamic_config.h | 2 +- .../lib/ydb_cli/commands/ydb_latency.cpp | 4 +- ydb/public/lib/ydb_cli/commands/ydb_ping.cpp | 4 +- ydb/public/lib/ydb_cli/commands/ydb_ping.h | 4 +- .../lib/ydb_cli/commands/ydb_profile.cpp | 14 +- .../lib/ydb_cli/commands/ydb_root_common.cpp | 26 +- .../ydb_cli/commands/ydb_sdk_core_access.cpp | 2 +- .../ydb_cli/commands/ydb_sdk_core_access.h | 4 +- .../lib/ydb_cli/commands/ydb_service_auth.cpp | 2 +- .../commands/ydb_service_discovery.cpp | 16 +- .../ydb_cli/commands/ydb_service_discovery.h | 2 +- .../ydb_cli/commands/ydb_service_export.cpp | 4 +- .../lib/ydb_cli/commands/ydb_service_export.h | 2 +- .../ydb_cli/commands/ydb_service_import.cpp | 6 +- .../lib/ydb_cli/commands/ydb_service_import.h | 4 +- .../commands/ydb_service_monitoring.cpp | 4 +- .../ydb_cli/commands/ydb_service_monitoring.h | 2 +- .../commands/ydb_service_operation.cpp | 22 +- .../ydb_cli/commands/ydb_service_operation.h | 4 +- .../ydb_cli/commands/ydb_service_scheme.cpp | 140 +- .../lib/ydb_cli/commands/ydb_service_scheme.h | 32 +- .../commands/ydb_service_scripting.cpp | 18 +- .../ydb_cli/commands/ydb_service_scripting.h | 2 +- .../ydb_cli/commands/ydb_service_table.cpp | 90 +- .../lib/ydb_cli/commands/ydb_service_table.h | 16 +- .../ydb_cli/commands/ydb_service_topic.cpp | 32 +- .../lib/ydb_cli/commands/ydb_service_topic.h | 4 +- ydb/public/lib/ydb_cli/commands/ydb_sql.cpp | 18 +- ydb/public/lib/ydb_cli/commands/ydb_sql.h | 2 +- .../ydb_cli/commands/ydb_storage_config.cpp | 10 +- ydb/public/lib/ydb_cli/commands/ydb_tools.cpp | 10 +- .../lib/ydb_cli/commands/ydb_workload.cpp | 8 +- .../lib/ydb_cli/commands/ydb_workload.h | 4 +- .../ydb_cli/commands/ydb_workload_import.cpp | 22 +- ydb/public/lib/ydb_cli/commands/ydb_yql.cpp | 10 +- ydb/public/lib/ydb_cli/commands/ydb_yql.h | 2 +- ydb/public/lib/ydb_cli/common/aws.cpp | 2 +- ydb/public/lib/ydb_cli/common/aws.h | 2 +- ydb/public/lib/ydb_cli/common/command.h | 4 +- ydb/public/lib/ydb_cli/common/csv_parser.cpp | 18 +- ydb/public/lib/ydb_cli/common/csv_parser.h | 8 +- .../lib/ydb_cli/common/csv_parser_ut.cpp | 9 +- ydb/public/lib/ydb_cli/common/format.cpp | 6 +- ydb/public/lib/ydb_cli/common/format.h | 4 +- ydb/public/lib/ydb_cli/common/interactive.h | 2 +- ydb/public/lib/ydb_cli/common/interruptible.h | 4 +- ydb/public/lib/ydb_cli/common/parameters.cpp | 2 +- ydb/public/lib/ydb_cli/common/parameters.h | 4 +- .../lib/ydb_cli/common/print_operation.cpp | 34 +- .../lib/ydb_cli/common/print_operation.h | 10 +- ydb/public/lib/ydb_cli/common/print_utils.h | 2 +- ydb/public/lib/ydb_cli/common/query_stats.h | 4 +- .../lib/ydb_cli/common/recursive_list.h | 2 +- .../lib/ydb_cli/common/recursive_remove.cpp | 8 +- .../lib/ydb_cli/common/recursive_remove.h | 8 +- ydb/public/lib/ydb_cli/common/retry_func.h | 2 +- .../lib/ydb_cli/common/scheme_printers.cpp | 18 +- .../lib/ydb_cli/common/scheme_printers.h | 4 +- ydb/public/lib/ydb_cli/common/sys.cpp | 8 +- ydb/public/lib/ydb_cli/common/sys.h | 2 + .../lib/ydb_cli/common/tabbed_table.cpp | 2 +- ydb/public/lib/ydb_cli/common/tabbed_table.h | 6 +- ydb/public/lib/ydb_cli/common/ya.make | 20 +- ydb/public/lib/ydb_cli/dump/dump.cpp | 2 +- ydb/public/lib/ydb_cli/dump/dump.h | 8 +- ydb/public/lib/ydb_cli/dump/restore_impl.cpp | 40 +- ydb/public/lib/ydb_cli/dump/restore_impl.h | 14 +- .../lib/ydb_cli/dump/restore_import_data.cpp | 24 +- ydb/public/lib/ydb_cli/dump/util/util.h | 18 +- ydb/public/lib/ydb_cli/dump/util/ya.make | 6 +- ydb/public/lib/ydb_cli/dump/ya.make | 6 +- ydb/public/lib/ydb_cli/import/import.cpp | 26 +- ydb/public/lib/ydb_cli/import/import.h | 8 +- ydb/public/lib/ydb_cli/import/ya.make | 2 +- ydb/public/lib/ydb_cli/topic/topic_read.cpp | 6 +- ydb/public/lib/ydb_cli/topic/topic_read.h | 2 +- .../lib/ydb_cli/topic/topic_read_ut.cpp | 4 +- ydb/public/lib/ydb_cli/topic/topic_write.cpp | 12 +- ydb/public/lib/ydb_cli/topic/topic_write.h | 2 +- ydb/public/lib/ydb_cli/topic/ut/ya.make | 10 +- ydb/public/lib/ydb_cli/topic/ya.make | 6 +- ydb/public/lib/yson_value/ya.make | 4 +- ydb/public/lib/yson_value/ydb_yson_value.cpp | 4 +- ydb/public/lib/yson_value/ydb_yson_value.h | 6 +- ydb/public/sdk/cpp/CMakeLists.txt | 99 + ydb/public/sdk/cpp/adapters/issue/issue.cpp | 40 + ydb/public/sdk/cpp/adapters/issue/issue.h | 13 + ydb/public/sdk/cpp/adapters/issue/ya.make | 15 + ydb/public/sdk/cpp/adapters/ya.make | 3 + ydb/public/sdk/cpp/client/README.md | 3 + .../cpp/client/draft/ut/helpers/grpc_server.h | 2 +- .../ut/helpers/grpc_services/scripting.cpp | 2 +- .../ut/helpers/grpc_services/scripting.h | 2 +- .../draft/ut/helpers/grpc_services/view.cpp | 2 +- .../draft/ut/helpers/grpc_services/view.h | 2 +- .../ut/ydb_scripting_response_headers_ut.cpp | 2 +- .../sdk/cpp/client/draft/ut/ydb_view_ut.cpp | 2 +- .../cpp/client/draft/ydb_dynamic_config.cpp | 2 +- .../sdk/cpp/client/draft/ydb_dynamic_config.h | 2 +- .../sdk/cpp/client/draft/ydb_replication.cpp | 2 +- .../sdk/cpp/client/draft/ydb_replication.h | 6 +- .../sdk/cpp/client/draft/ydb_scripting.cpp | 2 +- .../sdk/cpp/client/draft/ydb_scripting.h | 8 +- ydb/public/sdk/cpp/client/draft/ydb_view.cpp | 2 +- ydb/public/sdk/cpp/client/draft/ydb_view.h | 4 +- .../extensions/solomon_stats/pull_client.cpp | 2 +- .../extensions/solomon_stats/pull_client.h | 2 +- .../extensions/solomon_stats/pull_connector.h | 2 +- ydb/public/sdk/cpp/client/helpers/helpers.cpp | 2 +- ydb/public/sdk/cpp/client/helpers/helpers.h | 2 +- ydb/public/sdk/cpp/client/iam/common/iam.cpp | 2 +- ydb/public/sdk/cpp/client/iam/common/iam.h | 2 +- ydb/public/sdk/cpp/client/iam/iam.cpp | 2 +- ydb/public/sdk/cpp/client/iam_private/iam.cpp | 2 +- ydb/public/sdk/cpp/client/iam_private/iam.h | 2 +- .../client/impl/ydb_endpoints/endpoints.cpp | 2 +- .../cpp/client/impl/ydb_endpoints/endpoints.h | 2 +- .../sdk/cpp/client/impl/ydb_endpoints/ya.make | 1 + .../impl/ydb_internal/common/client_pid.cpp | 2 +- .../impl/ydb_internal/common/client_pid.h | 2 +- .../impl/ydb_internal/common/getenv.cpp | 2 +- .../client/impl/ydb_internal/common/getenv.h | 2 +- .../impl/ydb_internal/common/parser.cpp | 2 +- .../client/impl/ydb_internal/common/parser.h | 2 +- .../ydb_internal/common/ssl_credentials.h | 2 +- .../ydb_internal/common/string_helpers.cpp | 2 +- .../impl/ydb_internal/common/string_helpers.h | 2 +- .../impl/ydb_internal/common/type_switcher.h | 2 +- .../client/impl/ydb_internal/common/types.h | 2 +- .../db_driver_state/authenticator.cpp | 2 +- .../db_driver_state/authenticator.h | 2 +- .../db_driver_state/endpoint_pool.cpp | 2 +- .../db_driver_state/endpoint_pool.h | 2 +- .../ydb_internal/db_driver_state/state.cpp | 2 +- .../impl/ydb_internal/db_driver_state/state.h | 2 +- .../impl/ydb_internal/driver/constants.h | 2 +- .../ydb_internal/grpc_connections/actions.cpp | 2 +- .../ydb_internal/grpc_connections/actions.h | 2 +- .../grpc_connections/grpc_connections.cpp | 2 +- .../grpc_connections/grpc_connections.h | 2 +- .../ydb_internal/grpc_connections/params.h | 2 +- .../ydb_internal/internal_client/client.h | 2 +- .../kqp_session_common/kqp_session_common.cpp | 2 +- .../kqp_session_common/kqp_session_common.h | 2 +- .../ydb_internal/kqp_session_common/ya.make | 2 +- .../client/impl/ydb_internal/logger/log.cpp | 2 +- .../cpp/client/impl/ydb_internal/logger/log.h | 2 +- .../impl/ydb_internal/make_request/make.cpp | 2 +- .../impl/ydb_internal/make_request/make.h | 2 +- .../impl/ydb_internal/plain_status/status.cpp | 4 +- .../impl/ydb_internal/plain_status/status.h | 2 +- .../client/impl/ydb_internal/retry/retry.cpp | 2 +- .../client/impl/ydb_internal/retry/retry.h | 4 +- .../impl/ydb_internal/retry/retry_async.h | 2 +- .../impl/ydb_internal/retry/retry_sync.h | 2 +- .../rpc_request_settings/settings.h | 2 +- .../ydb_internal/scheme_helpers/helpers.h | 2 +- .../session_client/session_client.h | 2 +- .../session_pool/session_pool.cpp | 2 +- .../ydb_internal/session_pool/session_pool.h | 2 +- .../ydb_internal/stats_extractor/extractor.h | 2 +- .../impl/ydb_internal/table_helpers/helpers.h | 2 +- .../impl/ydb_internal/thread_pool/pool.h | 2 +- .../ydb_internal/value_helpers/helpers.cpp | 2 +- .../impl/ydb_internal/value_helpers/helpers.h | 2 +- .../sdk/cpp/client/impl/ydb_stats/stats.cpp | 2 +- .../sdk/cpp/client/impl/ydb_stats/stats.h | 2 +- .../sdk/cpp/client/resources/ydb_ca.cpp | 2 +- ydb/public/sdk/cpp/client/resources/ydb_ca.h | 2 +- .../cpp/client/resources/ydb_resources.cpp | 2 +- .../sdk/cpp/client/resources/ydb_resources.h | 2 +- ydb/public/sdk/cpp/client/ya.make | 56 + .../client/ydb_common_client/impl/client.h | 2 +- .../cpp/client/ydb_common_client/impl/iface.h | 2 +- .../cpp/client/ydb_common_client/settings.cpp | 2 +- .../cpp/client/ydb_common_client/settings.h | 12 +- .../client/ydb_coordination/coordination.cpp | 2 +- .../client/ydb_coordination/coordination.h | 52 +- .../ydb_coordination/coordination_ut.cpp | 2 +- .../ydb_coordination/proto_accessor.cpp | 2 +- .../client/ydb_datastreams/datastreams.cpp | 2 +- .../cpp/client/ydb_datastreams/datastreams.h | 64 +- .../sdk/cpp/client/ydb_debug/client.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_debug/client.h | 2 +- .../cpp/client/ydb_discovery/discovery.cpp | 2 +- .../sdk/cpp/client/ydb_discovery/discovery.h | 20 +- .../sdk/cpp/client/ydb_driver/driver.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_driver/driver.h | 4 +- ydb/public/sdk/cpp/client/ydb_driver/fwd.h | 8 + .../sdk/cpp/client/ydb_export/export.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_export/export.h | 26 +- .../cpp/client/ydb_extension/extension.cpp | 2 +- .../sdk/cpp/client/ydb_extension/extension.h | 2 +- .../ydb_federated_topic/federated_topic.h | 38 +- .../impl/federated_read_session.cpp | 2 +- .../impl/federated_read_session.h | 2 +- .../impl/federated_read_session_event.cpp | 4 +- .../impl/federated_topic.cpp | 2 +- .../impl/federated_topic_impl.cpp | 2 +- .../impl/federated_topic_impl.h | 2 +- .../impl/federated_write_session.cpp | 2 +- .../impl/federated_write_session.h | 2 +- .../impl/federation_observer.cpp | 2 +- .../impl/federation_observer.h | 2 +- .../ydb_federated_topic/ut/fds_mock/ya.make | 8 - .../cpp/client/ydb_federated_topic/ut/ya.make | 37 - .../cpp/client/ydb_federated_topic/ya.make | 4 - .../sdk/cpp/client/ydb_import/import.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_import/import.h | 14 +- .../cpp/client/ydb_monitoring/monitoring.cpp | 2 +- .../cpp/client/ydb_monitoring/monitoring.h | 8 +- .../cpp/client/ydb_operation/operation.cpp | 8 +- .../sdk/cpp/client/ydb_operation/operation.h | 2 +- ydb/public/sdk/cpp/client/ydb_params/fwd.h | 9 + ydb/public/sdk/cpp/client/ydb_params/impl.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_params/impl.h | 2 +- .../sdk/cpp/client/ydb_params/params.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_params/params.h | 6 +- ydb/public/sdk/cpp/client/ydb_params/ya.make | 4 - .../ut/ut_utils/data_plane_helpers.h | 3 - .../ut/ut_utils/test_server.h | 3 - .../ydb_persqueue_core/ut/ut_utils/ut_utils.h | 3 - .../ydb_persqueue_core/ut/ut_utils/ya.make | 13 - .../cpp/client/ydb_persqueue_core/ut/ya.make | 41 - .../sdk/cpp/client/ydb_persqueue_core/ya.make | 4 - .../ydb_persqueue_public/impl/aliases.h | 2 +- .../ydb_persqueue_public/impl/common.cpp | 2 +- .../client/ydb_persqueue_public/impl/common.h | 2 +- .../ydb_persqueue_public/impl/persqueue.cpp | 2 +- .../impl/persqueue_impl.cpp | 2 +- .../impl/persqueue_impl.h | 2 +- .../impl/read_session.cpp | 2 +- .../ydb_persqueue_public/impl/read_session.h | 2 +- .../impl/read_session_messages.cpp | 2 +- .../impl/write_session.cpp | 2 +- .../ydb_persqueue_public/impl/write_session.h | 2 +- .../impl/write_session_impl.cpp | 2 +- .../impl/write_session_impl.h | 2 +- .../ydb_persqueue_public/include/aliases.h | 2 +- .../ydb_persqueue_public/include/client.h | 8 +- .../include/control_plane.h | 68 +- .../include/read_events.h | 2 +- .../include/read_session.h | 60 +- .../include/write_events.h | 2 +- .../include/write_session.h | 48 +- .../cpp/client/ydb_persqueue_public/ya.make | 4 - .../sdk/cpp/client/ydb_proto/accessor.cpp | 2 +- .../sdk/cpp/client/ydb_proto/accessor.h | 2 +- ydb/public/sdk/cpp/client/ydb_proto/ya.make | 2 +- .../sdk/cpp/client/ydb_query/client.cpp | 4 +- ydb/public/sdk/cpp/client/ydb_query/client.h | 14 +- ydb/public/sdk/cpp/client/ydb_query/fwd.h | 37 + .../client/ydb_query/impl/client_session.cpp | 2 +- .../client/ydb_query/impl/client_session.h | 2 +- .../cpp/client/ydb_query/impl/exec_query.cpp | 2 +- .../cpp/client/ydb_query/impl/exec_query.h | 2 +- ydb/public/sdk/cpp/client/ydb_query/query.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_query/query.h | 30 +- ydb/public/sdk/cpp/client/ydb_query/stats.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_query/stats.h | 8 +- ydb/public/sdk/cpp/client/ydb_query/tx.h | 10 +- .../client/ydb_rate_limiter/rate_limiter.cpp | 2 +- .../client/ydb_rate_limiter/rate_limiter.h | 16 +- ydb/public/sdk/cpp/client/ydb_result/fwd.h | 8 + .../cpp/client/ydb_result/proto_accessor.cpp | 2 +- .../sdk/cpp/client/ydb_result/result.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_result/result.h | 6 +- ydb/public/sdk/cpp/client/ydb_retry/retry.h | 26 +- .../sdk/cpp/client/ydb_scheme/scheme.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_scheme/scheme.h | 2 +- .../sdk/cpp/client/ydb_ss_tasks/task.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_ss_tasks/task.h | 2 +- ydb/public/sdk/cpp/client/ydb_table/fwd.h | 116 + .../client/ydb_table/impl/client_session.cpp | 2 +- .../client/ydb_table/impl/client_session.h | 2 +- .../cpp/client/ydb_table/impl/data_query.cpp | 2 +- .../cpp/client/ydb_table/impl/data_query.h | 2 +- .../sdk/cpp/client/ydb_table/impl/readers.cpp | 2 +- .../sdk/cpp/client/ydb_table/impl/readers.h | 2 +- .../ydb_table/impl/request_migrator.cpp | 2 +- .../client/ydb_table/impl/request_migrator.h | 2 +- .../client/ydb_table/impl/table_client.cpp | 2 +- .../cpp/client/ydb_table/impl/table_client.h | 2 +- .../cpp/client/ydb_table/impl/transaction.cpp | 2 +- .../cpp/client/ydb_table/impl/transaction.h | 2 +- .../sdk/cpp/client/ydb_table/impl/ya.make | 2 +- .../cpp/client/ydb_table/proto_accessor.cpp | 2 +- .../client/ydb_table/query_stats/stats.cpp | 2 +- .../cpp/client/ydb_table/query_stats/stats.h | 2 +- ydb/public/sdk/cpp/client/ydb_table/table.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_table/table.h | 166 +- .../sdk/cpp/client/ydb_table/table_enum.h | 4 +- .../ydb_topic/common/callback_context.h | 2 +- .../client/ydb_topic/common/executor_impl.cpp | 2 +- .../client/ydb_topic/common/executor_impl.h | 2 +- .../client/ydb_topic/common/retry_policy.cpp | 2 +- .../sdk/cpp/client/ydb_topic/impl/common.cpp | 2 +- .../sdk/cpp/client/ydb_topic/impl/common.h | 2 +- .../client/ydb_topic/impl/counters_logger.h | 2 +- .../client/ydb_topic/impl/deferred_commit.cpp | 2 +- .../client/ydb_topic/impl/event_handlers.cpp | 2 +- .../ydb_topic/impl/offsets_collector.cpp | 2 +- .../client/ydb_topic/impl/offsets_collector.h | 2 +- .../client/ydb_topic/impl/read_session.cpp | 2 +- .../cpp/client/ydb_topic/impl/read_session.h | 2 +- .../ydb_topic/impl/read_session_event.cpp | 2 +- .../client/ydb_topic/impl/read_session_impl.h | 2 +- .../ydb_topic/impl/read_session_impl.ipp | 2 +- .../sdk/cpp/client/ydb_topic/impl/topic.cpp | 2 +- .../cpp/client/ydb_topic/impl/topic_impl.cpp | 2 +- .../cpp/client/ydb_topic/impl/topic_impl.h | 2 +- .../cpp/client/ydb_topic/impl/transaction.cpp | 2 +- .../cpp/client/ydb_topic/impl/transaction.h | 4 +- .../client/ydb_topic/impl/write_session.cpp | 2 +- .../cpp/client/ydb_topic/impl/write_session.h | 2 +- .../ydb_topic/impl/write_session_impl.cpp | 2 +- .../ydb_topic/impl/write_session_impl.h | 2 +- .../sdk/cpp/client/ydb_topic/include/client.h | 6 +- .../sdk/cpp/client/ydb_topic/include/codecs.h | 2 +- .../client/ydb_topic/include/control_plane.h | 86 +- .../cpp/client/ydb_topic/include/counters.h | 2 +- .../sdk/cpp/client/ydb_topic/include/errors.h | 2 +- .../client/ydb_topic/include/events_common.h | 2 +- .../cpp/client/ydb_topic/include/executor.h | 2 +- .../client/ydb_topic/include/read_events.h | 2 +- .../client/ydb_topic/include/read_session.h | 66 +- .../client/ydb_topic/include/retry_policy.h | 2 +- .../client/ydb_topic/include/write_events.h | 2 +- .../client/ydb_topic/include/write_session.h | 56 +- .../cpp/client/ydb_topic/proto_accessor.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_topic/ya.make | 4 - .../ydb_types/core_facility/core_facility.h | 2 +- .../ydb_types/credentials/credentials.cpp | 2 +- .../ydb_types/credentials/credentials.h | 4 +- .../ydb_types/credentials/login/login.cpp | 2 +- .../oauth2_token_exchange/credentials.cpp | 2 +- .../oauth2_token_exchange/credentials.h | 31 +- .../oauth2_token_exchange/from_file.cpp | 12 +- .../oauth2_token_exchange/from_file.h | 5 +- .../jwt_token_source.cpp | 2 +- .../oauth2_token_exchange/jwt_token_source.h | 17 +- .../ut/jwt_check_helper.h | 12 +- .../ydb_types/exceptions/exceptions.cpp | 2 +- .../client/ydb_types/exceptions/exceptions.h | 4 +- .../fatal_error_handlers/handlers.cpp | 2 +- .../ydb_types/fatal_error_handlers/handlers.h | 2 +- .../ydb_types/fluent_settings_helpers.h | 14 +- ydb/public/sdk/cpp/client/ydb_types/fwd.h | 34 + .../client/ydb_types/operation/operation.cpp | 2 +- .../client/ydb_types/operation/operation.h | 4 +- .../cpp/client/ydb_types/request_settings.h | 22 +- .../sdk/cpp/client/ydb_types/s3_settings.h | 16 +- .../cpp/client/ydb_types/status/status.cpp | 24 +- .../sdk/cpp/client/ydb_types/status/status.h | 28 +- .../sdk/cpp/client/ydb_types/status_codes.h | 4 +- ydb/public/sdk/cpp/client/ydb_types/ydb.h | 4 +- ydb/public/sdk/cpp/client/ydb_value/fwd.h | 15 + ydb/public/sdk/cpp/client/ydb_value/value.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_value/value.h | 4 +- ydb/public/sdk/cpp/client/ydb_value/ya.make | 4 - ydb/public/sdk/cpp/examples/CMakeLists.txt | 8 + .../cpp/examples/basic_example/CMakeLists.txt | 38 + .../examples/basic_example/basic_example.cpp | 136 +- .../examples/basic_example/basic_example.h | 10 +- .../basic_example/basic_example_data.cpp | 32 +- .../sdk/cpp/examples/basic_example/main.cpp | 21 +- .../sdk/cpp/examples/basic_example/ya.make | 8 +- .../bulk_upsert_simple/CMakeLists.txt | 34 + .../cpp/examples/bulk_upsert_simple/main.cpp | 65 +- .../cpp/examples/bulk_upsert_simple/ya.make | 4 +- .../cpp/examples/pagination/CMakeLists.txt | 36 + .../sdk/cpp/examples/pagination/main.cpp | 9 +- .../cpp/examples/pagination/pagination.cpp | 83 +- .../sdk/cpp/examples/pagination/pagination.h | 6 +- .../examples/pagination/pagination_data.cpp | 10 +- .../sdk/cpp/examples/pagination/ya.make | 4 +- .../examples/secondary_index/CMakeLists.txt | 41 + .../sdk/cpp/examples/secondary_index/main.cpp | 17 +- .../secondary_index/secondary_index.cpp | 12 +- .../secondary_index/secondary_index.h | 61 +- .../secondary_index_create.cpp | 9 +- .../secondary_index_delete.cpp | 19 +- .../secondary_index/secondary_index_drop.cpp | 7 +- .../secondary_index_generate.cpp | 60 +- .../secondary_index/secondary_index_list.cpp | 87 +- .../secondary_index_update.cpp | 17 +- .../sdk/cpp/examples/secondary_index/ya.make | 4 +- .../secondary_index_builtin/CMakeLists.txt | 40 + .../examples/secondary_index_builtin/main.cpp | 15 +- .../secondary_index.cpp | 12 +- .../secondary_index_builtin/secondary_index.h | 74 +- .../secondary_index_create.cpp | 5 +- .../secondary_index_drop.cpp | 5 +- .../secondary_index_fill.cpp | 24 +- .../secondary_index_select.cpp | 21 +- .../secondary_index_select_join.cpp | 21 +- .../examples/secondary_index_builtin/ya.make | 4 +- .../cpp/examples/topic_reader/CMakeLists.txt | 3 + .../topic_reader/eventloop/CMakeLists.txt | 34 + .../examples/topic_reader/eventloop/main.cpp | 31 +- .../examples/topic_reader/eventloop/ya.make | 4 +- .../topic_reader/simple/CMakeLists.txt | 34 + .../cpp/examples/topic_reader/simple/main.cpp | 25 +- .../cpp/examples/topic_reader/simple/ya.make | 4 +- .../topic_reader/transaction/CMakeLists.txt | 36 + .../topic_reader/transaction/application.cpp | 17 +- .../topic_reader/transaction/application.h | 18 +- .../topic_reader/transaction/main.cpp | 10 +- .../topic_reader/transaction/options.h | 13 +- .../examples/topic_reader/transaction/ya.make | 4 +- .../topic_writer/transaction/main.cpp | 19 +- .../sdk/cpp/examples/ttl/CMakeLists.txt | 35 + ydb/public/sdk/cpp/examples/ttl/main.cpp | 10 +- ydb/public/sdk/cpp/examples/ttl/ttl.cpp | 146 +- ydb/public/sdk/cpp/examples/ttl/ttl.h | 6 +- ydb/public/sdk/cpp/examples/ttl/util.h | 35 +- ydb/public/sdk/cpp/examples/ttl/ya.make | 4 +- .../cpp/examples/vector_index/CMakeLists.txt | 35 + .../sdk/cpp/examples/vector_index/main.cpp | 13 +- .../examples/vector_index/vector_index.cpp | 50 +- .../cpp/examples/vector_index/vector_index.h | 31 +- .../sdk/cpp/examples/vector_index/ya.make | 4 +- .../include/ydb-cpp-sdk/client/CMakeLists.txt | 1 + .../client/bsconfig/storage_config.h | 56 + .../client/common_client/settings.h | 61 + .../client/common_client/ssl_credentials.h | 26 + .../client/coordination/coordination.h | 363 ++ .../client/datastreams/datastreams.h | 373 ++ .../include/ydb-cpp-sdk/client/debug/client.h | 83 + .../ydb-cpp-sdk/client/discovery/discovery.h | 144 + .../client/draft/ydb_dynamic_config.h | 247 + .../client/draft/ydb_replication.h | 191 + .../ydb-cpp-sdk/client/draft/ydb_scripting.h | 155 + .../ydb-cpp-sdk/client/draft/ydb_view.h | 58 + .../ydb-cpp-sdk/client/driver/driver.h | 149 + .../include/ydb-cpp-sdk/client/driver/fwd.h | 8 + .../ydb-cpp-sdk/client/export/export.h | 126 + .../client/extension_common/extension.h | 72 + .../discovery_mutator/discovery_mutator.h | 29 + .../extensions/solomon_stats/pull_client.h | 57 + .../extensions/solomon_stats/pull_connector.h | 71 + .../client/federated_topic/federated_topic.h | 553 ++ .../ydb-cpp-sdk/client/helpers/helpers.h | 19 + .../client/iam/common/CMakeLists.txt | 10 + .../ydb-cpp-sdk/client/iam/common/types.h | 52 + .../ydb-cpp-sdk/client/iam/common/ya.make | 14 + .../cpp/include/ydb-cpp-sdk/client/iam/iam.h | 19 + .../ydb-cpp-sdk/client/iam_private/iam.h | 13 + .../ydb-cpp-sdk/client/import/import.h | 104 + .../client/monitoring/monitoring.h | 58 + .../ydb-cpp-sdk/client/operation/operation.h | 70 + .../include/ydb-cpp-sdk/client/params/fwd.h | 9 + .../ydb-cpp-sdk/client/params/params.h | 99 + .../ydb-cpp-sdk/client/proto/accessor.h | 76 + .../include/ydb-cpp-sdk/client/query/client.h | 266 + .../include/ydb-cpp-sdk/client/query/fwd.h | 37 + .../include/ydb-cpp-sdk/client/query/query.h | 187 + .../include/ydb-cpp-sdk/client/query/stats.h | 44 + .../cpp/include/ydb-cpp-sdk/client/query/tx.h | 119 + .../client/rate_limiter/rate_limiter.h | 174 + .../ydb-cpp-sdk/client/resources/ydb_ca.h | 9 + .../client/resources/ydb_resources.h | 23 + .../include/ydb-cpp-sdk/client/result/fwd.h | 8 + .../ydb-cpp-sdk/client/result/result.h | 110 + .../include/ydb-cpp-sdk/client/retry/retry.h | 44 + .../ydb-cpp-sdk/client/scheme/scheme.h | 211 + .../include/ydb-cpp-sdk/client/table/fwd.h | 116 + .../client/table/query_stats/stats.h | 44 + .../include/ydb-cpp-sdk/client/table/table.h | 2198 ++++++++ .../ydb-cpp-sdk/client/table/table_enum.h | 63 + .../include/ydb-cpp-sdk/client/topic/client.h | 64 + .../include/ydb-cpp-sdk/client/topic/codecs.h | 96 + .../ydb-cpp-sdk/client/topic/control_plane.h | 759 +++ .../ydb-cpp-sdk/client/topic/counters.h | 172 + .../include/ydb-cpp-sdk/client/topic/errors.h | 13 + .../ydb-cpp-sdk/client/topic/events_common.h | 48 + .../ydb-cpp-sdk/client/topic/executor.h | 47 + .../ydb-cpp-sdk/client/topic/read_events.h | 440 ++ .../ydb-cpp-sdk/client/topic/read_session.h | 256 + .../ydb-cpp-sdk/client/topic/retry_policy.h | 41 + .../ydb-cpp-sdk/client/topic/write_events.h | 111 + .../ydb-cpp-sdk/client/topic/write_session.h | 284 + .../include/ydb-cpp-sdk/client/topic/ya.make | 32 + .../client/types/credentials/credentials.h | 43 + .../oauth2_token_exchange/credentials.h | 64 + .../oauth2_token_exchange/from_file.h | 48 + .../oauth2_token_exchange/jwt_token_source.h | 80 + .../client/types/exceptions/exceptions.h | 20 + .../types/fatal_error_handlers/handlers.h | 9 + .../client/types/fluent_settings_helpers.h | 50 + .../include/ydb-cpp-sdk/client/types/fwd.h | 34 + .../client/types/operation/operation.h | 55 + .../client/types/request_settings.h | 77 + .../ydb-cpp-sdk/client/types/s3_settings.h | 28 + .../ydb-cpp-sdk/client/types/status/status.h | 74 + .../ydb-cpp-sdk/client/types/status_codes.h | 56 + .../include/ydb-cpp-sdk/client/types/ydb.h | 30 + .../include/ydb-cpp-sdk/client/value/fwd.h | 15 + .../include/ydb-cpp-sdk/client/value/value.h | 541 ++ .../ydb-cpp-sdk/library/issue/yql_issue.h | 351 ++ .../cpp/include/ydb-cpp-sdk/library/jwt/jwt.h | 20 + .../library/operation_id/operation_id.h | 73 + .../ydb-cpp-sdk/library/retry/retry_policy.h | 289 + .../library/string_utils/helpers/helpers.h | 83 + .../library/string_utils/misc/misc.h | 61 + .../sdk/cpp/include/ydb-cpp-sdk/stlfwd.h | 15 + .../cpp/include/ydb-cpp-sdk/type_switcher.h | 17 + ydb/public/sdk/cpp/sdk_common.inc | 7 + ydb/public/sdk/cpp/src/CMakeLists.txt | 3 + ydb/public/sdk/cpp/src/client/CMakeLists.txt | 32 + .../cpp/src/client/bsconfig/CMakeLists.txt | 20 + .../src/client/bsconfig/storage_config.cpp | 78 + .../sdk/cpp/src/client/bsconfig/ya.make | 16 + .../src/client/common_client/CMakeLists.txt | 15 + .../client/common_client/impl/CMakeLists.txt | 12 + .../src/client/common_client/impl/client.cpp | 1 + .../src/client/common_client/impl/client.h | 133 + .../cpp/src/client/common_client/impl/iface.h | 14 + .../cpp/src/client/common_client/impl/ya.make | 13 + .../cpp/src/client/common_client/settings.cpp | 20 + .../sdk/cpp/src/client/common_client/ya.make | 14 + .../src/client/coordination/CMakeLists.txt | 27 + .../src/client/coordination/coordination.cpp | 2096 ++++++++ .../client/coordination/proto_accessor.cpp | 11 + .../sdk/cpp/src/client/coordination/ya.make | 23 + .../cpp/src/client/datastreams/CMakeLists.txt | 17 + .../src/client/datastreams/datastreams.cpp | 957 ++++ .../sdk/cpp/src/client/datastreams/ya.make | 18 + .../sdk/cpp/src/client/debug/CMakeLists.txt | 17 + .../sdk/cpp/src/client/debug/client.cpp | 76 + ydb/public/sdk/cpp/src/client/debug/ya.make | 15 + .../cpp/src/client/discovery/CMakeLists.txt | 17 + .../cpp/src/client/discovery/discovery.cpp | 284 + .../sdk/cpp/src/client/discovery/ya.make | 14 + .../sdk/cpp/src/client/draft/CMakeLists.txt | 25 + ydb/public/sdk/cpp/src/client/draft/ya.make | 22 + .../src/client/draft/ydb_dynamic_config.cpp | 434 ++ .../cpp/src/client/draft/ydb_replication.cpp | 274 + .../cpp/src/client/draft/ydb_scripting.cpp | 383 ++ .../sdk/cpp/src/client/draft/ydb_view.cpp | 92 + .../sdk/cpp/src/client/driver/CMakeLists.txt | 17 + .../sdk/cpp/src/client/driver/driver.cpp | 226 + ydb/public/sdk/cpp/src/client/driver/ya.make | 17 + .../sdk/cpp/src/client/export/CMakeLists.txt | 25 + .../sdk/cpp/src/client/export/export.cpp | 203 + ydb/public/sdk/cpp/src/client/export/out.cpp | 9 + ydb/public/sdk/cpp/src/client/export/ya.make | 21 + .../client/extension_common/CMakeLists.txt | 13 + .../src/client/extension_common/extension.cpp | 44 + .../cpp/src/client/extension_common/ya.make | 14 + .../cpp/src/client/extensions/CMakeLists.txt | 2 + .../discovery_mutator/CMakeLists.txt | 12 + .../discovery_mutator/discovery_mutator.cpp | 1 + .../extensions/discovery_mutator/ya.make | 13 + .../extensions/solomon_stats/CMakeLists.txt | 17 + .../client/extensions/solomon_stats/README.md | 92 + .../extensions/solomon_stats/pull_client.cpp | 47 + .../solomon_stats/pull_connector.cpp | 1 + .../client/extensions/solomon_stats/ya.make | 18 + .../sdk/cpp/src/client/extensions/ya.make | 4 + .../src/client/federated_topic/CMakeLists.txt | 18 + .../federated_topic/impl/CMakeLists.txt | 31 + .../impl/federated_read_session.cpp | 291 + .../impl/federated_read_session.h | 249 + .../impl/federated_read_session_event.cpp | 168 + .../federated_topic/impl/federated_topic.cpp | 89 + .../impl/federated_topic_impl.cpp | 61 + .../impl/federated_topic_impl.h | 92 + .../impl/federated_write_session.cpp | 470 ++ .../impl/federated_write_session.h | 209 + .../impl/federation_observer.cpp | 203 + .../impl/federation_observer.h | 124 + .../src/client/federated_topic/impl/ya.make | 33 + .../federated_topic}/ut/basic_usage_ut.cpp | 78 +- .../federated_topic}/ut/fds_mock/fds_mock.h | 0 .../federated_topic/ut/fds_mock/ya.make | 9 + .../cpp/src/client/federated_topic/ut/ya.make | 38 + .../cpp/src/client/federated_topic/ya.make | 21 + .../sdk/cpp/src/client/helpers/CMakeLists.txt | 16 + .../sdk/cpp/src/client/helpers/helpers.cpp | 77 + ydb/public/sdk/cpp/src/client/helpers/ya.make | 15 + .../sdk/cpp/src/client/iam/CMakeLists.txt | 20 + .../cpp/src/client/iam/common/CMakeLists.txt | 11 + .../sdk/cpp/src/client/iam/common/iam.h | 228 + .../sdk/cpp/src/client/iam/common/ya.make | 15 + ydb/public/sdk/cpp/src/client/iam/iam.cpp | 113 + ydb/public/sdk/cpp/src/client/iam/ya.make | 16 + .../cpp/src/client/iam_private/CMakeLists.txt | 17 + .../sdk/cpp/src/client/iam_private/iam.cpp | 28 + .../sdk/cpp/src/client/iam_private/ya.make | 14 + .../sdk/cpp/src/client/impl/CMakeLists.txt | 3 + .../client/impl/ydb_endpoints/CMakeLists.txt | 13 + .../client/impl/ydb_endpoints/endpoints.cpp | 273 + .../src/client/impl/ydb_endpoints/endpoints.h | 150 + .../cpp/src/client/impl/ydb_endpoints/ya.make | 14 + .../client/impl/ydb_internal/CMakeLists.txt | 11 + .../impl/ydb_internal/common/CMakeLists.txt | 15 + .../impl/ydb_internal/common/client_pid.cpp | 33 + .../impl/ydb_internal/common/client_pid.h | 9 + .../impl/ydb_internal/common/getenv.cpp | 12 + .../client/impl/ydb_internal/common/getenv.h | 10 + .../impl/ydb_internal/common/parser.cpp | 51 + .../client/impl/ydb_internal/common/parser.h | 16 + .../client/impl/ydb_internal/common/types.h | 24 + .../client/impl/ydb_internal/common/ya.make | 16 + .../db_driver_state/CMakeLists.txt | 19 + .../db_driver_state/authenticator.cpp | 34 + .../db_driver_state/authenticator.h | 28 + .../db_driver_state/endpoint_pool.cpp | 191 + .../db_driver_state/endpoint_pool.h | 68 + .../ydb_internal/db_driver_state/state.cpp | 287 + .../impl/ydb_internal/db_driver_state/state.h | 109 + .../impl/ydb_internal/db_driver_state/ya.make | 20 + .../impl/ydb_internal/driver/constants.h | 15 + .../grpc_connections/CMakeLists.txt | 20 + .../ydb_internal/grpc_connections/actions.cpp | 113 + .../ydb_internal/grpc_connections/actions.h | 232 + .../grpc_connections/grpc_connections.cpp | 427 ++ .../grpc_connections/grpc_connections.h | 727 +++ .../ydb_internal/grpc_connections/params.h | 37 + .../ydb_internal/grpc_connections/ya.make | 21 + .../ydb_internal/internal_client/client.h | 35 + .../impl/ydb_internal/internal_header.h | 20 + .../kqp_session_common/CMakeLists.txt | 14 + .../kqp_session_common/kqp_session_common.cpp | 191 + .../kqp_session_common/kqp_session_common.h | 96 + .../ydb_internal/kqp_session_common/ya.make | 16 + .../impl/ydb_internal/logger/CMakeLists.txt | 12 + .../client/impl/ydb_internal/logger/log.cpp | 49 + .../src/client/impl/ydb_internal/logger/log.h | 12 + .../client/impl/ydb_internal/logger/ya.make | 13 + .../ydb_internal/make_request/CMakeLists.txt | 13 + .../impl/ydb_internal/make_request/make.cpp | 12 + .../impl/ydb_internal/make_request/make.h | 49 + .../impl/ydb_internal/make_request/ya.make | 14 + .../ydb_internal/plain_status/CMakeLists.txt | 14 + .../impl/ydb_internal/plain_status/status.cpp | 71 + .../impl/ydb_internal/plain_status/status.h | 79 + .../impl/ydb_internal/plain_status/ya.make | 15 + .../impl/ydb_internal/retry/CMakeLists.txt | 12 + .../client/impl/ydb_internal/retry/retry.cpp | 41 + .../client/impl/ydb_internal/retry/retry.h | 115 + .../impl/ydb_internal/retry/retry_async.h | 173 + .../impl/ydb_internal/retry/retry_sync.h | 131 + .../client/impl/ydb_internal/retry/ya.make | 14 + .../rpc_request_settings/settings.h | 35 + .../ydb_internal/scheme_helpers/helpers.h | 18 + .../session_client/session_client.h | 19 + .../ydb_internal/session_pool/CMakeLists.txt | 15 + .../session_pool/session_pool.cpp | 386 ++ .../ydb_internal/session_pool/session_pool.h | 149 + .../impl/ydb_internal/session_pool/ya.make | 16 + .../ydb_internal/stats_extractor/extractor.h | 44 + .../impl/ydb_internal/table_helpers/helpers.h | 29 + .../ydb_internal/thread_pool/CMakeLists.txt | 11 + .../impl/ydb_internal/thread_pool/pool.cpp | 2 + .../impl/ydb_internal/thread_pool/pool.h | 21 + .../impl/ydb_internal/thread_pool/ya.make | 9 + .../ydb_internal/value_helpers/CMakeLists.txt | 13 + .../ydb_internal/value_helpers/helpers.cpp | 112 + .../impl/ydb_internal/value_helpers/helpers.h | 11 + .../impl/ydb_internal/value_helpers/ya.make | 14 + .../src/client/impl/ydb_stats/CMakeLists.txt | 13 + .../cpp/src/client/impl/ydb_stats/stats.cpp | 45 + .../sdk/cpp/src/client/impl/ydb_stats/stats.h | 403 ++ .../sdk/cpp/src/client/impl/ydb_stats/ya.make | 14 + .../sdk/cpp/src/client/import/CMakeLists.txt | 26 + .../sdk/cpp/src/client/import/import.cpp | 175 + ydb/public/sdk/cpp/src/client/import/out.cpp | 5 + ydb/public/sdk/cpp/src/client/import/ya.make | 21 + .../cpp/src/client/monitoring/CMakeLists.txt | 22 + .../cpp/src/client/monitoring/monitoring.cpp | 98 + .../sdk/cpp/src/client/monitoring/ya.make | 19 + .../cpp/src/client/operation/CMakeLists.txt | 20 + .../cpp/src/client/operation/operation.cpp | 210 + .../sdk/cpp/src/client/operation/ya.make | 21 + .../sdk/cpp/src/client/params/CMakeLists.txt | 15 + ydb/public/sdk/cpp/src/client/params/impl.cpp | 47 + ydb/public/sdk/cpp/src/client/params/impl.h | 21 + .../sdk/cpp/src/client/params/params.cpp | 179 + ydb/public/sdk/cpp/src/client/params/ya.make | 16 + .../client/persqueue_public/CMakeLists.txt | 35 + .../persqueue_public/codecs/CMakeLists.txt | 5 + .../client/persqueue_public/codecs/codecs.h | 3 + .../client/persqueue_public/codecs/ya.make | 13 + .../persqueue_public/impl/CMakeLists.txt | 26 + .../client/persqueue_public/impl/aliases.h | 47 + .../client/persqueue_public/impl/common.cpp | 58 + .../src/client/persqueue_public/impl/common.h | 10 + .../persqueue_public/impl/persqueue.cpp | 202 + .../persqueue_public/impl/persqueue_impl.cpp | 88 + .../persqueue_public/impl/persqueue_impl.h | 275 + .../persqueue_public/impl/read_session.cpp | 929 ++++ .../persqueue_public/impl/read_session.h | 128 + .../impl/read_session_messages.cpp | 244 + .../persqueue_public/impl/write_session.cpp | 152 + .../persqueue_public/impl/write_session.h | 97 + .../impl/write_session_impl.cpp | 1405 +++++ .../impl/write_session_impl.h | 445 ++ .../src/client/persqueue_public/impl/ya.make | 38 + .../client/persqueue_public/include/aliases.h | 47 + .../client/persqueue_public/include/client.h | 70 + .../persqueue_public/include/control_plane.h | 325 ++ .../persqueue_public/include/read_events.h | 474 ++ .../persqueue_public/include/read_session.h | 262 + .../persqueue_public/include/write_events.h | 84 + .../persqueue_public/include/write_session.h | 205 + .../client/persqueue_public/include/ya.make | 26 + .../src/client/persqueue_public/persqueue.h | 3 + .../persqueue_public}/ut/basic_usage_ut.cpp | 16 +- .../client/persqueue_public}/ut/common_ut.cpp | 2 +- .../ut/compress_executor_ut.cpp | 2 +- .../persqueue_public}/ut/compression_ut.cpp | 2 +- .../persqueue_public}/ut/read_session_ut.cpp | 74 +- .../persqueue_public}/ut/retry_policy_ut.cpp | 4 +- .../ut/ut_utils/data_plane_helpers.cpp | 11 +- .../ut/ut_utils/data_plane_helpers.h | 14 +- .../ut/ut_utils/sdk_test_setup.h | 4 +- .../ut/ut_utils/test_server.cpp | 0 .../ut/ut_utils/test_server.h | 2 +- .../ut/ut_utils/test_utils.h | 0 .../ut/ut_utils/ut_utils.cpp | 0 .../persqueue_public}/ut/ut_utils/ut_utils.h | 16 +- .../persqueue_public}/ut/ut_utils/ya.make | 12 +- .../ut/with_offset_ranges_mode_ut/ya.make | 10 +- .../client/persqueue_public}/ut/ya.make | 16 +- .../cpp/src/client/persqueue_public/ya.make | 29 + .../sdk/cpp/src/client/proto/CMakeLists.txt | 17 + .../sdk/cpp/src/client/proto/accessor.cpp | 138 + ydb/public/sdk/cpp/src/client/proto/ya.make | 17 + .../sdk/cpp/src/client/query/CMakeLists.txt | 27 + .../sdk/cpp/src/client/query/client.cpp | 732 +++ .../cpp/src/client/query/impl/CMakeLists.txt | 16 + .../src/client/query/impl/client_session.cpp | 148 + .../src/client/query/impl/client_session.h | 50 + .../cpp/src/client/query/impl/exec_query.cpp | 338 ++ .../cpp/src/client/query/impl/exec_query.h | 23 + .../sdk/cpp/src/client/query/impl/ya.make | 18 + ydb/public/sdk/cpp/src/client/query/query.cpp | 69 + ydb/public/sdk/cpp/src/client/query/stats.cpp | 73 + ydb/public/sdk/cpp/src/client/query/tx.cpp | 1 + ydb/public/sdk/cpp/src/client/query/ya.make | 24 + .../src/client/rate_limiter/CMakeLists.txt | 14 + .../src/client/rate_limiter/rate_limiter.cpp | 215 + .../sdk/cpp/src/client/rate_limiter/ya.make | 15 + .../cpp/src/client/resources/CMakeLists.txt | 29 + .../sdk/cpp/src/client/resources/ya.make | 15 + .../sdk/cpp/src/client/resources/ydb_ca.cpp | 11 + .../src/client/resources/ydb_resources.cpp | 35 + .../cpp/src/client/resources/ydb_root_ca.pem | 4738 +++++++++++++++++ .../src/client/resources/ydb_sdk_version.txt | 1 + .../sdk/cpp/src/client/result/CMakeLists.txt | 17 + ydb/public/sdk/cpp/src/client/result/out.cpp | 5 + .../cpp/src/client/result/proto_accessor.cpp | 11 + .../sdk/cpp/src/client/result/result.cpp | 235 + ydb/public/sdk/cpp/src/client/result/ya.make | 18 + .../sdk/cpp/src/client/scheme/CMakeLists.txt | 22 + ydb/public/sdk/cpp/src/client/scheme/out.cpp | 17 + .../sdk/cpp/src/client/scheme/scheme.cpp | 368 ++ ydb/public/sdk/cpp/src/client/scheme/ya.make | 18 + .../cpp/src/client/ss_tasks/CMakeLists.txt | 24 + .../sdk/cpp/src/client/ss_tasks/out.cpp | 1 + .../sdk/cpp/src/client/ss_tasks/task.cpp | 23 + ydb/public/sdk/cpp/src/client/ss_tasks/task.h | 30 + .../sdk/cpp/src/client/ss_tasks/ya.make | 21 + .../sdk/cpp/src/client/table/CMakeLists.txt | 37 + .../cpp/src/client/table/impl/CMakeLists.txt | 30 + .../src/client/table/impl/client_session.cpp | 79 + .../src/client/table/impl/client_session.h | 65 + .../cpp/src/client/table/impl/data_query.cpp | 61 + .../cpp/src/client/table/impl/data_query.h | 40 + .../sdk/cpp/src/client/table/impl/readers.cpp | 106 + .../sdk/cpp/src/client/table/impl/readers.h | 66 + .../client/table/impl/request_migrator.cpp | 87 + .../src/client/table/impl/request_migrator.h | 68 + .../src/client/table/impl/table_client.cpp | 1181 ++++ .../cpp/src/client/table/impl/table_client.h | 325 ++ .../cpp/src/client/table/impl/transaction.cpp | 74 + .../cpp/src/client/table/impl/transaction.h | 37 + .../sdk/cpp/src/client/table/impl/ya.make | 23 + ydb/public/sdk/cpp/src/client/table/out.cpp | 85 + .../cpp/src/client/table/proto_accessor.cpp | 69 + .../client/table/query_stats/CMakeLists.txt | 14 + .../src/client/table/query_stats/stats.cpp | 36 + .../cpp/src/client/table/query_stats/ya.make | 15 + ydb/public/sdk/cpp/src/client/table/table.cpp | 3310 ++++++++++++ ydb/public/sdk/cpp/src/client/table/ya.make | 29 + .../sdk/cpp/src/client/topic/CMakeLists.txt | 45 + .../src/client/topic/codecs/CMakeLists.txt | 21 + .../cpp/src/client/topic/codecs/codecs.cpp | 66 + .../sdk/cpp/src/client/topic/codecs/ya.make | 13 + .../src/client/topic/common/CMakeLists.txt | 15 + .../client/topic/common/callback_context.h | 202 + .../src/client/topic/common/executor_impl.cpp | 98 + .../src/client/topic/common/executor_impl.h | 87 + .../cpp/src/client/topic/common/log_lazy.h | 10 + .../src/client/topic/common/retry_policy.cpp | 32 + .../cpp/src/client/topic/common/trace_lazy.h | 21 + .../sdk/cpp/src/client/topic/common/ya.make | 24 + .../cpp/src/client/topic/impl/CMakeLists.txt | 40 + .../sdk/cpp/src/client/topic/impl/common.cpp | 109 + .../sdk/cpp/src/client/topic/impl/common.h | 426 ++ .../src/client/topic/impl/counters_logger.h | 151 + .../src/client/topic/impl/deferred_commit.cpp | 131 + .../src/client/topic/impl/event_handlers.cpp | 151 + .../client/topic/impl/offsets_collector.cpp | 72 + .../src/client/topic/impl/offsets_collector.h | 35 + .../src/client/topic/impl/read_session.cpp | 327 ++ .../cpp/src/client/topic/impl/read_session.h | 88 + .../client/topic/impl/read_session_event.cpp | 446 ++ .../src/client/topic/impl/read_session_impl.h | 1383 +++++ .../client/topic/impl/read_session_impl.ipp | 3014 +++++++++++ .../sdk/cpp/src/client/topic/impl/topic.cpp | 538 ++ .../cpp/src/client/topic/impl/topic_impl.cpp | 78 + .../cpp/src/client/topic/impl/topic_impl.h | 395 ++ .../cpp/src/client/topic/impl/transaction.cpp | 26 + .../cpp/src/client/topic/impl/transaction.h | 32 + .../src/client/topic/impl/write_session.cpp | 181 + .../cpp/src/client/topic/impl/write_session.h | 93 + .../client/topic/impl/write_session_impl.cpp | 1748 ++++++ .../client/topic/impl/write_session_impl.h | 500 ++ .../sdk/cpp/src/client/topic/impl/ya.make | 42 + ydb/public/sdk/cpp/src/client/topic/out.cpp | 9 + .../cpp/src/client/topic/proto_accessor.cpp | 38 + .../client/topic}/ut/basic_usage_ut.cpp | 58 +- .../client/topic}/ut/describe_topic_ut.cpp | 32 +- .../client/topic}/ut/local_partition_ut.cpp | 44 +- .../client/topic}/ut/topic_to_table_ut.cpp | 64 +- .../client/topic}/ut/trace_ut.cpp | 2 +- .../topic}/ut/ut_utils/managed_executor.cpp | 0 .../topic}/ut/ut_utils/managed_executor.h | 4 +- .../ut/ut_utils/topic_sdk_test_setup.cpp | 2 +- .../topic}/ut/ut_utils/topic_sdk_test_setup.h | 14 +- .../client/topic}/ut/ut_utils/trace.cpp | 2 +- .../client/topic}/ut/ut_utils/trace.h | 4 +- .../client/topic}/ut/ut_utils/ya.make | 10 +- .../ydb_topic => src/client/topic}/ut/ya.make | 23 +- ydb/public/sdk/cpp/src/client/topic/ya.make | 29 + .../sdk/cpp/src/client/types/CMakeLists.txt | 29 + .../types/core_facility/core_facility.h | 19 + .../client/types/credentials/CMakeLists.txt | 20 + .../client/types/credentials/credentials.cpp | 82 + .../types/credentials/login/CMakeLists.txt | 18 + .../client/types/credentials/login/login.cpp | 249 + .../client/types/credentials/login/ya.make | 17 + .../oauth2_token_exchange/CMakeLists.txt | 30 + .../oauth2_token_exchange/credentials.cpp | 531 ++ .../oauth2_token_exchange/from_file.cpp | 271 + .../jwt_token_source.cpp | 86 + .../credentials/oauth2_token_exchange/ya.make | 24 + .../cpp/src/client/types/credentials/ya.make | 19 + .../client/types/exceptions/CMakeLists.txt | 11 + .../client/types/exceptions/exceptions.cpp | 12 + .../cpp/src/client/types/exceptions/ya.make | 9 + .../types/fatal_error_handlers/CMakeLists.txt | 12 + .../types/fatal_error_handlers/handlers.cpp | 10 + .../client/types/fatal_error_handlers/ya.make | 13 + .../src/client/types/operation/CMakeLists.txt | 16 + .../src/client/types/operation/operation.cpp | 130 + .../cpp/src/client/types/operation/out.cpp | 5 + .../cpp/src/client/types/operation/ya.make | 17 + .../src/client/types/status/CMakeLists.txt | 16 + .../cpp/src/client/types/status/status.cpp | 124 + .../sdk/cpp/src/client/types/status/ya.make | 17 + ydb/public/sdk/cpp/src/client/types/ya.make | 14 + .../sdk/cpp/src/client/value/CMakeLists.txt | 26 + ydb/public/sdk/cpp/src/client/value/out.cpp | 5 + ydb/public/sdk/cpp/src/client/value/value.cpp | 3372 ++++++++++++ ydb/public/sdk/cpp/src/client/value/ya.make | 21 + ydb/public/sdk/cpp/src/client/ya.make | 52 + ydb/public/sdk/cpp/src/library/CMakeLists.txt | 18 + .../cpp/src/library/benchmark/CMakeLists.txt | 1 + .../sdk/cpp/src/library/benchmark/clobber.h | 26 + .../cpp/src/library/decimal/CMakeLists.txt | 12 + .../sdk/cpp/src/library/decimal/ya.make | 9 + .../cpp/src/library/decimal/yql_decimal.cpp | 241 + .../sdk/cpp/src/library/decimal/yql_decimal.h | 54 + .../cpp/src/library/decimal/yql_wide_int.h | 344 ++ .../src/library/grpc/client/CMakeLists.txt | 15 + .../library/grpc/client/grpc_client_low.cpp | 34 +- .../src/library/grpc/client/grpc_client_low.h | 1440 +++++ .../cpp/src/library/grpc/client/grpc_common.h | 87 + .../sdk/cpp/src/library/grpc/client/ya.make | 14 + .../cpp/src/library/grpc/common/constants.h | 9 + .../sdk/cpp/src/library/issue/CMakeLists.txt | 21 + ydb/public/sdk/cpp/src/library/issue/utf8.cpp | 58 + ydb/public/sdk/cpp/src/library/issue/utf8.h | 9 + ydb/public/sdk/cpp/src/library/issue/ya.make | 18 + .../sdk/cpp/src/library/issue/yql_issue.cpp | 302 ++ .../src/library/issue/yql_issue_message.cpp | 93 + .../cpp/src/library/issue/yql_issue_message.h | 15 + .../sdk/cpp/src/library/jwt/CMakeLists.txt | 14 + .../{lib => sdk/cpp/src/library}/jwt/jwt.cpp | 12 +- ydb/public/sdk/cpp/src/library/jwt/ya.make | 14 + .../cpp/src/library/malloc/api/CMakeLists.txt | 5 + .../sdk/cpp/src/library/malloc/api/malloc.cpp | 43 + .../sdk/cpp/src/library/malloc/api/malloc.h | 32 + .../src/library/operation_id/CMakeLists.txt | 17 + .../src/library/operation_id/operation_id.cpp | 309 ++ .../operation_id/protos/CMakeLists.txt | 13 + .../operation_id/protos/operation_id.proto | 0 .../src/library}/operation_id/protos/ya.make | 0 .../sdk/cpp/src/library/operation_id/ya.make | 15 + .../persqueue/obfuscate/CMakeLists.txt | 11 + .../library/persqueue/obfuscate/obfuscate.cpp | 4 +- .../library/persqueue/obfuscate/obfuscate.h | 9 + .../src/library/persqueue/obfuscate/ya.make | 10 + .../topic_parser_public/CMakeLists.txt | 11 + .../topic_parser_public/topic_parser.cpp | 54 +- .../topic_parser_public/topic_parser.h | 32 + .../persqueue/topic_parser_public/ya.make | 10 + .../src/library/proto_output/CMakeLists.txt | 14 + .../src/library/proto_output/proto_output.cpp | 56 + .../sdk/cpp/src/library/retry/CMakeLists.txt | 20 + .../library/retry/protos/retry_options.proto | 9 + .../sdk/cpp/src/library/retry/retry.cpp | 51 + ydb/public/sdk/cpp/src/library/retry/retry.h | 223 + .../cpp/src/library/retry/retry_policy_ut.cpp | 108 + .../sdk/cpp/src/library/retry/retry_ut.cpp | 117 + .../sdk/cpp/src/library/retry/utils.cpp | 20 + ydb/public/sdk/cpp/src/library/retry/utils.h | 10 + .../string_utils/base64/CMakeLists.txt | 12 + .../library/string_utils/base64/base64.cpp | 224 + .../src/library/string_utils/base64/base64.h | 154 + .../base64/base64_decode_uneven_ut.cpp | 46 + .../library/string_utils/base64/base64_ut.cpp | 539 ++ .../string_utils/helpers/CMakeLists.txt | 8 + .../library/string_utils/helpers/helpers.cpp | 245 + .../src/library/string_utils/helpers/ya.make | 9 + .../library/string_utils/misc/CMakeLists.txt | 8 + .../sdk/cpp/src/library/uuid/CMakeLists.txt | 11 + ydb/public/sdk/cpp/src/library/uuid/uuid.cpp | 82 + ydb/public/sdk/cpp/src/library/uuid/uuid.h | 125 + ydb/public/sdk/cpp/src/library/uuid/ya.make | 9 + ydb/public/sdk/cpp/src/library/ya.make | 11 + ydb/public/sdk/cpp/src/ya.make | 4 + ydb/public/sdk/cpp/tests/CMakeLists.txt | 2 + .../sdk/cpp/tests/integration/CMakeLists.txt | 2 + .../basic_example_it/CMakeLists.txt | 15 + .../basic_example_it/basic_example.cpp | 516 ++ .../basic_example_it/basic_example.h | 29 + .../basic_example_it/basic_example_data.cpp | 202 + .../integration/basic_example_it/main.cpp | 222 + .../integration/basic_example_it/ya.make | 28 + .../bulk_upsert_simple_it/CMakeLists.txt | 12 + .../bulk_upsert_simple_it/bulk_upsert.cpp | 158 + .../bulk_upsert_simple_it/bulk_upsert.h | 39 + .../bulk_upsert_simple_it/main.cpp | 54 + .../integration/bulk_upsert_simple_it/ya.make | 25 + ydb/public/sdk/cpp/tests/integration/ya.make | 4 + ydb/public/sdk/cpp/tests/unit/CMakeLists.txt | 2 + .../sdk/cpp/tests/unit/client/CMakeLists.txt | 119 + .../client/coordination/coordination_ut.cpp | 298 ++ .../tests/unit/client/coordination/ya.make | 23 + .../discovery_mutator_ut.cpp | 40 + .../unit/client/discovery_mutator/ya.make | 23 + .../unit/client/draft/helpers/CMakeLists.txt | 13 + .../unit/client/draft/helpers/grpc_server.h | 18 + .../draft/helpers/grpc_services/scripting.cpp | 21 + .../draft/helpers/grpc_services/scripting.h | 16 + .../draft/helpers/grpc_services/view.cpp | 19 + .../client/draft/helpers/grpc_services/view.h | 18 + .../tests/unit/client/draft/helpers/ya.make | 15 + .../sdk/cpp/tests/unit/client/draft/ya.make | 24 + .../ydb_scripting_response_headers_ut.cpp | 30 + .../tests/unit/client/draft/ydb_view_ut.cpp | 28 + .../tests/unit/client/driver/driver_ut.cpp | 186 + .../sdk/cpp/tests/unit/client/driver/ya.make | 23 + .../unit/client/endpoints/endpoints_ut.cpp | 314 ++ .../cpp/tests/unit/client/endpoints/ya.make | 22 + .../oauth2_token_exchange/CMakeLists.txt | 14 + .../oauth2_token_exchange/credentials_ut.cpp | 1241 +++++ .../oauth2_token_exchange/jwt_check_helper.h | 98 + .../jwt_token_source_ut.cpp | 63 + .../unit/client/oauth2_token_exchange/ya.make | 16 + .../unit/client/params}/params_ut.cpp | 11 +- .../ut => tests/unit/client/params}/ya.make | 5 +- .../tests/unit/client/result/result_ut.cpp | 237 + .../sdk/cpp/tests/unit/client/result/ya.make | 23 + .../unit/client/value}/value_ut.cpp | 49 +- .../ut => tests/unit/client/value}/ya.make | 7 +- ydb/public/sdk/cpp/tests/unit/client/ya.make | 11 + .../sdk/cpp/tests/unit/library/CMakeLists.txt | 4 + .../tests/unit/library/decimal/CMakeLists.txt | 11 + .../cpp/tests/unit/library/decimal/ya.make | 14 + .../unit/library/decimal/yql_decimal_ut.cpp | 195 + .../unit/library/decimal/yql_wide_int_ut.cpp | 466 ++ .../unit/library/grpc_client/CMakeLists.txt | 9 + .../grpc_client}/grpc_client_low_ut.cpp | 4 +- .../tests/unit/library/grpc_client/ya.make | 15 + .../tests/unit/library/issue/CMakeLists.txt | 12 + .../cpp/tests/unit/library/issue/utf8_ut.cpp | 18 + .../sdk/cpp/tests/unit/library/issue/ya.make | 17 + .../tests/unit/library/issue/yql_issue_ut.cpp | 101 + .../unit/library/operation_id/CMakeLists.txt | 11 + .../library}/operation_id/operation_id_ut.cpp | 61 +- .../tests/unit/library/operation_id}/ya.make | 5 +- ydb/public/sdk/cpp/tests/unit/library/ya.make | 6 + ydb/public/sdk/cpp/tests/unit/ya.make | 4 + ydb/public/sdk/cpp/tests/ya.make | 4 + ydb/public/sdk/cpp/ya.make | 69 +- ydb/services/auth/grpc_service.cpp | 2 +- ydb/services/auth/ya.make | 2 +- ydb/services/backup/grpc_service.cpp | 2 +- ydb/services/backup/ya.make | 2 +- ydb/services/bsconfig/bsconfig_ut.cpp | 4 +- ydb/services/cms/cms_ut.cpp | 10 +- ydb/services/cms/ut/ya.make | 2 +- ydb/services/cms/ya.make | 4 +- ydb/services/datastreams/datastreams_ut.cpp | 18 +- ydb/services/datastreams/ut/ya.make | 12 +- ydb/services/datastreams/ya.make | 6 +- .../deprecated/persqueue_v0/grpc_pq_actor.h | 1 + .../grpc_pq_clusters_updater_actor.cpp | 2 +- ydb/services/discovery/ya.make | 2 +- ydb/services/dynamic_config/ut/ya.make | 2 +- ydb/services/dynamic_config/ya.make | 4 +- ydb/services/ext_index/ut/ut_ext_index.cpp | 2 +- ydb/services/fq/ut_integration/fq_ut.cpp | 2 +- ydb/services/fq/ut_integration/ut_utils.cpp | 2 +- ydb/services/fq/ut_integration/ut_utils.h | 2 +- ydb/services/fq/ut_integration/ya.make | 2 +- ydb/services/kesus/grpc_service.cpp | 2 +- ydb/services/kesus/ya.make | 2 +- ydb/services/keyvalue/grpc_service_ut.cpp | 4 +- ydb/services/lib/actors/pq_schema_actor.cpp | 4 +- ydb/services/lib/actors/ya.make | 6 +- ydb/services/local_discovery/ya.make | 2 +- ydb/services/metadata/abstract/ya.make | 1 + .../metadata/initializer/ut/ut_init.cpp | 2 +- ydb/services/metadata/secret/ut/ut_secret.cpp | 2 +- ydb/services/monitoring/ya.make | 2 +- .../persqueue_v1/actors/schema_actors.cpp | 2 +- .../actors/write_session_actor.cpp | 2 +- .../persqueue_v1/actors/write_session_actor.h | 1 + .../persqueue_v1/first_class_src_ids_ut.cpp | 8 +- .../persqueue_v1/persqueue_compat_ut.cpp | 10 +- .../persqueue_new_schemecache_ut.cpp | 44 +- ydb/services/persqueue_v1/persqueue_ut.cpp | 64 +- ydb/services/persqueue_v1/topic_yql_ut.cpp | 2 +- ydb/services/persqueue_v1/ut/demo_tx.cpp | 20 +- .../ut/describes_ut/describe_topic_ut.cpp | 3 +- .../ut/describes_ut/ic_cache_ut.cpp | 2 +- .../persqueue_v1/ut/describes_ut/ya.make | 7 +- .../ut/functions_executor_wrapper.h | 2 +- .../ut/new_schemecache_ut/ya.make | 10 +- .../persqueue_v1/ut/persqueue_common_tests.h | 8 +- .../persqueue_v1/ut/persqueue_test_fixture.h | 16 +- ydb/services/persqueue_v1/ut/pq_data_writer.h | 2 +- .../persqueue_v1/ut/topic_service_ut.cpp | 12 +- ydb/services/persqueue_v1/ut/ya.make | 18 +- ydb/services/persqueue_v1/ya.make | 2 +- ydb/services/rate_limiter/rate_limiter_ut.cpp | 22 +- ydb/services/rate_limiter/ut/ya.make | 4 +- ydb/services/ydb/backup_ut/ya.make | 12 +- ydb/services/ydb/backup_ut/ydb_backup_ut.cpp | 10 +- .../sdk_sessions_pool_ut.cpp | 2 +- ydb/services/ydb/sdk_sessions_pool_ut/ya.make | 4 +- .../ydb/sdk_sessions_ut/sdk_sessions_ut.cpp | 22 +- ydb/services/ydb/sdk_sessions_ut/ya.make | 4 +- ydb/services/ydb/table_split_ut/ya.make | 16 +- ydb/services/ydb/ut/ya.make | 16 +- ydb/services/ydb/ydb_bulk_upsert_olap_ut.cpp | 20 +- ydb/services/ydb/ydb_bulk_upsert_ut.cpp | 8 +- ydb/services/ydb/ydb_common_ut.h | 2 +- ydb/services/ydb/ydb_coordination_ut.cpp | 20 +- ydb/services/ydb/ydb_import_ut.cpp | 6 +- ydb/services/ydb/ydb_index_table_ut.cpp | 8 +- ydb/services/ydb/ydb_ldap_login_ut.cpp | 2 +- ydb/services/ydb/ydb_login_ut.cpp | 4 +- ydb/services/ydb/ydb_logstore_ut.cpp | 10 +- ydb/services/ydb/ydb_monitoring_ut.cpp | 6 +- ydb/services/ydb/ydb_object_storage_ut.cpp | 12 +- ydb/services/ydb/ydb_olapstore_ut.cpp | 4 +- ydb/services/ydb/ydb_register_node_ut.cpp | 14 +- ydb/services/ydb/ydb_scripting_ut.cpp | 8 +- ydb/services/ydb/ydb_stats_ut.cpp | 6 +- ydb/services/ydb/ydb_table_split_ut.cpp | 8 +- ydb/services/ydb/ydb_table_ut.cpp | 188 +- ydb/services/ydb/ydb_ut.cpp | 33 +- ydb/services/ymq/ya.make | 4 +- .../ydb_control_plane_storage_internal_ut.cpp | 32 +- .../ydb_control_plane_storage_queries_ut.cpp | 2 +- .../ydb_control_plane_storage_quotas_ut.cpp | 2 +- .../ydb_test_bootstrap.h | 2 +- ydb/tests/fq/pq_async_io/mock_pq_gateway.cpp | 16 +- ydb/tests/fq/pq_async_io/mock_pq_gateway.h | 2 +- ydb/tests/fq/pq_async_io/ut/ya.make | 4 +- ydb/tests/fq/pq_async_io/ut_helpers.cpp | 8 +- ydb/tests/fq/pq_async_io/ut_helpers.h | 4 +- ydb/tests/fq/pq_async_io/ya.make | 2 +- ydb/tests/functional/backup/backup_ut.cpp | 14 +- .../s3_path_style/s3_path_style_backup_ut.cpp | 10 +- .../functional/backup/s3_path_style/ya.make | 8 +- ydb/tests/functional/backup/ya.make | 8 +- ydb/tests/functional/kqp/kqp_indexes/main.cpp | 6 +- ydb/tests/functional/kqp/kqp_indexes/ya.make | 4 +- .../functional/kqp/kqp_query_session/main.cpp | 8 +- .../functional/kqp/kqp_query_session/ya.make | 4 +- .../functional/kqp/kqp_query_svc/main.cpp | 14 +- .../functional/kqp/kqp_query_svc/ya.make | 6 +- ydb/tests/functional/replication/main.cpp | 8 +- ydb/tests/functional/replication/ya.make | 6 +- .../sdk_credprovider/dummy_provider_ut.cpp | 14 +- .../sdk/cpp/sdk_credprovider/ya.make | 8 +- ydb/tests/tools/idx_test/main.cpp | 37 +- ydb/tests/tools/idx_test/ya.make | 4 +- ydb/tests/tools/kqprun/src/ydb_setup.cpp | 4 +- ydb/tests/tools/kqprun/src/ydb_setup.h | 2 +- ydb/tests/tools/pq_read/main.cpp | 6 +- ydb/tests/tools/pq_read/ya.make | 2 +- ydb/tools/query_replay/common_deps.inc | 4 +- ydb/tools/query_replay/query_proccessor.cpp | 2 +- ydb/tools/query_replay/query_replay.cpp | 10 +- ydb/tools/query_replay/query_replay.h | 6 +- ydb/tools/query_replay_yt/common_deps.inc | 4 +- 1837 files changed, 80750 insertions(+), 7088 deletions(-) delete mode 100644 ydb/library/grpc/client/ut/ya.make create mode 100644 ydb/public/lib/deprecated/json_value/ut/ya.make create mode 100644 ydb/public/lib/deprecated/json_value/ya.make create mode 100644 ydb/public/lib/deprecated/json_value/ydb_json_value.cpp create mode 100644 ydb/public/lib/deprecated/json_value/ydb_json_value.h create mode 100644 ydb/public/lib/deprecated/json_value/ydb_json_value_ut.cpp create mode 100644 ydb/public/lib/deprecated/yson_value/ya.make create mode 100644 ydb/public/lib/deprecated/yson_value/ydb_yson_value.cpp create mode 100644 ydb/public/lib/deprecated/yson_value/ydb_yson_value.h delete mode 100644 ydb/public/lib/operation_id/operation_id.cpp create mode 100644 ydb/public/sdk/cpp/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/adapters/issue/issue.cpp create mode 100644 ydb/public/sdk/cpp/adapters/issue/issue.h create mode 100644 ydb/public/sdk/cpp/adapters/issue/ya.make create mode 100644 ydb/public/sdk/cpp/adapters/ya.make create mode 100644 ydb/public/sdk/cpp/client/README.md create mode 100644 ydb/public/sdk/cpp/client/ya.make create mode 100644 ydb/public/sdk/cpp/client/ydb_driver/fwd.h delete mode 100644 ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/ya.make delete mode 100644 ydb/public/sdk/cpp/client/ydb_federated_topic/ut/ya.make create mode 100644 ydb/public/sdk/cpp/client/ydb_params/fwd.h delete mode 100644 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h delete mode 100644 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h delete mode 100644 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ut_utils.h delete mode 100644 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make delete mode 100644 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make create mode 100644 ydb/public/sdk/cpp/client/ydb_query/fwd.h create mode 100644 ydb/public/sdk/cpp/client/ydb_result/fwd.h create mode 100644 ydb/public/sdk/cpp/client/ydb_table/fwd.h create mode 100644 ydb/public/sdk/cpp/client/ydb_types/fwd.h create mode 100644 ydb/public/sdk/cpp/client/ydb_value/fwd.h create mode 100644 ydb/public/sdk/cpp/examples/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/basic_example/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/bulk_upsert_simple/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/pagination/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/secondary_index/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/secondary_index_builtin/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/topic_reader/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/topic_reader/eventloop/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/topic_reader/simple/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/topic_reader/transaction/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/ttl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/examples/vector_index/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/settings.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/ssl_credentials.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/coordination/coordination.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/datastreams/datastreams.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/debug/client.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/discovery/discovery.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_dynamic_config.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_scripting.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_view.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/export/export.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extension_common/extension.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/discovery_mutator/discovery_mutator.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_client.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_connector.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/helpers/helpers.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/types.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/ya.make create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/iam.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam_private/iam.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/monitoring/monitoring.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/params.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/stats.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/tx.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/rate_limiter/rate_limiter.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_ca.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_resources.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/query_stats/stats.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table_enum.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/codecs.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/counters.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/errors.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/events_common.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/executor.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_session.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/retry_policy.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_events.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_session.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/ya.make create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/from_file.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/exceptions/exceptions.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fatal_error_handlers/handlers.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fluent_settings_helpers.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/operation/operation.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/s3_settings.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/ydb.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/fwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/issue/yql_issue.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/jwt/jwt.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/operation_id/operation_id.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/retry/retry_policy.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/helpers/helpers.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/misc/misc.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/stlfwd.h create mode 100644 ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h create mode 100644 ydb/public/sdk/cpp/sdk_common.inc create mode 100644 ydb/public/sdk/cpp/src/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/bsconfig/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp create mode 100644 ydb/public/sdk/cpp/src/client/bsconfig/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/common_client/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/common_client/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/common_client/impl/client.cpp create mode 100644 ydb/public/sdk/cpp/src/client/common_client/impl/client.h create mode 100644 ydb/public/sdk/cpp/src/client/common_client/impl/iface.h create mode 100644 ydb/public/sdk/cpp/src/client/common_client/impl/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/common_client/settings.cpp create mode 100644 ydb/public/sdk/cpp/src/client/common_client/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/coordination/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/coordination/coordination.cpp create mode 100644 ydb/public/sdk/cpp/src/client/coordination/proto_accessor.cpp create mode 100644 ydb/public/sdk/cpp/src/client/coordination/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/datastreams/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/datastreams/datastreams.cpp create mode 100644 ydb/public/sdk/cpp/src/client/datastreams/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/debug/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/debug/client.cpp create mode 100644 ydb/public/sdk/cpp/src/client/debug/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/discovery/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/discovery/discovery.cpp create mode 100644 ydb/public/sdk/cpp/src/client/discovery/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/draft/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/draft/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp create mode 100644 ydb/public/sdk/cpp/src/client/draft/ydb_replication.cpp create mode 100644 ydb/public/sdk/cpp/src/client/draft/ydb_scripting.cpp create mode 100644 ydb/public/sdk/cpp/src/client/draft/ydb_view.cpp create mode 100644 ydb/public/sdk/cpp/src/client/driver/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/driver/driver.cpp create mode 100644 ydb/public/sdk/cpp/src/client/driver/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/export/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/export/export.cpp create mode 100644 ydb/public/sdk/cpp/src/client/export/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/export/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/extension_common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/extension_common/extension.cpp create mode 100644 ydb/public/sdk/cpp/src/client/extension_common/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/extensions/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/discovery_mutator.cpp create mode 100644 ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/extensions/solomon_stats/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/extensions/solomon_stats/README.md create mode 100644 ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_client.cpp create mode 100644 ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_connector.cpp create mode 100644 ydb/public/sdk/cpp/src/client/extensions/solomon_stats/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/extensions/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.h create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session_event.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.h create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.cpp create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.h create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/impl/ya.make rename ydb/public/sdk/cpp/{client/ydb_federated_topic => src/client/federated_topic}/ut/basic_usage_ut.cpp (94%) rename ydb/public/sdk/cpp/{client/ydb_federated_topic => src/client/federated_topic}/ut/fds_mock/fds_mock.h (100%) create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/ut/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/federated_topic/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/helpers/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/helpers/helpers.cpp create mode 100644 ydb/public/sdk/cpp/src/client/helpers/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/iam/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/iam/common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/iam/common/iam.h create mode 100644 ydb/public/sdk/cpp/src/client/iam/common/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/iam/iam.cpp create mode 100644 ydb/public/sdk/cpp/src/client/iam/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/iam_private/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/iam_private/iam.cpp create mode 100644 ydb/public/sdk/cpp/src/client/iam_private/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/types.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/driver/constants.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/params.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_client/client.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_header.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_async.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_sync.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/rpc_request_settings/settings.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/scheme_helpers/helpers.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_client/session_client.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/stats_extractor/extractor.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/table_helpers/helpers.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_stats/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.cpp create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.h create mode 100644 ydb/public/sdk/cpp/src/client/impl/ydb_stats/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/import/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/import/import.cpp create mode 100644 ydb/public/sdk/cpp/src/client/import/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/import/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/monitoring/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/monitoring/monitoring.cpp create mode 100644 ydb/public/sdk/cpp/src/client/monitoring/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/operation/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/operation/operation.cpp create mode 100644 ydb/public/sdk/cpp/src/client/operation/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/params/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/params/impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/params/impl.h create mode 100644 ydb/public/sdk/cpp/src/client/params/params.cpp create mode 100644 ydb/public/sdk/cpp/src/client/params/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/codecs/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/codecs/codecs.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/codecs/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/aliases.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session_messages.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/impl/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/aliases.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/client.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/control_plane.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/read_events.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/read_session.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/write_events.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/write_session.h create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/include/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/persqueue.h rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/basic_usage_ut.cpp (98%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/common_ut.cpp (93%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/compress_executor_ut.cpp (98%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/compression_ut.cpp (98%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/read_session_ut.cpp (96%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/retry_policy_ut.cpp (99%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/data_plane_helpers.cpp (94%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/data_plane_helpers.h (87%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/sdk_test_setup.h (98%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/test_server.cpp (100%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/test_server.h (98%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/test_utils.h (100%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/ut_utils.cpp (100%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/ut_utils.h (96%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ut_utils/ya.make (54%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/with_offset_ranges_mode_ut/ya.make (62%) rename ydb/public/sdk/cpp/{client/ydb_persqueue_public => src/client/persqueue_public}/ut/ya.make (54%) create mode 100644 ydb/public/sdk/cpp/src/client/persqueue_public/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/proto/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/proto/accessor.cpp create mode 100644 ydb/public/sdk/cpp/src/client/proto/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/query/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/query/client.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/client_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/client_session.h create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/exec_query.h create mode 100644 ydb/public/sdk/cpp/src/client/query/impl/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/query/query.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/stats.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/tx.cpp create mode 100644 ydb/public/sdk/cpp/src/client/query/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/rate_limiter/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/rate_limiter/rate_limiter.cpp create mode 100644 ydb/public/sdk/cpp/src/client/rate_limiter/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/resources/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/resources/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/resources/ydb_ca.cpp create mode 100644 ydb/public/sdk/cpp/src/client/resources/ydb_resources.cpp create mode 100644 ydb/public/sdk/cpp/src/client/resources/ydb_root_ca.pem create mode 100644 ydb/public/sdk/cpp/src/client/resources/ydb_sdk_version.txt create mode 100644 ydb/public/sdk/cpp/src/client/result/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/result/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/result/proto_accessor.cpp create mode 100644 ydb/public/sdk/cpp/src/client/result/result.cpp create mode 100644 ydb/public/sdk/cpp/src/client/result/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/scheme/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/scheme/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/scheme/scheme.cpp create mode 100644 ydb/public/sdk/cpp/src/client/scheme/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/ss_tasks/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/ss_tasks/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/ss_tasks/task.cpp create mode 100644 ydb/public/sdk/cpp/src/client/ss_tasks/task.h create mode 100644 ydb/public/sdk/cpp/src/client/ss_tasks/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/table/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/client_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/client_session.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/data_query.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/data_query.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/readers.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/readers.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/request_migrator.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/request_migrator.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/table_client.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/table_client.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/transaction.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/transaction.h create mode 100644 ydb/public/sdk/cpp/src/client/table/impl/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/table/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/proto_accessor.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/query_stats/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/table/query_stats/stats.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/query_stats/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/table/table.cpp create mode 100644 ydb/public/sdk/cpp/src/client/table/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/topic/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/topic/codecs/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/topic/codecs/codecs.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/codecs/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/callback_context.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/executor_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/executor_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/log_lazy.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/retry_policy.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/trace_lazy.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/common/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/common.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/common.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/counters_logger.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/deferred_commit.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/event_handlers.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/read_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/read_session.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/read_session_event.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.ipp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/transaction.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/transaction.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/write_session.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/write_session.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h create mode 100644 ydb/public/sdk/cpp/src/client/topic/impl/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/topic/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/topic/proto_accessor.cpp rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/basic_usage_ut.cpp (95%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/describe_topic_ut.cpp (95%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/local_partition_ut.cpp (94%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/topic_to_table_ut.cpp (97%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/trace_ut.cpp (99%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/managed_executor.cpp (100%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/managed_executor.h (89%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/topic_sdk_test_setup.cpp (98%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/topic_sdk_test_setup.h (66%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/trace.cpp (98%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/trace.h (94%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ut_utils/ya.make (56%) rename ydb/public/sdk/cpp/{client/ydb_topic => src/client/topic}/ut/ya.make (50%) create mode 100644 ydb/public/sdk/cpp/src/client/topic/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/core_facility/core_facility.h create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/credentials.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/login/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/login/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/from_file.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/credentials/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/exceptions/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/exceptions/exceptions.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/exceptions/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/handlers.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/operation/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/operation/operation.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/operation/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/operation/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/status/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/types/status/status.cpp create mode 100644 ydb/public/sdk/cpp/src/client/types/status/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/types/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/value/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/client/value/out.cpp create mode 100644 ydb/public/sdk/cpp/src/client/value/value.cpp create mode 100644 ydb/public/sdk/cpp/src/client/value/ya.make create mode 100644 ydb/public/sdk/cpp/src/client/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/benchmark/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/benchmark/clobber.h create mode 100644 ydb/public/sdk/cpp/src/library/decimal/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/decimal/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/decimal/yql_decimal.cpp create mode 100644 ydb/public/sdk/cpp/src/library/decimal/yql_decimal.h create mode 100644 ydb/public/sdk/cpp/src/library/decimal/yql_wide_int.h create mode 100644 ydb/public/sdk/cpp/src/library/grpc/client/CMakeLists.txt rename ydb/{ => public/sdk/cpp/src}/library/grpc/client/grpc_client_low.cpp (95%) create mode 100644 ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.h create mode 100644 ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.h create mode 100644 ydb/public/sdk/cpp/src/library/grpc/client/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/grpc/common/constants.h create mode 100644 ydb/public/sdk/cpp/src/library/issue/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/issue/utf8.cpp create mode 100644 ydb/public/sdk/cpp/src/library/issue/utf8.h create mode 100644 ydb/public/sdk/cpp/src/library/issue/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/issue/yql_issue.cpp create mode 100644 ydb/public/sdk/cpp/src/library/issue/yql_issue_message.cpp create mode 100644 ydb/public/sdk/cpp/src/library/issue/yql_issue_message.h create mode 100644 ydb/public/sdk/cpp/src/library/jwt/CMakeLists.txt rename ydb/public/{lib => sdk/cpp/src/library}/jwt/jwt.cpp (86%) create mode 100644 ydb/public/sdk/cpp/src/library/jwt/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/malloc/api/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/malloc/api/malloc.cpp create mode 100644 ydb/public/sdk/cpp/src/library/malloc/api/malloc.h create mode 100644 ydb/public/sdk/cpp/src/library/operation_id/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/operation_id/operation_id.cpp create mode 100644 ydb/public/sdk/cpp/src/library/operation_id/protos/CMakeLists.txt rename ydb/public/{lib => sdk/cpp/src/library}/operation_id/protos/operation_id.proto (100%) rename ydb/public/{lib => sdk/cpp/src/library}/operation_id/protos/ya.make (100%) create mode 100644 ydb/public/sdk/cpp/src/library/operation_id/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/obfuscate/CMakeLists.txt rename ydb/{ => public/sdk/cpp/src}/library/persqueue/obfuscate/obfuscate.cpp (75%) create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.h create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/obfuscate/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/CMakeLists.txt rename ydb/{ => public/sdk/cpp/src}/library/persqueue/topic_parser_public/topic_parser.cpp (64%) create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.h create mode 100644 ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/proto_output/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/proto_output/proto_output.cpp create mode 100644 ydb/public/sdk/cpp/src/library/retry/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/retry/protos/retry_options.proto create mode 100644 ydb/public/sdk/cpp/src/library/retry/retry.cpp create mode 100644 ydb/public/sdk/cpp/src/library/retry/retry.h create mode 100644 ydb/public/sdk/cpp/src/library/retry/retry_policy_ut.cpp create mode 100644 ydb/public/sdk/cpp/src/library/retry/retry_ut.cpp create mode 100644 ydb/public/sdk/cpp/src/library/retry/utils.cpp create mode 100644 ydb/public/sdk/cpp/src/library/retry/utils.h create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/base64/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/base64/base64.cpp create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/base64/base64.h create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/base64/base64_decode_uneven_ut.cpp create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/base64/base64_ut.cpp create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/helpers/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/helpers/helpers.cpp create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/helpers/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/string_utils/misc/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/uuid/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/src/library/uuid/uuid.cpp create mode 100644 ydb/public/sdk/cpp/src/library/uuid/uuid.h create mode 100644 ydb/public/sdk/cpp/src/library/uuid/ya.make create mode 100644 ydb/public/sdk/cpp/src/library/ya.make create mode 100644 ydb/public/sdk/cpp/src/ya.make create mode 100644 ydb/public/sdk/cpp/tests/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/integration/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.cpp create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.h create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example_data.cpp create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/main.cpp create mode 100644 ydb/public/sdk/cpp/tests/integration/basic_example_it/ya.make create mode 100644 ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp create mode 100644 ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.h create mode 100644 ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/main.cpp create mode 100644 ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/ya.make create mode 100644 ydb/public/sdk/cpp/tests/integration/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/client/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/client/coordination/coordination_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/coordination/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_server.h create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.h create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.h create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/helpers/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/ydb_scripting_response_headers_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/draft/ydb_view_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/driver/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/endpoints/endpoints_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/endpoints/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_check_helper.h create mode 100644 ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_token_source_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/ya.make rename ydb/public/sdk/cpp/{client/ydb_params => tests/unit/client/params}/params_ut.cpp (96%) rename ydb/public/sdk/cpp/{client/ydb_params/ut => tests/unit/client/params}/ya.make (62%) create mode 100644 ydb/public/sdk/cpp/tests/unit/client/result/result_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/client/result/ya.make rename ydb/public/sdk/cpp/{client/ydb_value => tests/unit/client/value}/value_ut.cpp (96%) rename ydb/public/sdk/cpp/{client/ydb_value/ut => tests/unit/client/value}/ya.make (59%) create mode 100644 ydb/public/sdk/cpp/tests/unit/client/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/library/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/library/decimal/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/library/decimal/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/library/decimal/yql_decimal_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/library/decimal/yql_wide_int_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/library/grpc_client/CMakeLists.txt rename ydb/{library/grpc/client/ut => public/sdk/cpp/tests/unit/library/grpc_client}/grpc_client_low_ut.cpp (96%) create mode 100644 ydb/public/sdk/cpp/tests/unit/library/grpc_client/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/library/issue/CMakeLists.txt create mode 100644 ydb/public/sdk/cpp/tests/unit/library/issue/utf8_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/library/issue/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/library/issue/yql_issue_ut.cpp create mode 100644 ydb/public/sdk/cpp/tests/unit/library/operation_id/CMakeLists.txt rename ydb/public/{lib => sdk/cpp/tests/unit/library}/operation_id/operation_id_ut.cpp (62%) rename ydb/public/{lib/operation_id/ut => sdk/cpp/tests/unit/library/operation_id}/ya.make (58%) create mode 100644 ydb/public/sdk/cpp/tests/unit/library/ya.make create mode 100644 ydb/public/sdk/cpp/tests/unit/ya.make create mode 100644 ydb/public/sdk/cpp/tests/ya.make diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 5fbe85d6400b..f26467ee4037 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -86,8 +86,8 @@ ydb/library/actors/http/ut sole+chunk+chunk ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk ydb/library/yql/providers/generic/connector/tests/datasource/ydb test.py.test_select_positive[column_selection_col2_COL1-kqprun] -ydb/public/sdk/cpp/client/ydb_topic/ut [*/*] chunk chunk -ydb/public/sdk/cpp/client/ydb_topic/ut [*/*]+chunk+chunk +ydb/public/sdk/cpp/client/src/topic/ut [*/*] chunk chunk +ydb/public/sdk/cpp/client/src/topic/ut [*/*]+chunk+chunk ydb/services/datastreams/ut DataStreams.TestPutRecordsCornerCases ydb/services/keyvalue/ut sole chunk chunk ydb/services/keyvalue/ut sole+chunk+chunk diff --git a/ydb/apps/pgwire/pg_ydb_connection.cpp b/ydb/apps/pgwire/pg_ydb_connection.cpp index d39f6bc1a79c..0d10d3067390 100644 --- a/ydb/apps/pgwire/pg_ydb_connection.cpp +++ b/ydb/apps/pgwire/pg_ydb_connection.cpp @@ -3,9 +3,9 @@ #include "log_impl.h" #include #include -#include -#include -#include +#include +#include +#include #include // temporarry borrowed from postgresql/src/backend/catalog/pg_type_d.h @@ -104,7 +104,7 @@ class TPgYdbConnection : public TActorBootstrapped { try { NYdb::NScripting::TExecuteYqlResult result = NYdb::NScripting::TAsyncExecuteYqlResult(feature).ExtractValue(); if (result.IsSuccess()) { - const TVector& resultSets = result.GetResultSets(); + const std::vector& resultSets = result.GetResultSets(); if (!resultSets.empty()) { NYdb::TResultSet resultSet = resultSets[0]; @@ -282,7 +282,7 @@ class TPgYdbConnection : public TActorBootstrapped { try { NYdb::NScripting::TExecuteYqlResult result = NYdb::NScripting::TAsyncExecuteYqlResult(feature).ExtractValue(); if (result.IsSuccess()) { - const TVector& resultSets = result.GetResultSets(); + const std::vector& resultSets = result.GetResultSets(); if (!resultSets.empty()) { NYdb::TResultSet resultSet = resultSets[0]; @@ -381,7 +381,7 @@ class TPgYdbConnection : public TActorBootstrapped { try { NYdb::NScripting::TExecuteYqlResult result = NYdb::NScripting::TAsyncExecuteYqlResult(feature).ExtractValue(); if (result.IsSuccess()) { - const TVector& resultSets = result.GetResultSets(); + const std::vector& resultSets = result.GetResultSets(); if (!resultSets.empty()) { NYdb::TResultSet resultSet = resultSets[0]; diff --git a/ydb/apps/pgwire/pg_ydb_connection.h b/ydb/apps/pgwire/pg_ydb_connection.h index 251bd1eb4e91..1e4c2ddae085 100644 --- a/ydb/apps/pgwire/pg_ydb_connection.h +++ b/ydb/apps/pgwire/pg_ydb_connection.h @@ -1,7 +1,7 @@ #pragma once #include -#include -#include +#include +#include namespace NPGW { diff --git a/ydb/apps/pgwire/pg_ydb_proxy.cpp b/ydb/apps/pgwire/pg_ydb_proxy.cpp index c9fd57e21264..d01bc447aaf7 100644 --- a/ydb/apps/pgwire/pg_ydb_proxy.cpp +++ b/ydb/apps/pgwire/pg_ydb_proxy.cpp @@ -5,7 +5,7 @@ #include #include #define INCLUDE_YDB_INTERNAL_H -#include +#include namespace NPGW { diff --git a/ydb/apps/pgwire/pg_ydb_proxy.h b/ydb/apps/pgwire/pg_ydb_proxy.h index 557907e53b6e..04ec6c3bef47 100644 --- a/ydb/apps/pgwire/pg_ydb_proxy.h +++ b/ydb/apps/pgwire/pg_ydb_proxy.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include namespace NPGW { diff --git a/ydb/apps/pgwire/ya.make b/ydb/apps/pgwire/ya.make index 382b65a76a03..b971292b4dde 100644 --- a/ydb/apps/pgwire/ya.make +++ b/ydb/apps/pgwire/ya.make @@ -1,5 +1,9 @@ PROGRAM() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( appdata.h log_impl.h @@ -19,8 +23,8 @@ PEERDIR( ydb/core/local_pgwire ydb/core/protos ydb/public/api/grpc - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/draft yql/essentials/parser/pg_wrapper yql/essentials/public/udf/service/exception_policy ) diff --git a/ydb/apps/ydb/commands/ya.make b/ydb/apps/ydb/commands/ya.make index 0d9e06d6ebaf..18d82d4709cc 100644 --- a/ydb/apps/ydb/commands/ya.make +++ b/ydb/apps/ydb/commands/ya.make @@ -7,7 +7,7 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/iam/common + ydb/public/sdk/cpp/src/client/iam ydb/public/lib/ydb_cli/commands ) diff --git a/ydb/apps/ydb/commands/ydb_cloud_root.cpp b/ydb/apps/ydb/commands/ydb_cloud_root.cpp index 60b8deb46dde..44821e57800d 100644 --- a/ydb/apps/ydb/commands/ydb_cloud_root.cpp +++ b/ydb/apps/ydb/commands/ydb_cloud_root.cpp @@ -2,8 +2,8 @@ #include "ydb_update.h" #include "ydb_version.h" -#include -#include +#include +#include #include #include @@ -27,7 +27,7 @@ void TClientCommandRoot::SetCredentialsGetter(TConfig& config) { return CreateOAuthCredentialsProviderFactory(config.SecurityToken); } if (config.UseStaticCredentials) { - if (config.StaticCredentials.User) { + if (!config.StaticCredentials.User.empty()) { return CreateLoginCredentialsProviderFactory(config.StaticCredentials); } } diff --git a/ydb/apps/ydb/main.cpp b/ydb/apps/ydb/main.cpp index 73ad86557895..cedf3b885b08 100644 --- a/ydb/apps/ydb/main.cpp +++ b/ydb/apps/ydb/main.cpp @@ -22,7 +22,7 @@ int main(int argc, char **argv) { Cerr << "Try \"--help\" option for more info." << Endl; return EXIT_FAILURE; } - catch (const NYdb::NConsoleClient::TYdbErrorException& e) { + catch (const NYdb::NStatusHelpers::TYdbErrorException& e) { Cerr << e; return EXIT_FAILURE; } diff --git a/ydb/apps/ydb/ut/workload-topic.cpp b/ydb/apps/ydb/ut/workload-topic.cpp index 955c6bc3ed8c..a090b9990fc7 100644 --- a/ydb/apps/ydb/ut/workload-topic.cpp +++ b/ydb/apps/ydb/ut/workload-topic.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/apps/ydb/ut/workload-transfer-topic-to-table.cpp b/ydb/apps/ydb/ut/workload-transfer-topic-to-table.cpp index 718e06b12b57..2525e113cd92 100644 --- a/ydb/apps/ydb/ut/workload-transfer-topic-to-table.cpp +++ b/ydb/apps/ydb/ut/workload-transfer-topic-to-table.cpp @@ -5,8 +5,8 @@ #include #include -#include -#include +#include +#include Y_UNIT_TEST_SUITE(YdbWorkloadTransferTopicToTable) { diff --git a/ydb/apps/ydb/ut/ya.make b/ydb/apps/ydb/ut/ya.make index ae225a5bd814..865f1380671c 100644 --- a/ydb/apps/ydb/ut/ya.make +++ b/ydb/apps/ydb/ut/ya.make @@ -24,8 +24,8 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/table ydb/public/lib/ydb_cli/commands/topic_workload ) diff --git a/ydb/apps/ydb/ut/ydb-dump.cpp b/ydb/apps/ydb/ut/ydb-dump.cpp index bdc36b575fa7..74f93380617e 100644 --- a/ydb/apps/ydb/ut/ydb-dump.cpp +++ b/ydb/apps/ydb/ut/ydb-dump.cpp @@ -5,13 +5,15 @@ #include #include -#include +#include #include #include #include +#include + Y_UNIT_TEST_SUITE(YdbDump) { Y_UNIT_TEST(NotNullTypeDump) { diff --git a/ydb/apps/ydbd/ya.make b/ydb/apps/ydbd/ya.make index 75cb114d9319..5e9770216802 100644 --- a/ydb/apps/ydbd/ya.make +++ b/ydb/apps/ydbd/ya.make @@ -68,7 +68,7 @@ PEERDIR( yql/essentials/udfs/common/url_base yql/essentials/udfs/common/yson2 yql/essentials/udfs/logs/dsv - ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs + ydb/public/sdk/cpp/src/client/persqueue_public/codecs ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/base/appdata_fwd.h b/ydb/core/base/appdata_fwd.h index ae2bc9ab9fa5..8884de838fa6 100644 --- a/ydb/core/base/appdata_fwd.h +++ b/ydb/core/base/appdata_fwd.h @@ -112,7 +112,7 @@ namespace NMonitoring { class TBusNgMonPage; } -namespace NYdb { +namespace NYdb::inline V3 { class TDriver; } diff --git a/ydb/core/client/scheme_cache_lib/ya.make b/ydb/core/client/scheme_cache_lib/ya.make index 205ac5e18274..3835aa1482a4 100644 --- a/ydb/core/client/scheme_cache_lib/ya.make +++ b/ydb/core/client/scheme_cache_lib/ya.make @@ -8,7 +8,7 @@ SRCS( PEERDIR( contrib/libs/protobuf ydb/library/actors/core - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/threading/future ydb/core/base ydb/core/client/minikql_compile diff --git a/ydb/core/client/server/msgbus_securereq.h b/ydb/core/client/server/msgbus_securereq.h index 3265bc875a04..58353cfd3dad 100644 --- a/ydb/core/client/server/msgbus_securereq.h +++ b/ydb/core/client/server/msgbus_securereq.h @@ -19,7 +19,7 @@ class TMessageBusSecureRequest> : public TMessageBusServerRequestBase>>::HandleError( MSTATUS_ERROR, TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::AccessDenied, - error.Message, + TString{error.Message}, ctx); } @@ -46,7 +46,7 @@ class TMessageBusSecureRequest response(new TBusResponseStatus(status, message)); + TAutoPtr response(new TBusResponseStatus(status, TString{message})); if (proxyStatus != TEvTxUserProxy::TResultStatus::Unknown) response->Record.SetProxyErrorCode(proxyStatus); @@ -99,7 +99,7 @@ class TMessageBusSecureRequest #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index cb428e42f003..6b679b21304e 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR || defined BLOG_NOTICE #error log macro definition clash diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index 1e0675764eac..e1af8f391f15 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/ydb/core/cms/console/ya.make b/ydb/core/cms/console/ya.make index d99c7b57ab46..98c0cafe1a8d 100644 --- a/ydb/core/cms/console/ya.make +++ b/ydb/core/cms/console/ya.make @@ -97,7 +97,8 @@ PEERDIR( ydb/library/aclib ydb/library/yaml_config ydb/public/api/protos - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/library/operation_id/protos ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp index 28d5b06bb37f..574ee1081f7c 100644 --- a/ydb/core/config/init/init.cpp +++ b/ydb/core/config/init/init.cpp @@ -161,23 +161,23 @@ class TDefaultNodeBrokerClient if (node.NodeId == result.GetNodeId()) { auto &nodeInfo = *dnConfig.MutableNodeInfo(); nodeInfo.SetNodeId(node.NodeId); - nodeInfo.SetHost(node.Host); + nodeInfo.SetHost(TString{node.Host}); nodeInfo.SetPort(node.Port); - nodeInfo.SetResolveHost(node.ResolveHost); - nodeInfo.SetAddress(node.Address); + nodeInfo.SetResolveHost(TString{node.ResolveHost}); + nodeInfo.SetAddress(TString{node.Address}); nodeInfo.SetExpire(node.Expire); NConfig::CopyNodeLocation(nodeInfo.MutableLocation(), node.Location); if (result.HasNodeName()) { - nodeInfo.SetName(result.GetNodeName()); + nodeInfo.SetName(TString{result.GetNodeName()}); outNodeName = result.GetNodeName(); } } else { auto &info = *nsConfig.AddNode(); info.SetNodeId(node.NodeId); - info.SetAddress(node.Address); + info.SetAddress(TString{node.Address}); info.SetPort(node.Port); - info.SetHost(node.Host); - info.SetInterconnectHost(node.ResolveHost); + info.SetHost(TString{node.Host}); + info.SetInterconnectHost(TString{node.ResolveHost}); NConfig::CopyNodeLocation(info.MutableLocation(), node.Location); } } @@ -478,16 +478,16 @@ void CopyNodeLocation(NActorsInterconnect::TNodeLocation* dst, const NYdb::NDisc dst->SetBody(src.Body.value()); } if (src.DataCenter) { - dst->SetDataCenter(src.DataCenter.value()); + dst->SetDataCenter(TString{src.DataCenter.value()}); } if (src.Module) { - dst->SetModule(src.Module.value()); + dst->SetModule(TString{src.Module.value()}); } if (src.Rack) { - dst->SetRack(src.Rack.value()); + dst->SetRack(TString{src.Rack.value()}); } if (src.Unit) { - dst->SetUnit(src.Unit.value()); + dst->SetUnit(TString{src.Unit.value()}); } } diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index 558db1609d2f..4d72f3d76cdd 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -2,8 +2,8 @@ #include "init.h" -#include -#include +#include +#include #include #include diff --git a/ydb/core/config/init/ya.make b/ydb/core/config/init/ya.make index f4bd3af263de..58af302fc54f 100644 --- a/ydb/core/config/init/ya.make +++ b/ydb/core/config/init/ya.make @@ -17,8 +17,8 @@ PEERDIR( yql/essentials/minikql yql/essentials/public/udf ydb/public/lib/deprecated/kicli - ydb/public/sdk/cpp/client/ydb_discovery - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/discovery + ydb/public/sdk/cpp/src/client/driver ) GENERATE_ENUM_SERIALIZATION(init.h) diff --git a/ydb/core/driver_lib/base_utils/ya.make b/ydb/core/driver_lib/base_utils/ya.make index f8f53f928e36..53a4b8715f01 100644 --- a/ydb/core/driver_lib/base_utils/ya.make +++ b/ydb/core/driver_lib/base_utils/ya.make @@ -12,7 +12,7 @@ SRCS( PEERDIR( library/cpp/deprecated/enum_codegen - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/blobstorage/pdisk ydb/core/client/server ydb/core/driver_lib/cli_config_base diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 0a620256f483..a1e823a94bc0 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -7,13 +7,13 @@ #include -#include +#include -#include +#include #include -#include +#include #include #include @@ -902,13 +902,13 @@ class TClientCommandSchemaTableOptions : public TClientCommand { virtual int Run(TConfig& config) override { int res = 0; - if (!ClientConfig.Locator) { + if (ClientConfig.Locator.empty()) { Cerr << "GRPC call error: GRPC server is not specified (MBus protocol is not supported for this command)." << Endl; return -2; } NYdbGrpc::TCallMeta meta; - if (config.SecurityToken) { + if (config.SecurityToken.empty()) { meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, config.SecurityToken}); } @@ -1002,7 +1002,7 @@ class TClientCommandSchemaTableCopy : public TClientCommand { } virtual int Run(TConfig& config) override { - if (!ClientConfig.Locator) { + if (ClientConfig.Locator.empty()) { Cerr << "GRPC call error: GRPC server is not specified (MBus protocol is not supported for this command)." << Endl; return -2; } diff --git a/ydb/core/driver_lib/cli_base/cli_grpc.h b/ydb/core/driver_lib/cli_base/cli_grpc.h index 37d22bcd69e9..21b554231194 100644 --- a/ydb/core/driver_lib/cli_base/cli_grpc.h +++ b/ydb/core/driver_lib/cli_base/cli_grpc.h @@ -2,9 +2,9 @@ #include "cli_command.h" -#include +#include #include -#include +#include #include #include #include @@ -23,7 +23,7 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, { int res = 0; - if (!clientConfig.Locator) { + if (clientConfig.Locator.empty()) { Cerr << "GRPC call error: GRPC server is not specified (MBus protocol is not supported for this command)." << Endl; return -2; } @@ -102,8 +102,8 @@ class TClientGRpcCommand : public TClientCommand { if (!commandConfig.StaticCredentials.User.empty()) { Ydb::Auth::LoginRequest request; Ydb::Operations::Operation response; - request.set_user(commandConfig.StaticCredentials.User); - request.set_password(commandConfig.StaticCredentials.Password); + request.set_user(TString(commandConfig.StaticCredentials.User)); + request.set_password(TString(commandConfig.StaticCredentials.Password)); res = DoGRpcRequest(clientConfig, request, response, &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, {}); diff --git a/ydb/core/driver_lib/cli_base/cli_kicli.cpp b/ydb/core/driver_lib/cli_base/cli_kicli.cpp index 686889a17828..3f426f2c47a1 100644 --- a/ydb/core/driver_lib/cli_base/cli_kicli.cpp +++ b/ydb/core/driver_lib/cli_base/cli_kicli.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include #include "cli_kicli.h" diff --git a/ydb/core/driver_lib/cli_base/ya.make b/ydb/core/driver_lib/cli_base/ya.make index fb3f08875db5..fa77fadf8d4e 100644 --- a/ydb/core/driver_lib/cli_base/ya.make +++ b/ydb/core/driver_lib/cli_base/ya.make @@ -12,10 +12,10 @@ PEERDIR( ydb/library/aclib ydb/public/lib/deprecated/kicli ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/resources - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/resources + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/types/credentials ydb/public/lib/ydb_cli/commands/sdk_core_access ydb/public/lib/ydb_cli/commands/ydb_discovery ) diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp index 36a9348ea063..9264c7d8d45b 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp @@ -2,7 +2,7 @@ #include "cli_cmds.h" #include -#include +#include #include namespace NKikimr { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp index f40408b2a12f..e8f527886ce4 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp @@ -2,8 +2,8 @@ #include "cli_cmds.h" -#include -#include +#include +#include #include #include diff --git a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp index 479bc3e4d6cc..536f8f95fedb 100644 --- a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include #include diff --git a/ydb/core/driver_lib/cli_utils/ya.make b/ydb/core/driver_lib/cli_utils/ya.make index e7970cd82ebf..c3ff2ddc353b 100644 --- a/ydb/core/driver_lib/cli_utils/ya.make +++ b/ydb/core/driver_lib/cli_utils/ya.make @@ -50,12 +50,12 @@ PEERDIR( ydb/core/scheme ydb/library/aclib ydb/library/folder_service/proto - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/yaml_config ydb/public/api/grpc ydb/public/api/grpc/draft ydb/public/lib/deprecated/client - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver ydb/public/lib/ydb_cli/commands/ydb_discovery ) diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index a99f06fd03de..34d6553c3486 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -10,6 +10,10 @@ ELSE() ) ENDIF() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( auto_config_initializer.cpp config.cpp @@ -40,7 +44,7 @@ PEERDIR( ydb/library/actors/protos ydb/library/actors/util library/cpp/getopt/small - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/grpc/server ydb/library/grpc/server/actors library/cpp/logger diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp index 853842c7ac70..1146e91feb3e 100644 --- a/ydb/core/external_sources/object_storage.cpp +++ b/ydb/core/external_sources/object_storage.cpp @@ -20,7 +20,8 @@ #include #include #include -#include +#include +#include #include #include @@ -479,7 +480,7 @@ struct TObjectStorageExternalSource : public IExternalSource { auto promise = NThreading::NewPromise(); auto schemaToMetadata = [meta](NThreading::TPromise metaPromise, NObjectStorage::TEvInferredFileSchema&& response) { if (!response.Status.IsSuccess()) { - metaPromise.SetValue(NYql::NCommon::ResultFromError(response.Status.GetIssues())); + metaPromise.SetValue(NYql::NCommon::ResultFromError(NYdb::NAdapters::ToYqlIssues(response.Status.GetIssues()))); return; } meta->Changed = true; diff --git a/ydb/core/external_sources/object_storage/events.h b/ydb/core/external_sources/object_storage/events.h index a327644fe1a7..65b0d5c4498a 100644 --- a/ydb/core/external_sources/object_storage/events.h +++ b/ydb/core/external_sources/object_storage/events.h @@ -11,7 +11,8 @@ #include #include #include -#include +#include +#include namespace NKikimr::NExternalSource::NObjectStorage { @@ -142,7 +143,7 @@ struct TEvInferredFileSchema : public NActors::TEventLocal #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include @@ -57,7 +57,7 @@ void WaitBucket(std::shared_ptr kikimr, const TString& externalDa ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); if (readyOp.Metadata().ExecStatus == EExecStatus::Completed) { @@ -107,7 +107,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -136,7 +136,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -167,7 +167,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Failed, readyOp.Status().GetIssues().ToString()); @@ -192,7 +192,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -220,7 +220,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) { SELECT "Hello, world!" AS Data )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); diff --git a/ydb/core/external_sources/s3/ut/ya.make b/ydb/core/external_sources/s3/ut/ya.make index eb021a8a5889..2f9f1e485d8c 100644 --- a/ydb/core/external_sources/s3/ut/ya.make +++ b/ydb/core/external_sources/s3/ut/ya.make @@ -61,7 +61,7 @@ PEERDIR( ydb/core/kqp/ut/common ydb/core/kqp/ut/federated_query/common yql/essentials/sql/pg_dummy - ydb/public/sdk/cpp/client/ydb_types/operation + ydb/public/sdk/cpp/src/client/types/operation ydb/library/actors/core ) diff --git a/ydb/core/external_sources/ya.make b/ydb/core/external_sources/ya.make index 8465ec89603b..9a93b7bb484e 100644 --- a/ydb/core/external_sources/ya.make +++ b/ydb/core/external_sources/ya.make @@ -22,8 +22,9 @@ PEERDIR( ydb/library/yql/providers/s3/object_listers ydb/library/yql/providers/s3/path_generator yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_params - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/value ) END() diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index ef033e7047e4..57064632679c 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -610,7 +610,7 @@ class TDatabaseResolver: public TActor auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, databaseAuth.StructuredToken, databaseAuth.AddBearerToToken); auto token = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); - if (token) { + if (!token.empty()) { httpRequest->Set("Authorization", token); } diff --git a/ydb/core/fq/libs/actors/nodes_manager.cpp b/ydb/core/fq/libs/actors/nodes_manager.cpp index e5310dfb1ddb..1b1906e285a5 100644 --- a/ydb/core/fq/libs/actors/nodes_manager.cpp +++ b/ydb/core/fq/libs/actors/nodes_manager.cpp @@ -8,8 +8,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/ydb/core/fq/libs/actors/pending_fetcher.cpp b/ydb/core/fq/libs/actors/pending_fetcher.cpp index 3cbc2bfe4b6d..982432daeb7e 100644 --- a/ydb/core/fq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/fq/libs/actors/pending_fetcher.cpp @@ -37,10 +37,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/actors/result_writer.cpp b/ydb/core/fq/libs/actors/result_writer.cpp index 79029c288dc8..5b239463742a 100644 --- a/ydb/core/fq/libs/actors/result_writer.cpp +++ b/ydb/core/fq/libs/actors/result_writer.cpp @@ -136,7 +136,7 @@ class TResultWriter : public NActors::TActorBootstrapped { statusCode = NYql::NDqProto::StatusIds::EXTERNAL_ERROR; } if (statusCode != NYql::NDqProto::StatusIds::UNSPECIFIED) { - SendIssuesAndSetErrorFlag(issues, statusCode); + SendIssuesAndSetErrorFlag(NYdb::NAdapters::ToYqlIssues(issues), statusCode); return; } diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp index 950f935c03a1..ebb4a350d8f8 100644 --- a/ydb/core/fq/libs/actors/run_actor.cpp +++ b/ydb/core/fq/libs/actors/run_actor.cpp @@ -1445,7 +1445,7 @@ class TRunActor : public NActors::TActorBootstrapped { LOG_D("Rate limiter resource creation finished. Success: " << ev->Get()->Status.IsSuccess()); RateLimiterResourceCreatorId = {}; if (!ev->Get()->Status.IsSuccess()) { - AddIssueWithSubIssues("Problems with rate limiter resource creation", ev->Get()->Status.GetIssues()); + AddIssueWithSubIssues("Problems with rate limiter resource creation", NYdb::NAdapters::ToYqlIssues(ev->Get()->Status.GetIssues())); LOG_D(Issues.ToOneLineString()); Finish(FederatedQuery::QueryMeta::FAILED); } else { diff --git a/ydb/core/fq/libs/actors/ya.make b/ydb/core/fq/libs/actors/ya.make index 6aff3c94bc9e..80da96cdb259 100644 --- a/ydb/core/fq/libs/actors/ya.make +++ b/ydb/core/fq/libs/actors/ya.make @@ -87,9 +87,9 @@ PEERDIR( ydb/library/yql/utils/actor_log ydb/public/api/protos ydb/public/lib/fq - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/checkpoint_storage/ut/gc_ut.cpp b/ydb/core/fq/libs/checkpoint_storage/ut/gc_ut.cpp index 212fa0e5ac17..b0e030a9385d 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ut/gc_ut.cpp +++ b/ydb/core/fq/libs/checkpoint_storage/ut/gc_ut.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make index b3e6265e4f46..e60da2de830f 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make @@ -11,7 +11,7 @@ PEERDIR( ydb/core/fq/libs/checkpoint_storage/events ydb/core/testlib/default ydb/library/security - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc) diff --git a/ydb/core/fq/libs/checkpoint_storage/ya.make b/ydb/core/fq/libs/checkpoint_storage/ya.make index 75eb06a3e6cd..f32538ded0f5 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ya.make @@ -22,8 +22,9 @@ PEERDIR( ydb/library/security ydb/library/yql/dq/actors/compute ydb/library/yql/dq/proto - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp b/ydb/core/fq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp index c270f8e7201f..8357d6f8655f 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp +++ b/ydb/core/fq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp @@ -4,7 +4,9 @@ #include #include -#include +#include +#include + #include #include #include @@ -196,7 +198,7 @@ TFuture CreateCheckpoint(const TCheckpointContextPtr& context) { TString serializedGraphDescription; if (!graphDescContext->NewGraphDescription->SerializeToString(&serializedGraphDescription)) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("Failed to serialize graph description proto"); return MakeFuture(TStatus(EStatus::BAD_REQUEST, std::move(issues))); } @@ -279,7 +281,7 @@ bool GraphDescIdExists(const TFuture& result) { TFuture GenerateGraphDescId(const TCheckpointContextPtr& context) { if (context->CheckpointGraphDescriptionContext->GraphDescId) { // already given - return MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } Y_ABORT_UNLESS(context->EntityIdGenerator); @@ -291,7 +293,7 @@ TFuture GenerateGraphDescId(const TCheckpointContextPtr& context) { return MakeFuture(result.GetValue()); } if (!GraphDescIdExists(result)) { - return MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } else { context->CheckpointGraphDescriptionContext->GraphDescId = {}; // Regenerate return GenerateGraphDescId(context); @@ -415,10 +417,10 @@ TFuture ProcessCheckpoints( *parser.ColumnParser("modified_by").GetOptionalTimestamp()); if (loadGraphDescription) { - if (const TMaybe graphDescription = parser.ColumnParser("graph_description").GetOptionalString(); graphDescription && *graphDescription) { + if (const std::optional graphDescription = parser.ColumnParser("graph_description").GetOptionalString(); graphDescription && !graphDescription.value().empty()) { NProto::TCheckpointGraphDescription graphDesc; if (!graphDesc.ParseFromString(*graphDescription)) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("Failed to deserialize graph description proto"); return MakeFuture(TStatus(EStatus::INTERNAL_ERROR, std::move(issues))); } @@ -637,7 +639,7 @@ TFuture TCheckpointStorage::Init() if (YdbConnection->DB != YdbConnection->TablePathPrefix) { auto status = YdbConnection->SchemeClient.MakeDirectory(YdbConnection->TablePathPrefix).GetValueSync(); if (!status.IsSuccess() && status.GetStatus() != EStatus::ALREADY_EXISTS) { - issues = status.GetIssues(); + issues = NYdb::NAdapters::ToYqlIssues(status.GetIssues()); TStringStream ss; ss << "Failed to create path '" << YdbConnection->TablePathPrefix << "': " << status.GetStatus(); @@ -656,7 +658,7 @@ TFuture TCheckpointStorage::Init() tableName, \ std::move(desc)).GetValueSync(); \ if (!IsTableCreated(status)) { \ - issues = status.GetIssues(); \ + issues = NYdb::NAdapters::ToYqlIssues(status.GetIssues()); \ \ TStringStream ss; \ ss << "Failed to create " << tableName \ @@ -1088,7 +1090,7 @@ TFuture TCheckpointStor TResultSetParser parser = queryResult.GetResultSetParser(0); if (parser.TryNextRow()) { - result->Size = parser.ColumnParser(0).GetOptionalUint64().GetOrElse(0); + result->Size = parser.ColumnParser(0).GetOptionalUint64().value_or(0); } else { result->Size = 0; } diff --git a/ydb/core/fq/libs/checkpoint_storage/ydb_state_storage.cpp b/ydb/core/fq/libs/checkpoint_storage/ydb_state_storage.cpp index c18e0127ce57..a1b86fdb8d8e 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ydb_state_storage.cpp +++ b/ydb/core/fq/libs/checkpoint_storage/ydb_state_storage.cpp @@ -4,7 +4,8 @@ #include #include -#include +#include +#include #include @@ -374,7 +375,7 @@ TFuture TStateStorage::Init() { //LOG_STREAMS_STORAGE_SERVICE_INFO("Creating directory: " << YdbConnection->TablePathPrefix); auto status = YdbConnection->SchemeClient.MakeDirectory(YdbConnection->TablePathPrefix).GetValueSync(); if (!status.IsSuccess() && status.GetStatus() != EStatus::ALREADY_EXISTS) { - issues = status.GetIssues(); + issues = NYdb::NAdapters::ToYqlIssues(status.GetIssues()); TStringStream ss; ss << "Failed to create path '" << YdbConnection->TablePathPrefix << "': " << status.GetStatus(); @@ -401,7 +402,7 @@ TFuture TStateStorage::Init() { auto status = CreateTable(YdbConnection, StatesTable, std::move(stateDesc)).GetValueSync(); if (!IsTableCreated(status)) { - issues = status.GetIssues(); + issues = NYdb::NAdapters::ToYqlIssues(status.GetIssues()); TStringStream ss; ss << "Failed to create " << StatesTable << " table: " << status.GetStatus(); @@ -550,9 +551,9 @@ TFuture TStateStorage::GetState( }) .Apply([context, thisPtr = TIntrusivePtr(this)](const TFuture& result) { if (!result.GetValue().IsSuccess()) { - return IStateStorage::TGetStateResult{{}, result.GetValue().GetIssues()}; + return IStateStorage::TGetStateResult{{}, NYdb::NAdapters::ToYqlIssues(result.GetValue().GetIssues())}; } - NYql::TIssues issues = result.GetValue().GetIssues(); + NYql::TIssues issues = NYdb::NAdapters::ToYqlIssues(result.GetValue().GetIssues()); auto states = thisPtr->ApplyIncrements(context, issues); return IStateStorage::TGetStateResult{std::move(states), issues}; @@ -678,21 +679,21 @@ TFuture TStateStorage::ListStates(const TContextPtr& context) { auto cnt = parser.ColumnParser("cnt").GetUint64(); if (!taskId || !coordinatorGeneration || !seqNo) { - return TStatus(EStatus::BAD_REQUEST, NYql::TIssues{NYql::TIssue{"Unexpected empty field"}}); + return TStatus(EStatus::BAD_REQUEST, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"Unexpected empty field"}}); } const auto taskIt = std::find_if(context->Tasks.begin(), context->Tasks.end(), [&] (const auto& item) { return item.TaskId == *taskId; } ); if (taskIt == context->Tasks.end()) { - return TStatus(EStatus::BAD_REQUEST, NYql::TIssues{NYql::TIssue{"Got unexpected task id"}}); + return TStatus(EStatus::BAD_REQUEST, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"Got unexpected task id"}}); } auto& taskInfo = *taskIt; TCheckpointId checkpointId(*coordinatorGeneration, *seqNo); taskInfo.ListOfStatesForReading.push_back(TContext::TStateInfo{checkpointId, cnt}); - LOG_STORAGE_DEBUG(context, "taskId " << taskId << " checkpoint id: " << checkpointId << ", rows count: " << cnt); + LOG_STORAGE_DEBUG(context, "taskId " << (taskId ? ToString(taskId.value()) : "(empty maybe)") << " checkpoint id: " << checkpointId << ", rows count: " << cnt); } } catch (const std::exception& e) { - return TStatus(EStatus::BAD_REQUEST, NYql::TIssues{NYql::TIssue{e.what()}}); + return TStatus(EStatus::BAD_REQUEST, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{e.what()}}); } return status; }); @@ -800,7 +801,7 @@ TFuture TStateStorage::SelectRowState(const TContextPtr& context) { return ProcessRowState(future.GetValue(), context); } catch (const std::exception& e) { - return TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{e.what()}}); + return TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{e.what()}}); } }); }); @@ -911,17 +912,17 @@ TFuture TStateStorage::SkipStatesInFuture(const TContextPtr& context) { } if (it->CheckpointId < context->CheckpointId) { // ListOfStatesForReading sorted by "time" - return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{"Checkpoint is not found"}}}); + return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"Checkpoint is not found"}}}); } it = taskInfo.ListOfStatesForReading.erase(it); ++eraseCount; } if (taskInfo.ListOfStatesForReading.empty()) { - return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{"Checkpoint is not found"}}}); + return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"Checkpoint is not found"}}}); } } LOG_STORAGE_DEBUG(context, "SkipStatesInFuture, skip " << eraseCount << " checkpoints"); - return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); + return MakeFuture(TStatus{EStatus::SUCCESS, NYdb::NIssue::TIssues{}}); } TFuture TStateStorage::ReadRows(const TContextPtr& context) { @@ -945,7 +946,7 @@ TFuture TStateStorage::ReadRows(const TContextPtr& context) { taskInfo.ListOfStatesForReading.pop_front(); taskInfo.CurrentProcessingRow = 0; if (taskInfo.ListOfStatesForReading.empty()) { - promise.SetValue(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{"Checkpoint is not found"}}}); + promise.SetValue(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"Checkpoint is not found"}}}); context->Callback = nullptr; return; } @@ -956,14 +957,14 @@ TFuture TStateStorage::ReadRows(const TContextPtr& context) { } else { context->Callback = nullptr; - promise.SetValue(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); + promise.SetValue(TStatus{EStatus::SUCCESS, NYdb::NIssue::TIssues{}}); return; } } auto nextFuture = thisPtr->SelectRowState(context); nextFuture.Subscribe(context->Callback); } catch (...) { - promise.SetValue(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}); + promise.SetValue(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{CurrentExceptionMessage()}}}); context->Callback = nullptr; return; } diff --git a/ydb/core/fq/libs/common/debug_info.cpp b/ydb/core/fq/libs/common/debug_info.cpp index 9f03cf48f647..500b23cc7ab5 100644 --- a/ydb/core/fq/libs/common/debug_info.cpp +++ b/ydb/core/fq/libs/common/debug_info.cpp @@ -1,6 +1,6 @@ #include "debug_info.h" -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/common/debug_info.h b/ydb/core/fq/libs/common/debug_info.h index 4c6d3f311ca8..b4e669fb8271 100644 --- a/ydb/core/fq/libs/common/debug_info.h +++ b/ydb/core/fq/libs/common/debug_info.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/common/rows_proto_splitter_ut.cpp b/ydb/core/fq/libs/common/rows_proto_splitter_ut.cpp index aa731dc9e27c..2458da7566fc 100644 --- a/ydb/core/fq/libs/common/rows_proto_splitter_ut.cpp +++ b/ydb/core/fq/libs/common/rows_proto_splitter_ut.cpp @@ -1,6 +1,6 @@ #include "rows_proto_splitter.h" -#include +#include #include namespace NFq { diff --git a/ydb/core/fq/libs/common/util.cpp b/ydb/core/fq/libs/common/util.cpp index 940fc4350414..7cc7d880f154 100644 --- a/ydb/core/fq/libs/common/util.cpp +++ b/ydb/core/fq/libs/common/util.cpp @@ -49,7 +49,7 @@ class TIssueDatabaseRemover { : DatabasePath(databasePath) {} TIntrusivePtr Run(const NYql::TIssue& issue) { - auto msg = RemoveDatabaseFromStr(issue.GetMessage(), DatabasePath); + auto msg = RemoveDatabaseFromStr(TString(issue.GetMessage()), DatabasePath); auto newIssue = MakeIntrusive(issue.Position, issue.EndPosition, msg); newIssue->SetCode(issue.GetCode(), issue.GetSeverity()); for (auto issue : issue.GetSubIssues()) { diff --git a/ydb/core/fq/libs/common/util.h b/ydb/core/fq/libs/common/util.h index 630a6bc98efa..b6cf8361eca5 100644 --- a/ydb/core/fq/libs/common/util.h +++ b/ydb/core/fq/libs/common/util.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/common/pinger.cpp b/ydb/core/fq/libs/compute/common/pinger.cpp index 84b03f40a6bb..b61b3c5de3f2 100644 --- a/ydb/core/fq/libs/compute/common/pinger.cpp +++ b/ydb/core/fq/libs/compute/common/pinger.cpp @@ -47,7 +47,7 @@ struct TEvPingResponse : public NActors::TEventLocal -#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/common/run_actor_params.cpp b/ydb/core/fq/libs/compute/common/run_actor_params.cpp index 5aaa6d65a549..055e49bf4e18 100644 --- a/ydb/core/fq/libs/compute/common/run_actor_params.cpp +++ b/ydb/core/fq/libs/compute/common/run_actor_params.cpp @@ -149,7 +149,7 @@ IOutputStream& operator<<(IOutputStream& out, const TRunActorParams& params) { << " DqGraphIndex: " << params.DqGraphIndex << " Resource.TopicConsumers: " << params.Resources.topic_consumers().size() << " ExecutionId: " << params.ExecutionId - << " OperationId: " << (params.OperationId.GetKind() != Ydb::TOperationId::UNUSED ? ProtoToString(params.OperationId) : "") + << " OperationId: " << (params.OperationId.GetKind() != NKikimr::NOperationId::TOperationId::UNUSED ? params.OperationId.ToString() : "") << " ComputeConnection: " << params.ComputeConnection.ShortDebugString() << " ResultTtl: " << params.ResultTtl << " QueryParameters: " << params.QueryParameters.size() diff --git a/ydb/core/fq/libs/compute/common/utils.cpp b/ydb/core/fq/libs/compute/common/utils.cpp index 0a70743e1e23..d43b325272fc 100644 --- a/ydb/core/fq/libs/compute/common/utils.cpp +++ b/ydb/core/fq/libs/compute/common/utils.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/compute/common/utils.h b/ydb/core/fq/libs/compute/common/utils.h index b217241b6cdb..c0a233902467 100644 --- a/ydb/core/fq/libs/compute/common/utils.h +++ b/ydb/core/fq/libs/compute/common/utils.h @@ -8,8 +8,8 @@ #include #include -#include -#include +#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/compute/ydb/actors_factory.h b/ydb/core/fq/libs/compute/ydb/actors_factory.h index 69ca9172b4b5..ac21842888a4 100644 --- a/ydb/core/fq/libs/compute/ydb/actors_factory.h +++ b/ydb/core/fq/libs/compute/ydb/actors_factory.h @@ -5,7 +5,7 @@ #include -#include +#include #include diff --git a/ydb/core/fq/libs/compute/ydb/base_compute_actor.h b/ydb/core/fq/libs/compute/ydb/base_compute_actor.h index 883599ca4a5d..e1effa9a24d2 100644 --- a/ydb/core/fq/libs/compute/ydb/base_compute_actor.h +++ b/ydb/core/fq/libs/compute/ydb/base_compute_actor.h @@ -6,7 +6,7 @@ #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/ydb/control_plane/compute_database_control_plane_service.cpp b/ydb/core/fq/libs/compute/ydb/control_plane/compute_database_control_plane_service.cpp index e6cb95298d41..73a7af92ad00 100644 --- a/ydb/core/fq/libs/compute/ydb/control_plane/compute_database_control_plane_service.cpp +++ b/ydb/core/fq/libs/compute/ydb/control_plane/compute_database_control_plane_service.cpp @@ -11,8 +11,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/ydb/control_plane/ya.make b/ydb/core/fq/libs/compute/ydb/control_plane/ya.make index 486917d343ee..66e3cc3f7a54 100644 --- a/ydb/core/fq/libs/compute/ydb/control_plane/ya.make +++ b/ydb/core/fq/libs/compute/ydb/control_plane/ya.make @@ -24,7 +24,7 @@ PEERDIR( ydb/library/yql/utils/actors ydb/public/api/grpc ydb/public/api/grpc/draft - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id/protos yql/essentials/public/issue yql/essentials/utils ) diff --git a/ydb/core/fq/libs/compute/ydb/events/events.h b/ydb/core/fq/libs/compute/ydb/events/events.h index ff9bd602c50e..715bac7ea9b5 100644 --- a/ydb/core/fq/libs/compute/ydb/events/events.h +++ b/ydb/core/fq/libs/compute/ydb/events/events.h @@ -4,9 +4,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include diff --git a/ydb/core/fq/libs/compute/ydb/events/ya.make b/ydb/core/fq/libs/compute/ydb/events/ya.make index 65ce0b42bb21..1943578b4ebb 100644 --- a/ydb/core/fq/libs/compute/ydb/events/ya.make +++ b/ydb/core/fq/libs/compute/ydb/events/ya.make @@ -10,8 +10,8 @@ PEERDIR( ydb/core/fq/libs/control_plane_storage/proto ydb/core/fq/libs/protos ydb/public/api/grpc/draft - ydb/public/lib/operation_id/protos - ydb/public/sdk/cpp/client/ydb_query + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/query ) END() diff --git a/ydb/core/fq/libs/compute/ydb/executer_actor.cpp b/ydb/core/fq/libs/compute/ydb/executer_actor.cpp index b8464eb6f4f6..578f821422df 100644 --- a/ydb/core/fq/libs/compute/ydb/executer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/executer_actor.cpp @@ -8,8 +8,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -88,14 +88,14 @@ class TExecuterActor : public TBaseComputeActor { pingCounters->LatencyMs->Collect((TInstant::Now() - StartTime).MilliSeconds()); if (ev.Get()->Get()->Success) { pingCounters->Ok->Inc(); - LOG_I("Information about the operation id and execution id is stored. ExecutionId: " << ExecutionId << " OperationId: " << ProtoToString(OperationId)); + LOG_I("Information about the operation id and execution id is stored. ExecutionId: " << ExecutionId << " OperationId: " << OperationId.ToString()); Send(Parent, new TEvYdbCompute::TEvExecuterResponse(OperationId, ExecutionId, NYdb::EStatus::SUCCESS)); CompleteAndPassAway(); } else { pingCounters->Error->Inc(); // Without the idempotency key, we lose the running operation here - LOG_E("Error saving information about the operation id and execution id. ExecutionId: " << ExecutionId << " OperationId: " << ProtoToString(OperationId)); - Send(Parent, new TEvYdbCompute::TEvExecuterResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Error saving information about the operation id and execution id. ExecutionId: " << ExecutionId << " OperationId: " << ProtoToString(OperationId)}}, NYdb::EStatus::INTERNAL_ERROR)); + LOG_E("Error saving information about the operation id and execution id. ExecutionId: " << ExecutionId << " OperationId: " << OperationId.ToString()); + Send(Parent, new TEvYdbCompute::TEvExecuterResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Error saving information about the operation id and execution id. ExecutionId: " << ExecutionId << " OperationId: " << OperationId.ToString()}}, NYdb::EStatus::INTERNAL_ERROR)); FailedAndPassAway(); } } @@ -110,7 +110,7 @@ class TExecuterActor : public TBaseComputeActor { } ExecutionId = response.ExecutionId; OperationId = response.OperationId; - LOG_I("Execution has been created. ExecutionId: " << ExecutionId << " OperationId: " << ProtoToString(OperationId)); + LOG_I("Execution has been created. ExecutionId: " << ExecutionId << " OperationId: " << OperationId.ToString()); SendPingTask(); } @@ -153,7 +153,7 @@ class TExecuterActor : public TBaseComputeActor { void SendPingTask() { Fq::Private::PingTaskRequest pingTaskRequest; pingTaskRequest.set_execution_id(ExecutionId); - pingTaskRequest.set_operation_id(ProtoToString(OperationId)); + pingTaskRequest.set_operation_id(OperationId.ToString()); pingTaskRequest.set_status(::FederatedQuery::QueryMeta::RUNNING); auto pingCounters = Counters.GetCounters(ERequestType::RT_PING); pingCounters->InFly->Inc(); diff --git a/ydb/core/fq/libs/compute/ydb/executer_actor.h b/ydb/core/fq/libs/compute/ydb/executer_actor.h index 89c17bce799e..afb778256b46 100644 --- a/ydb/core/fq/libs/compute/ydb/executer_actor.h +++ b/ydb/core/fq/libs/compute/ydb/executer_actor.h @@ -6,7 +6,7 @@ #include -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/compute/ydb/finalizer_actor.cpp b/ydb/core/fq/libs/compute/ydb/finalizer_actor.cpp index 98f4c856a730..2407837e1278 100644 --- a/ydb/core/fq/libs/compute/ydb/finalizer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/finalizer_actor.cpp @@ -9,8 +9,8 @@ #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/ydb/finalizer_actor.h b/ydb/core/fq/libs/compute/ydb/finalizer_actor.h index 9aefc217cb7f..ff92c45a4eec 100644 --- a/ydb/core/fq/libs/compute/ydb/finalizer_actor.h +++ b/ydb/core/fq/libs/compute/ydb/finalizer_actor.h @@ -4,7 +4,7 @@ #include -#include +#include #include diff --git a/ydb/core/fq/libs/compute/ydb/initializer_actor.cpp b/ydb/core/fq/libs/compute/ydb/initializer_actor.cpp index c06bac610600..95342a1c9e78 100644 --- a/ydb/core/fq/libs/compute/ydb/initializer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/initializer_actor.cpp @@ -10,8 +10,8 @@ #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/compute/ydb/initializer_actor.h b/ydb/core/fq/libs/compute/ydb/initializer_actor.h index 9d812ada16c9..5033bbce391f 100644 --- a/ydb/core/fq/libs/compute/ydb/initializer_actor.h +++ b/ydb/core/fq/libs/compute/ydb/initializer_actor.h @@ -4,7 +4,7 @@ #include -#include +#include #include diff --git a/ydb/core/fq/libs/compute/ydb/resources_cleaner_actor.cpp b/ydb/core/fq/libs/compute/ydb/resources_cleaner_actor.cpp index ac955bcb9108..2d0f0bffba58 100644 --- a/ydb/core/fq/libs/compute/ydb/resources_cleaner_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/resources_cleaner_actor.cpp @@ -10,8 +10,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -20,11 +20,11 @@ #include -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResourcesCleaner] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) namespace NFq { diff --git a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp index 1a663109ebd0..159f81d8bece 100644 --- a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp @@ -12,9 +12,9 @@ #include -#include -#include -#include +#include +#include +#include #include #include @@ -25,11 +25,11 @@ #include -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [ResultWriter] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) namespace NFq { @@ -329,7 +329,7 @@ class TResultWriterActor : public TBaseComputeActor { auto& meta = *PingTaskRequest.add_result_set_meta(); for (const auto& column: resultSetMeta.Columns) { auto& new_column = *meta.add_column(); - new_column.set_name(column.Name); + new_column.set_name(TString{column.Name}); *new_column.mutable_type() = column.Type.GetProto(); } } @@ -349,7 +349,7 @@ class TResultWriterActor : public TBaseComputeActor { } else { pingCounters->Error->Inc(); LOG_E("Move result error"); - Send(Parent, new TEvYdbCompute::TEvResultWriterResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Move result error. OperationId: " << ProtoToString(OperationId)}}, NYdb::EStatus::INTERNAL_ERROR)); + Send(Parent, new TEvYdbCompute::TEvResultWriterResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Move result error. OperationId: " << OperationId.ToString()}}, NYdb::EStatus::INTERNAL_ERROR)); FailedAndPassAway(); } } diff --git a/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp b/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp index c3da2d251762..ff929a8c4057 100644 --- a/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp @@ -14,8 +14,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -24,11 +24,11 @@ #include -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [StatusTracker] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) namespace NFq { @@ -123,7 +123,7 @@ class TStatusTrackerActor : public TBaseComputeActor { CompleteAndPassAway(); } else { LOG_E("Error saving information about the status of operation"); - Send(Parent, new TEvYdbCompute::TEvStatusTrackerResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Error saving information about the status of operation: " << ProtoToString(OperationId)}}, NYdb::EStatus::INTERNAL_ERROR, ExecStatus, ComputeStatus)); + Send(Parent, new TEvYdbCompute::TEvStatusTrackerResponse(NYql::TIssues{NYql::TIssue{TStringBuilder{} << "Error saving information about the status of operation: " << OperationId.ToString()}}, NYdb::EStatus::INTERNAL_ERROR, ExecStatus, ComputeStatus)); FailedAndPassAway(); } } diff --git a/ydb/core/fq/libs/compute/ydb/stopper_actor.cpp b/ydb/core/fq/libs/compute/ydb/stopper_actor.cpp index 0e14f76e75df..bea43670e4f6 100644 --- a/ydb/core/fq/libs/compute/ydb/stopper_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/stopper_actor.cpp @@ -13,8 +13,8 @@ #include -#include -#include +#include +#include #include #include @@ -23,11 +23,11 @@ #include -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << ProtoToString(OperationId) << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_I(stream) LOG_INFO_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] [Stopper] CloudId: " << Params.CloudId << " Scope: " << Params.Scope.ToString() << " QueryId: " << Params.QueryId << " JobId: " << Params.JobId << " OperationId: " << OperationId.ToString() << " " << stream) namespace NFq { diff --git a/ydb/core/fq/libs/compute/ydb/synchronization_service/synchronization_service.cpp b/ydb/core/fq/libs/compute/ydb/synchronization_service/synchronization_service.cpp index fe8df83ebd6c..236e65eaf6a1 100644 --- a/ydb/core/fq/libs/compute/ydb/synchronization_service/synchronization_service.cpp +++ b/ydb/core/fq/libs/compute/ydb/synchronization_service/synchronization_service.cpp @@ -13,8 +13,9 @@ #include #include -#include -#include +#include +#include +#include #include #include @@ -278,7 +279,7 @@ class TSynchronizeScopeActor : public NActors::TActorBootstrapped #include -#include -#include +#include +#include +#include #include @@ -70,13 +71,13 @@ class TYdbConnectorActor : public NActors::TActorBootstrappedSend(recipient, new TEvYdbCompute::TEvExecuteScriptResponse(response.Id(), response.Metadata().ExecutionId), 0, cookie); + actorSystem->Send(recipient, new TEvYdbCompute::TEvExecuteScriptResponse(response.Id(), TString{response.Metadata().ExecutionId}), 0, cookie); } else { actorSystem->Send( recipient, MakeResponse( database, - response.Status().GetIssues(), + NYdb::NAdapters::ToYqlIssues(response.Status().GetIssues()), response.Status().GetStatus()), 0, cookie); } @@ -98,7 +99,7 @@ class TYdbConnectorActor : public NActors::TActorBootstrappedSender, cookie = ev->Cookie, database = ComputeConnection.database()](auto future) { try { auto response = future.ExtractValueSync(); - if (response.Id().GetKind() != Ydb::TOperationId::UNUSED) { + if (response.Id().GetKind() != NKikimr::NOperationId::TOperationId::UNUSED) { actorSystem->Send( recipient, new TEvYdbCompute::TEvGetOperationResponse( @@ -106,7 +107,7 @@ class TYdbConnectorActor : public NActors::TActorBootstrapped(response.Status().GetStatus()), response.Metadata().ResultSetsMeta, response.Metadata().ExecStats, - RemoveDatabaseFromIssues(response.Status().GetIssues(), database), + RemoveDatabaseFromIssues(NYdb::NAdapters::ToYqlIssues(response.Status().GetIssues()), database), response.Ready()), 0, cookie); } else { @@ -114,7 +115,7 @@ class TYdbConnectorActor : public NActors::TActorBootstrapped( database, - response.Status().GetIssues(), + NYdb::NAdapters::ToYqlIssues(response.Status().GetIssues()), response.Status().GetStatus(), true), 0, cookie); @@ -142,13 +143,13 @@ class TYdbConnectorActor : public NActors::TActorBootstrappedSend(recipient, new TEvYdbCompute::TEvFetchScriptResultResponse(response.ExtractResultSet(), response.GetNextFetchToken()), 0, cookie); + actorSystem->Send(recipient, new TEvYdbCompute::TEvFetchScriptResultResponse(response.ExtractResultSet(), TString{response.GetNextFetchToken()}), 0, cookie); } else { actorSystem->Send( recipient, MakeResponse( database, - response.GetIssues(), + NYdb::NAdapters::ToYqlIssues(response.GetIssues()), response.GetStatus()), 0, cookie); } @@ -174,7 +175,7 @@ class TYdbConnectorActor : public NActors::TActorBootstrapped( database, - response.GetIssues(), + NYdb::NAdapters::ToYqlIssues(response.GetIssues()), response.GetStatus()), 0, cookie); } catch (...) { @@ -199,7 +200,7 @@ class TYdbConnectorActor : public NActors::TActorBootstrapped( database, - response.GetIssues(), + NYdb::NAdapters::ToYqlIssues(response.GetIssues()), response.GetStatus()), 0, cookie); } catch (...) { diff --git a/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp b/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp index 62067e83a7c4..d1a6c369add5 100644 --- a/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp @@ -21,9 +21,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "[ydb] QueryId: " << Params.QueryId << " " << stream) @@ -93,7 +93,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { } Params.ExecutionId = response.ExecutionId; Params.OperationId = response.OperationId; - LOG_I("ExecuterResponse (success). ExecutionId: " << Params.ExecutionId << " OperationId: " << ProtoToString(Params.OperationId)); + LOG_I("ExecuterResponse (success). ExecutionId: " << Params.ExecutionId << " OperationId: " << Params.OperationId.ToString()); Register(ActorFactory->CreateStatusTracker(SelfId(), Connector, Pinger, Params.OperationId).release()); } @@ -165,7 +165,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { void Handle(TEvents::TEvQueryActionResult::TPtr& ev) { LOG_I("QueryActionResult: " << FederatedQuery::QueryAction_Name(ev->Get()->Action)); // Start cancel operation only when StatusTracker or ResultWriter is running - if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED && !IsAborted && !FinalizationStarted) { + if (Params.OperationId.GetKind() != NKikimr::NOperationId::TOperationId::UNUSED && !IsAborted && !FinalizationStarted) { IsAborted = true; Register(ActorFactory->CreateStopper(SelfId(), Connector, Pinger, Params.OperationId).release()); } @@ -188,14 +188,14 @@ class TYdbRunActor : public NActors::TActorBootstrapped { Register(ActorFactory->CreateInitializer(SelfId(), Pinger).release()); break; case FederatedQuery::QueryMeta::RUNNING: - if (Params.OperationId.GetKind() == Ydb::TOperationId::UNUSED) { + if (Params.OperationId.GetKind() == NKikimr::NOperationId::TOperationId::UNUSED) { Register(ActorFactory->CreateExecuter(SelfId(), Connector, Pinger).release()); // restart query } else { Register(ActorFactory->CreateStatusTracker(SelfId(), Connector, Pinger, Params.OperationId).release()); } break; case FederatedQuery::QueryMeta::COMPLETING: - if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED) { + if (Params.OperationId.GetKind() != NKikimr::NOperationId::TOperationId::UNUSED) { Register(ActorFactory->CreateResultWriter(SelfId(), Connector, Pinger, Params.OperationId, false).release()); } else { CreateFinalizer(Params.Status); @@ -204,7 +204,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { case FederatedQuery::QueryMeta::FAILING: case FederatedQuery::QueryMeta::ABORTING_BY_USER: case FederatedQuery::QueryMeta::ABORTING_BY_SYSTEM: - if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED) { + if (Params.OperationId.GetKind() != NKikimr::NOperationId::TOperationId::UNUSED) { Register(ActorFactory->CreateStatusTracker(SelfId(), Connector, Pinger, Params.OperationId).release()); } else { CreateFinalizer(Params.Status); diff --git a/ydb/core/fq/libs/control_plane_config/control_plane_config.cpp b/ydb/core/fq/libs/control_plane_config/control_plane_config.cpp index 5a9f4962c1c5..ec5d9c274450 100644 --- a/ydb/core/fq/libs/control_plane_config/control_plane_config.cpp +++ b/ydb/core/fq/libs/control_plane_config/control_plane_config.cpp @@ -132,7 +132,7 @@ class TControlPlaneConfigActor : public NActors::TActorBootstrapped& resultSets) { + [=](TTenantExecuter& executer, const std::vector& resultSets) { auto& info = *executer.State; diff --git a/ydb/core/fq/libs/control_plane_config/ya.make b/ydb/core/fq/libs/control_plane_config/ya.make index 67b742f13f46..0ca65fc02faa 100644 --- a/ydb/core/fq/libs/control_plane_config/ya.make +++ b/ydb/core/fq/libs/control_plane_config/ya.make @@ -21,8 +21,8 @@ PEERDIR( ydb/library/security ydb/library/protobuf_printer yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/value ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ya.make b/ydb/core/fq/libs/control_plane_proxy/actors/ya.make index 8f316072b7ca..90bb786d9c10 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/actors/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/core/fq/libs/result_formatter ydb/core/kqp/provider ydb/library/db_pool/protos + ydb/public/sdk/cpp/adapters/issue ) GENERATE_ENUM_SERIALIZATION(ydb_schema_query_actor.h) diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp index e62849f7daea..94ba54a61b0b 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp +++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp @@ -10,7 +10,8 @@ #include #include #include -#include +#include +#include namespace NFq::NPrivate { @@ -313,7 +314,7 @@ class TSchemaQueryYDBActor : void SaveIssues(const TString& message, const TStatus& status) { auto issue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, message); - for (const auto& subIssue : RemoveDatabaseFromIssues(status.GetIssues(), DBPath)) { + for (const auto& subIssue : RemoveDatabaseFromIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues()), DBPath)) { issue.AddSubIssue(MakeIntrusive(subIssue)); } @@ -332,7 +333,7 @@ class TSchemaQueryYDBActor : try { return std::move(future.GetValueSync()); // can throw an exception } catch (...) { - return TStatus{EStatus::BAD_REQUEST, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}; + return TStatus{EStatus::BAD_REQUEST, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{CurrentExceptionMessage()}}}; } } @@ -406,7 +407,7 @@ class TGenerateRecoverySQLIfExternalDataSourceAlreadyExistsActor : IEventBase* MakeTimeoutEventImpl(NYql::TIssue issue) override { return new TEvPrivate::TEvRecoveryResponse( - Nothing(), TStatus{EStatus::TIMEOUT, NYql::TIssues{std::move(issue)}}); + Nothing(), TStatus{EStatus::TIMEOUT, NYdb::NIssue::TIssues{NYdb::NAdapters::ToSdkIssue(std::move(issue))}}); }; void CheckConnectionExistenceInCPS() { @@ -481,7 +482,7 @@ class TGenerateRecoverySQLIfExternalDataTableAlreadyExistsActor : IEventBase* MakeTimeoutEventImpl(NYql::TIssue issue) override { return new TEvPrivate::TEvRecoveryResponse( - Nothing(), TStatus{EStatus::TIMEOUT, NYql::TIssues{std::move(issue)}}); + Nothing(), TStatus{EStatus::TIMEOUT, NYdb::NIssue::TIssues{NYdb::NAdapters::ToSdkIssue(std::move(issue))}}); } void CheckBindingExistenceInCPS() { @@ -527,11 +528,11 @@ class TGenerateRecoverySQLIfExternalDataTableAlreadyExistsActor : bool IsPathDoesNotExistIssue(const TStatus& status) { auto oneLineError = status.GetIssues().ToOneLineString(); - return oneLineError.Contains("Path does not exist") || oneLineError.Contains("path hasn't been resolved"); + return oneLineError.contains("Path does not exist") || oneLineError.contains("path hasn't been resolved"); } bool IsPathExistsIssue(const TStatus& status) { - return status.GetIssues().ToOneLineString().Contains("error: path exist"); + return status.GetIssues().ToOneLineString().contains("error: path exist"); } } // namespace @@ -587,7 +588,7 @@ IActor* MakeCreateConnectionActor( &counters, permissions](TActorId sender, const TStatus& status) { if (status.GetStatus() == EStatus::ALREADY_EXISTS || - status.GetIssues().ToOneLineString().Contains("error: path exist")) { + status.GetIssues().ToOneLineString().contains("error: path exist")) { TActivationContext::ActorSystem()->Register( new TGenerateRecoverySQLIfExternalDataSourceAlreadyExistsActor( sender, @@ -871,7 +872,7 @@ IActor* MakeCreateBindingActor(const TActorId& proxyActorId, &counters, permissions](TActorId sender, const TStatus& status) { if (status.GetStatus() == EStatus::ALREADY_EXISTS || - status.GetIssues().ToOneLineString().Contains("error: path exist")) { + status.GetIssues().ToOneLineString().contains("error: path exist")) { TActivationContext::ActorSystem()->Register( new TGenerateRecoverySQLIfExternalDataTableAlreadyExistsActor( sender, diff --git a/ydb/core/fq/libs/control_plane_proxy/events/events.h b/ydb/core/fq/libs/control_plane_proxy/events/events.h index b434cad182ca..cab26b0f0c45 100644 --- a/ydb/core/fq/libs/control_plane_proxy/events/events.h +++ b/ydb/core/fq/libs/control_plane_proxy/events/events.h @@ -3,7 +3,7 @@ #include "type_traits.h" #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/control_plane_storage/events/events.h b/ydb/core/fq/libs/control_plane_storage/events/events.h index e6e730d1e9d7..2ea1f8911543 100644 --- a/ydb/core/fq/libs/control_plane_storage/events/events.h +++ b/ydb/core/fq/libs/control_plane_storage/events/events.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include @@ -19,7 +19,7 @@ #include #include #include -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/control_plane_storage/events/internal_events.h b/ydb/core/fq/libs/control_plane_storage/events/internal_events.h index b43f7917b262..320a62051801 100644 --- a/ydb/core/fq/libs/control_plane_storage/events/internal_events.h +++ b/ydb/core/fq/libs/control_plane_storage/events/internal_events.h @@ -7,14 +7,14 @@ namespace NFq { struct TEvControlPlaneStorageInternal { // Private struct TEvDbRequestResult : public NActors::TEventLocal { - explicit TEvDbRequestResult(const NYdb::TAsyncStatus& status, std::shared_ptr> resultSets = nullptr) + explicit TEvDbRequestResult(const NYdb::TAsyncStatus& status, std::shared_ptr> resultSets = nullptr) : Status(status) , ResultSets(std::move(resultSets)) { } NYdb::TAsyncStatus Status; - std::shared_ptr> ResultSets; + std::shared_ptr> ResultSets; }; }; diff --git a/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp b/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp index 652098e00de3..3b8057b7f8f2 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp @@ -60,7 +60,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth "WHERE `" TENANT_COLUMN_NAME"` = $tenant AND `" EXPIRE_AT_COLUMN_NAME "` >= $now;\n" ); - auto prepareParams = [=, tablePathPrefix=YdbConnection->TablePathPrefix](const TVector& resultSets) { + auto prepareParams = [=, tablePathPrefix=YdbConnection->TablePathPrefix](const std::vector& resultSets) { for (const auto& resultSet : resultSets) { TResultSetParser parser(resultSet); while (parser.TryNextRow()) { @@ -73,7 +73,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth node->set_active_workers(*parser.ColumnParser(ACTIVE_WORKERS_COLUMN_NAME).GetOptionalUint64()); node->set_memory_limit(*parser.ColumnParser(MEMORY_LIMIT_COLUMN_NAME).GetOptionalUint64()); node->set_memory_allocated(*parser.ColumnParser(MEMORY_ALLOCATED_COLUMN_NAME).GetOptionalUint64()); - node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().GetOrElse(0)); + node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().value_or(0)); node->set_node_address(*parser.ColumnParser(NODE_ADDRESS_COLUMN_NAME).GetOptionalString()); node->set_data_center(*parser.ColumnParser(DATA_CENTER_COLUMN_NAME).GetOptionalString()); } diff --git a/ydb/core/fq/libs/control_plane_storage/internal/rate_limiter_resources.cpp b/ydb/core/fq/libs/control_plane_storage/internal/rate_limiter_resources.cpp index 52fb34934638..3a1ddb3b6209 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/rate_limiter_resources.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/rate_limiter_resources.cpp @@ -73,11 +73,11 @@ class TRateLimiterRequestActor : public TControlPlaneRequestActorGet()->Status.GetValueSync(); CPS_LOG_D(TDerived::RequestTypeName << "Request. Got response from database: " << status.GetStatus()); if (!status.IsSuccess()) { - ReplyWithError(status.GetIssues()); + ReplyWithError(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); return; } - const TVector& resultSets = *ev->Get()->ResultSets; + const std::vector& resultSets = *ev->Get()->ResultSets; if (resultSets.size() != 2) { ReplyWithError(TStringBuilder() << "Result set size is not equal to 2 but equal to " << resultSets.size() << ". Please contact internal support"); return; diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp index 41bad0bb970d..fe8fcab98a6a 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp @@ -117,7 +117,7 @@ std::pair MakeSql(const TTaskInternal& taskInternal, con } // namespace -std::tuple(const TVector&)>> MakeGetTaskUpdateQuery( +std::tuple(const std::vector&)>> MakeGetTaskUpdateQuery( const TTaskInternal& taskInternal, const std::shared_ptr& responseTasks, const TInstant& nowTimestamp, @@ -145,7 +145,7 @@ std::tuple& resultSets) mutable { + auto prepareParams = [=, taskInternal=taskInternal, responseTasks=responseTasks, tenantInfo=tenantInfo](const std::vector& resultSets) mutable { auto& task = taskInternal.Task; const auto shouldAbortTask = taskInternal.ShouldAbortTask; constexpr size_t expectedResultSetsSize = 2; @@ -157,7 +157,7 @@ std::tupleParseProtobufError->Inc(); @@ -298,7 +298,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ actorSystem=NActors::TActivationContext::ActorSystem(), responseTasks=responseTasks, tenantInfo=ev->Get()->TenantInfo - ](const TVector& resultSets) mutable { + ](const std::vector& resultSets) mutable { TVector tasks; TVector pickTaskParams; const auto now = TInstant::Now(); @@ -321,13 +321,13 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ auto previousOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString(); taskInternal.RetryLimiter.Assign( - parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0), - parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero()), - parser.ColumnParser(RETRY_RATE_COLUMN_NAME).GetOptionalDouble().GetOrElse(0.0) + parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().value_or(0), + parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().value_or(TInstant::Zero()), + parser.ColumnParser(RETRY_RATE_COLUMN_NAME).GetOptionalDouble().value_or(0.0) ); - auto lastSeenAt = parser.ColumnParser(LAST_SEEN_AT_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero()); + auto lastSeenAt = parser.ColumnParser(LAST_SEEN_AT_COLUMN_NAME).GetOptionalTimestamp().value_or(TInstant::Zero()); - if (previousOwner) { // task lease timeout case only, other cases are updated at ping time + if (!previousOwner.empty()) { // task lease timeout case only, other cases are updated at ping time CPS_LOG_AS_T(*actorSystem, "Task (Query): " << task.QueryId << " Lease TIMEOUT, RetryCounterUpdatedAt " << taskInternal.RetryLimiter.RetryCounterUpdatedAt << " LastSeenAt: " << lastSeenAt); taskInternal.ShouldAbortTask = !taskInternal.RetryLimiter.UpdateOnRetry(lastSeenAt, Config->TaskLeaseRetryPolicy, now); @@ -393,7 +393,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ debugInfo->insert(debugInfo->end(), info->begin(), info->end()); } } - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; auto status = MakeFuture(TStatus{EStatus::SUCCESS, std::move(issues)}); try { future.GetValue(); diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp index 976cceeca87b..6dc0e76d93c7 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp @@ -48,7 +48,7 @@ THashMap DeserializeFlatStats(const google::protobuf::RepeatedPtrF struct TPingTaskParams { TString Query; TParams Params; - const std::function(const TVector&)> Prepare; + const std::function(const std::vector&)> Prepare; std::shared_ptr> MeteringRecords; }; @@ -91,7 +91,7 @@ TPingTaskParams ConstructHardPingTask( auto meteringRecords = std::make_shared>(); - auto prepareParams = [=, counters=counters, actorSystem = NActors::TActivationContext::ActorSystem(), request=request](const TVector& resultSets) mutable { + auto prepareParams = [=, counters=counters, actorSystem = NActors::TActivationContext::ActorSystem(), request=request](const std::vector& resultSets) mutable { TString jobId; FederatedQuery::Query query; FederatedQuery::Internal::QueryInternal internal; @@ -140,9 +140,9 @@ TPingTaskParams ConstructHardPingTask( ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "OWNER of QUERY ID = \"" << request.query_id().value() << "\" MISMATCHED: \"" << request.owner_id() << "\" (received) != \"" << owner << "\" (selected)"; } retryLimiter.Assign( - parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0), - parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero()), - parser.ColumnParser(RETRY_RATE_COLUMN_NAME).GetOptionalDouble().GetOrElse(0.0) + parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().value_or(0), + parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().value_or(TInstant::Zero()), + parser.ColumnParser(RETRY_RATE_COLUMN_NAME).GetOptionalDouble().value_or(0.0) ); } @@ -572,7 +572,7 @@ TPingTaskParams ConstructSoftPingTask( "FROM `" PENDING_SMALL_TABLE_NAME "` WHERE `" TENANT_COLUMN_NAME "` = $tenant AND `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); - auto prepareParams = [=](const TVector& resultSets) { + auto prepareParams = [=](const std::vector& resultSets) { TString owner; FederatedQuery::Internal::QueryInternal internal; diff --git a/ydb/core/fq/libs/control_plane_storage/internal/utils.h b/ydb/core/fq/libs/control_plane_storage/internal/utils.h index 8a35170c094c..778b960081d4 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/utils.h +++ b/ydb/core/fq/libs/control_plane_storage/internal/utils.h @@ -2,7 +2,7 @@ #include -#include +#include #include diff --git a/ydb/core/fq/libs/control_plane_storage/internal/ya.make b/ydb/core/fq/libs/control_plane_storage/internal/ya.make index 0fee01e6f166..ec9d3d3466aa 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/ya.make +++ b/ydb/core/fq/libs/control_plane_storage/internal/ya.make @@ -32,8 +32,8 @@ PEERDIR( yql/essentials/public/issue yql/essentials/utils ydb/public/lib/fq - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/value ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/control_plane_storage/request_actor.h b/ydb/core/fq/libs/control_plane_storage/request_actor.h index caf23b5c76bb..4f1f6aae12d7 100644 --- a/ydb/core/fq/libs/control_plane_storage/request_actor.h +++ b/ydb/core/fq/libs/control_plane_storage/request_actor.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -121,8 +121,8 @@ class TControlPlaneRequestActor : public NActors::TActorBootstrapped, result = prepare(); } } else { - issues.AddIssues(status.GetIssues()); - internalIssues.AddIssues(status.GetIssues()); + issues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); + internalIssues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); } } catch (const NYql::TCodeLineException& exception) { NYql::TIssue issue = MakeErrorIssue(exception.Code, exception.GetRawMessage()); @@ -165,7 +165,7 @@ class TControlPlaneRequestActor : public NActors::TActorBootstrapped, return static_cast(this); } - void Subscribe(NYdb::TAsyncStatus& status, std::shared_ptr> resultSets = nullptr) { + void Subscribe(NYdb::TAsyncStatus& status, std::shared_ptr> resultSets = nullptr) { status.Subscribe( [actorSystem = NActors::TActivationContext::ActorSystem(), selfId = this->SelfId(), resultSets] (const NYdb::TAsyncStatus& status) mutable { actorSystem->Send(new IEventHandle(selfId, selfId, new TEvControlPlaneStorageInternal::TEvDbRequestResult(status, std::move(resultSets)))); diff --git a/ydb/core/fq/libs/control_plane_storage/validators.cpp b/ydb/core/fq/libs/control_plane_storage/validators.cpp index 84f80761149d..c7ec7f25718d 100644 --- a/ydb/core/fq/libs/control_plane_storage/validators.cpp +++ b/ydb/core/fq/libs/control_plane_storage/validators.cpp @@ -94,9 +94,9 @@ TValidationQuery CreateModifyUniqueNameValidator(const TString& tableName, static_cast( parser.ColumnParser(VISIBILITY_COLUMN_NAME) .GetOptionalInt64() - .GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + .value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); TString oldName = - parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().GetOrElse(""); + parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().value_or(""); if (oldVisibility == visibility && oldName == name) { return false; @@ -180,7 +180,7 @@ TValidationQuery CreateRevisionValidator(const TString& tableName, return false; } - i64 revision = parser.ColumnParser(REVISION_COLUMN_NAME).GetOptionalInt64().GetOrElse(0); + i64 revision = parser.ColumnParser(REVISION_COLUMN_NAME).GetOptionalInt64().value_or(0); if (revision != previousRevision) { ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << error; } @@ -220,8 +220,8 @@ static TValidationQuery CreateAccessValidatorImpl(const TString& tableName, ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << error; } - TString queryUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().GetOrElse(""); - FederatedQuery::Acl::Visibility visibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + TString queryUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().value_or(""); + FederatedQuery::Acl::Visibility visibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); bool hasAccess = HasAccessImpl(permissions, visibility, queryUser, user, privatePermission, publicPermission); if (!hasAccess) { ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << error; @@ -321,8 +321,8 @@ TValidationQuery CreateConnectionExistsValidator(const TString& scope, ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << error; } - FederatedQuery::Acl::Visibility connectionVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); - TString connectionUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().GetOrElse(""); + FederatedQuery::Acl::Visibility connectionVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + TString connectionUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().value_or(""); if (bindingVisibility == FederatedQuery::Acl::SCOPE && connectionVisibility == FederatedQuery::Acl::PRIVATE) { ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Binding with SCOPE visibility cannot refer to connection with PRIVATE visibility"; @@ -365,9 +365,9 @@ TValidationQuery CreateConnectionOverrideBindingValidator(const TString& scope, return false; } - TString bindingUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().GetOrElse(""); - TString bindingName = parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().GetOrElse(""); - FederatedQuery::Acl::Visibility bindingVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + TString bindingUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().value_or(""); + TString bindingName = parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().value_or(""); + FederatedQuery::Acl::Visibility bindingVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); if (HasViewAccess(permissions, bindingVisibility, bindingUser, user)) { ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Connection named " << connectionName << " overrides connection from binding " << bindingName << ". Please rename this connection"; @@ -406,8 +406,8 @@ TValidationQuery CreateBindingConnectionValidator(const TString& scope, return false; } - TString privateConnectionName = parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().GetOrElse(""); - TString privateConnectionId = parser.ColumnParser(CONNECTION_ID_COLUMN_NAME).GetOptionalString().GetOrElse(""); + TString privateConnectionName = parser.ColumnParser(NAME_COLUMN_NAME).GetOptionalString().value_or(""); + TString privateConnectionId = parser.ColumnParser(CONNECTION_ID_COLUMN_NAME).GetOptionalString().value_or(""); ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "The connection with id " << connectionId << " is overridden by the private conection with id " << privateConnectionId << " (" << privateConnectionName << "). Please rename the private connection or use another connection"; }; diff --git a/ydb/core/fq/libs/control_plane_storage/validators.h b/ydb/core/fq/libs/control_plane_storage/validators.h index ec5e7080928c..f74153d26485 100644 --- a/ydb/core/fq/libs/control_plane_storage/validators.h +++ b/ydb/core/fq/libs/control_plane_storage/validators.h @@ -8,10 +8,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/control_plane_storage/ya.make b/ydb/core/fq/libs/control_plane_storage/ya.make index f7723ca379b1..a9cd382c39ea 100644 --- a/ydb/core/fq/libs/control_plane_storage/ya.make +++ b/ydb/core/fq/libs/control_plane_storage/ya.make @@ -39,8 +39,9 @@ PEERDIR( ydb/library/yql/providers/s3/path_generator yql/essentials/public/issue ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ydb/library/db_pool ydb/library/yql/providers/s3/path_generator ) diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp index 362cb5d7ec6d..84b6f687f84d 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp @@ -15,7 +15,7 @@ void CollectDebugInfo(const TString& query, const TParams& params, TSession sess if (debugInfo) { try { auto explainResult = session.ExplainDataQuery(query).GetValue(TDuration::Minutes(1)); - debugInfo->push_back({query, params, explainResult.GetPlan(), explainResult.GetAst(), {}}); + debugInfo->push_back({query, params, TString{explainResult.GetPlan()}, TString{explainResult.GetAst()}, {}}); } catch (...) { debugInfo->push_back({query, params, {}, {}, CurrentExceptionMessage()}); } @@ -354,7 +354,7 @@ void ReadIdempotencyKeyQuery(TSqlQueryBuilder& builder, const TString& scope, co } } -std::pair>> TDbRequester::Read( +std::pair>> TDbRequester::Read( const TString& query, const NYdb::TParams& params, const TRequestCounters& requestCounters, @@ -363,7 +363,7 @@ std::pair>> TDbRequester bool retryOnTli) { NActors::TActorSystem* const actorSystem = TActivationContext::ActorSystem(); - auto resultSet = std::make_shared>(); + auto resultSet = std::make_shared>(); std::shared_ptr retryCount = std::make_shared(); auto handler = [=, requestCounters=requestCounters](TSession& session) mutable { @@ -378,13 +378,13 @@ std::pair>> TDbRequester *resultSet = result.GetResultSets(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}; } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", Query: " << query); } if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{status.GetIssues()}}; } return status; }); @@ -397,7 +397,7 @@ std::pair>> TDbRequester TAsyncStatus TDbRequester::Validate( NActors::TActorSystem* actorSystem, - std::shared_ptr> transaction, + std::shared_ptr> transaction, size_t item, const TVector& validators, TSession session, @@ -406,7 +406,7 @@ TAsyncStatus TDbRequester::Validate( TTxSettings transactionMode) { if (item >= validators.size()) { - return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); + return MakeFuture(TStatus{EStatus::SUCCESS, NYdb::NIssue::TIssues{}}); } const TValidationQuery& validatonItem = validators[item]; @@ -417,7 +417,7 @@ TAsyncStatus TDbRequester::Validate( *transaction = result.GetTransaction(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return MakeFuture(TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}); + return MakeFuture(TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}); } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", Query: " << query); @@ -425,7 +425,7 @@ TAsyncStatus TDbRequester::Validate( } *successFinish = validator(result); if (*successFinish) { - return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); + return MakeFuture(TStatus{EStatus::SUCCESS, NYdb::NIssue::TIssues{}}); } return Validate(actorSystem, transaction, item + 1, validators, session, successFinish, debugInfo); }); @@ -442,7 +442,7 @@ TAsyncStatus TDbRequester::Write( { NActors::TActorSystem* const actorSystem = TActivationContext::ActorSystem(); std::shared_ptr retryCount = std::make_shared(); - auto transaction = std::make_shared>(); + auto transaction = std::make_shared>(); auto writeHandler = [=, retryOnTli=retryOnTli] (TSession session) { CollectDebugInfo(query, params, session, debugInfo); auto result = session.ExecuteDataQuery(query, validators ? TTxControl::Tx(**transaction).CommitTx() : TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); @@ -450,13 +450,13 @@ TAsyncStatus TDbRequester::Write( NYdb::NTable::TDataQueryResult result = future.GetValue(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}; } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", Query: " << query); } if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{status.GetIssues()}}; } return status; }); @@ -484,13 +484,13 @@ TAsyncStatus TDbRequester::Write( } else { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); } - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{MakeErrorIssue(exception.Code, exception.GetRawMessage())}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NAdapters::ToSdkIssue(MakeErrorIssue(exception.Code, exception.GetRawMessage()))}}); } catch (const std::exception& exception) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{exception.what()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{exception.what()}}}); } catch (...) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{CurrentExceptionMessage()}}}); } }); }; @@ -520,7 +520,7 @@ NThreading::TFuture TYdbControlPlaneStorageActor::PickTask( TAsyncStatus TDbRequester::ReadModifyWrite( const TString& readQuery, const NYdb::TParams& readParams, - const std::function(const TVector&)>& prepare, + const std::function(const std::vector&)>& prepare, const TRequestCounters& requestCounters, TDebugInfoPtr debugInfo, const TVector& validators, @@ -529,8 +529,8 @@ TAsyncStatus TDbRequester::ReadModifyWrite( { NActors::TActorSystem* const actorSystem = TActivationContext::ActorSystem(); std::shared_ptr retryCount = std::make_shared(); - auto resultSets = std::make_shared>(); - auto transaction = std::make_shared>(); + auto resultSets = std::make_shared>(); + auto transaction = std::make_shared>(); auto readModifyWriteHandler = [=](TSession session) { CollectDebugInfo(readQuery, readParams, session, debugInfo); @@ -541,7 +541,7 @@ TAsyncStatus TDbRequester::ReadModifyWrite( *transaction = result.GetTransaction(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}; } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", Query: " << readQuery); @@ -561,11 +561,11 @@ TAsyncStatus TDbRequester::ReadModifyWrite( try { auto [writeQuery, params] = future.GetValue(); if (!writeQuery) { - return transaction->Get()->Commit().Apply([actorSystem=actorSystem] (const auto& future) { + return transaction->value().Commit().Apply([actorSystem=actorSystem] (const auto& future) { auto result = future.GetValue(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}; } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", COMMIT"); @@ -579,13 +579,13 @@ TAsyncStatus TDbRequester::ReadModifyWrite( NYdb::NTable::TDataQueryResult result = future.GetValue(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}; } if (!status.IsSuccess()) { CPS_LOG_AS_W(*actorSystem, "DB Error, Status: " << status.GetStatus() << ", Issues: " << status.GetIssues().ToOneLineString() << ", Query: " << writeQuery); } if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + return TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{status.GetIssues()}}; } return status; }); @@ -595,13 +595,13 @@ TAsyncStatus TDbRequester::ReadModifyWrite( } else { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); } - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{MakeErrorIssue(exception.Code, exception.GetRawMessage())}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NAdapters::ToSdkIssue(MakeErrorIssue(exception.Code, exception.GetRawMessage()))}}); } catch (const std::exception& exception) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{exception.what()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{exception.what()}}}); } catch (...) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{CurrentExceptionMessage()}}}); } }); }; @@ -629,13 +629,13 @@ TAsyncStatus TDbRequester::ReadModifyWrite( } else { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); } - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{MakeErrorIssue(exception.Code, exception.GetRawMessage())}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NAdapters::ToSdkIssue(MakeErrorIssue(exception.Code, exception.GetRawMessage()))}}); } catch (const std::exception& exception) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{exception.what()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{exception.what()}}}); } catch (...) { CPS_LOG_AS_D(*actorSystem, "Validation: " << CurrentExceptionMessage()); - return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}); + return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{CurrentExceptionMessage()}}}); } }); }; diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp index 75f5557ea197..3eb1d7cee0bc 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp @@ -446,7 +446,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi ); std::shared_ptr>> response = std::make_shared>>(); - auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const TVector& resultSets) { + auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const std::vector& resultSets) { if (resultSets.size() != 2) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support"; } @@ -471,7 +471,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << "Connection does not exist or permission denied. Please check the connectin id or your access rights"; } - connectionVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + connectionVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); } const FederatedQuery::Acl::Visibility requestBindingVisibility = request.content().acl().visibility(); diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_compute_database.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_compute_database.cpp index 0b7ac0861548..c0e796bb5a0a 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_compute_database.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_compute_database.cpp @@ -33,7 +33,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateDatab "WHERE `" SCOPE_COLUMN_NAME "` = $scope;" ); - auto prepareParams = [=](const TVector& resultSets) { + auto prepareParams = [=](const std::vector& resultSets) { if (resultSets.size() != 1) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets.size() << ". Please contact internal support"; } @@ -190,7 +190,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyDatab "WHERE `" SCOPE_COLUMN_NAME "` = $scope;" ); - auto prepareParams = [=, synchronized = ev->Get()->Synchronized, workloadManagerSynchronized = ev->Get()->WorkloadManagerSynchronized, commonCounters=requestCounters.Common](const TVector& resultSets) { + auto prepareParams = [=, synchronized = ev->Get()->Synchronized, workloadManagerSynchronized = ev->Get()->WorkloadManagerSynchronized, commonCounters=requestCounters.Common](const std::vector& resultSets) { if (resultSets.size() != 1) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets.size() << ". Please contact internal support"; } diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp index 426ce7594725..6a8692038993 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp @@ -447,7 +447,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyConne ); std::shared_ptr>> response = std::make_shared>>(); - auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const TVector& resultSets) { + auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const std::vector& resultSets) { if (resultSets.size() != 1) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets.size() << ". Please contact internal support"; } diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h index dc212439650d..6a5ebf9b75b2 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h @@ -19,7 +19,8 @@ #include #include -#include +#include +#include #include #include @@ -252,7 +253,7 @@ class TDbRequester { { } - std::pair>> Read( + std::pair>> Read( const TString& query, const NYdb::TParams& params, const TRequestCounters& requestCounters, @@ -262,7 +263,7 @@ class TDbRequester { TAsyncStatus Validate( NActors::TActorSystem* actorSystem, - std::shared_ptr> transaction, + std::shared_ptr> transaction, size_t item, const TVector& validators, TSession session, std::shared_ptr successFinish, @@ -281,7 +282,7 @@ class TDbRequester { TAsyncStatus ReadModifyWrite( const TString& readQuery, const NYdb::TParams& readParams, - const std::function(const TVector&)>& prepare, + const std::function(const std::vector&)>& prepare, const TRequestCounters& requestCounters, TDebugInfoPtr debugInfo = {}, const TVector& validators = {}, @@ -764,8 +765,8 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped(const TVector&)> PrepareParams; + std::function(const std::vector&)> PrepareParams; TString QueryId; bool RetryOnTli = false; }; diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index fa2e67f2e27c..ae9a1a30e912 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include @@ -179,7 +179,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery ); } - auto prepareParams = [=, as=TActivationContext::ActorSystem(), commonCounters=requestCounters.Common](const TVector& resultSets) mutable { + auto prepareParams = [=, as=TActivationContext::ActorSystem(), commonCounters=requestCounters.Common](const std::vector& resultSets) mutable { const size_t countSets = (idempotencyKey ? 1 : 0) + (request.execute_mode() != FederatedQuery::SAVE ? 2 : 0); if (resultSets.size() != countSets) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to " << countSets << " but equal " << resultSets.size() << ". Please contact internal support"; @@ -700,11 +700,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetQuerySta FederatedQuery::GetQueryStatusResult result; result.set_status(static_cast(*parser.ColumnParser(STATUS_COLUMN_NAME).GetOptionalInt64())); - result.set_meta_revision(parser.ColumnParser(META_REVISION_COLUMN_NAME).GetOptionalInt64().GetOrElse(0)); + result.set_meta_revision(parser.ColumnParser(META_REVISION_COLUMN_NAME).GetOptionalInt64().value_or(0)); const auto queryVisibility = static_cast(*parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64()); const auto queryUser = *parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString(); - const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user); + const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, TString{queryUser}, user); if (!hasViewAccess) { ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << "Query does not exist or permission denied. Please check the id of the query or your access rights"; } @@ -812,7 +812,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);" ); - auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const TVector& resultSets) { + auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const std::vector& resultSets) { const size_t countSets = 1 + (request.execute_mode() != FederatedQuery::SAVE ? 2 : 0); if (resultSets.size() != countSets) { @@ -838,7 +838,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery } *internal.mutable_execution_ttl() = NProtoInterop::CastToProto(TDuration::MilliSeconds(executionLimitMills)); - const TString resultId = request.execute_mode() == FederatedQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().GetOrElse("") : ""; + const TString resultId = request.execute_mode() == FederatedQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().value_or("") : ""; const auto queryVisibility = query.content().acl().visibility(); const auto queryUser = query.meta().common().created_by(); @@ -1302,7 +1302,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n" ); - auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const TVector& resultSets) { + auto prepareParams = [=, config=Config, commonCounters=requestCounters.Common](const std::vector& resultSets) { if (resultSets.size() != 2) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support"; } @@ -1337,7 +1337,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << "Job does not exist or permission denied. Please check the id query or your access rights"; } - if (!job.ParseFromString(parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString().GetOrElse(""))) { + if (!job.ParseFromString(parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString().value_or(""))) { commonCounters->ParseProtobufError->Inc(); ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support"; } @@ -1562,8 +1562,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; } - FederatedQuery::Acl::Visibility queryVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().GetOrElse(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); - TString queryUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().GetOrElse(""); + FederatedQuery::Acl::Visibility queryVisibility = static_cast(parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64().value_or(FederatedQuery::Acl::VISIBILITY_UNSPECIFIED)); + TString queryUser = parser.ColumnParser(USER_COLUMN_NAME).GetOptionalString().value_or(""); bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user); if (!hasViewAccess) { diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_quotas.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_quotas.cpp index 3cd6d9895ecc..37ca11a049cd 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_quotas.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_quotas.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include @@ -54,7 +54,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvQuotaService::TQuotaUsageRequest::T "GROUP BY `" SCOPE_COLUMN_NAME "`\n" ); }, - [=](TQuotaCountExecuter& executer, const TVector& resultSets) { + [=](TQuotaCountExecuter& executer, const std::vector& resultSets) { TResultSetParser parser(resultSets.front()); while (parser.TryNextRow()) { auto scope = *parser.ColumnParser(SCOPE_COLUMN_NAME).GetOptionalString(); @@ -66,9 +66,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvQuotaService::TQuotaUsageRequest::T "FROM `" QUERIES_TABLE_NAME "`\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope LIMIT 1;\n" ); - builder.AddString("scope", scope); + builder.AddString("scope", TString{scope}); }, - [=](TQuotaCountExecuter& executer, const TVector& resultSets) { + [=](TQuotaCountExecuter& executer, const std::vector& resultSets) { TResultSetParser parser(resultSets.front()); if (parser.TryNextRow()) { FederatedQuery::Internal::QueryInternal internal; diff --git a/ydb/core/fq/libs/db_schema/db_schema.cpp b/ydb/core/fq/libs/db_schema/db_schema.cpp index 00926ab8230e..479cecfdf901 100644 --- a/ydb/core/fq/libs/db_schema/db_schema.cpp +++ b/ydb/core/fq/libs/db_schema/db_schema.cpp @@ -1,6 +1,6 @@ #include "db_schema.h" -#include +#include #include namespace NFq { diff --git a/ydb/core/fq/libs/db_schema/db_schema.h b/ydb/core/fq/libs/db_schema/db_schema.h index bf46b474667c..45552011c004 100644 --- a/ydb/core/fq/libs/db_schema/db_schema.h +++ b/ydb/core/fq/libs/db_schema/db_schema.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/db_schema/ya.make b/ydb/core/fq/libs/db_schema/ya.make index 818f59c5ae76..364b1a041f36 100644 --- a/ydb/core/fq/libs/db_schema/ya.make +++ b/ydb/core/fq/libs/db_schema/ya.make @@ -5,8 +5,8 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_params - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/events/events.h b/ydb/core/fq/libs/events/events.h index 1c9e91826a2a..4cb7ddadbf04 100644 --- a/ydb/core/fq/libs/events/events.h +++ b/ydb/core/fq/libs/events/events.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/fq/libs/events/ya.make b/ydb/core/fq/libs/events/ya.make index 6f94a95a62ff..08d269499d13 100644 --- a/ydb/core/fq/libs/events/ya.make +++ b/ydb/core/fq/libs/events/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/library/yql/providers/pq/proto yql/essentials/public/issue ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/health/health.cpp b/ydb/core/fq/libs/health/health.cpp index c1995210a158..bb950aa0f517 100644 --- a/ydb/core/fq/libs/health/health.cpp +++ b/ydb/core/fq/libs/health/health.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include namespace NFq { namespace { diff --git a/ydb/core/fq/libs/health/ya.make b/ydb/core/fq/libs/health/ya.make index a87122c663a1..284690bd639d 100644 --- a/ydb/core/fq/libs/health/ya.make +++ b/ydb/core/fq/libs/health/ya.make @@ -8,7 +8,7 @@ PEERDIR( ydb/library/actors/core ydb/core/fq/libs/shared_resources ydb/core/mon - ydb/public/sdk/cpp/client/ydb_discovery + ydb/public/sdk/cpp/src/client/discovery ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/mock/ya.make b/ydb/core/fq/libs/mock/ya.make index 9f0a4157912d..219773f916e7 100644 --- a/ydb/core/fq/libs/mock/ya.make +++ b/ydb/core/fq/libs/mock/ya.make @@ -37,7 +37,7 @@ PEERDIR( yql/essentials/public/issue/protos yql/essentials/sql/settings ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/private_client/events.h b/ydb/core/fq/libs/private_client/events.h index db4d975113fe..1160cec1503e 100644 --- a/ydb/core/fq/libs/private_client/events.h +++ b/ydb/core/fq/libs/private_client/events.h @@ -7,12 +7,14 @@ #include #include -#include +#include #include #include +#include + namespace NFq { struct TEvInternalService { @@ -62,11 +64,11 @@ struct TEvInternalService { Result = wrappedResult.GetResult(); } } - explicit TInternalServiceResponseEvent(const TString& errorMessage) : Status(NYdb::EStatus::INTERNAL_ERROR, {NYql::TIssue(errorMessage).SetCode(NYql::UNEXPECTED_ERROR, NYql::TSeverityIds::S_ERROR)}) { + explicit TInternalServiceResponseEvent(const TString& errorMessage) : Status(NYdb::EStatus::INTERNAL_ERROR, {NYdb::NIssue::TIssue(errorMessage).SetCode(NYdb::NIssue::UNEXPECTED_ERROR, NYdb::NIssue::ESeverity::Error)}) { } - explicit TInternalServiceResponseEvent(const TProtoResult& result) : Status(NYdb::EStatus::SUCCESS, NYql::TIssues()), Result(result) { + explicit TInternalServiceResponseEvent(const TProtoResult& result) : Status(NYdb::EStatus::SUCCESS, NYdb::NIssue::TIssues()), Result(result) { } - TInternalServiceResponseEvent(NYdb::EStatus statusCode, NYql::TIssues&& issues) : Status(statusCode, std::move(issues)) { + TInternalServiceResponseEvent(NYdb::EStatus statusCode, NYql::TIssues&& issues) : Status(statusCode, NYdb::NAdapters::ToSdkIssues(std::move(issues))) { } }; diff --git a/ydb/core/fq/libs/private_client/internal_service.cpp b/ydb/core/fq/libs/private_client/internal_service.cpp index c30213930bf5..5bc59603ee1e 100644 --- a/ydb/core/fq/libs/private_client/internal_service.cpp +++ b/ydb/core/fq/libs/private_client/internal_service.cpp @@ -38,7 +38,7 @@ class TInternalService : public NActors::TActorBootstrapped { .DiscoveryEndpoint(privateApiConfig.GetTaskServiceEndpoint()) .CredentialsProviderFactory(credentialsProviderFactory({.SaKeyFile = privateApiConfig.GetSaKeyFile(), .IamEndpoint = privateApiConfig.GetIamEndpoint()})) .SslCredentials(NYdb::TSslCredentials(privateApiConfig.GetSecureTaskService())) - .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase() : TMaybe()), + .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase() : std::optional()), counters) { } diff --git a/ydb/core/fq/libs/private_client/internal_service.h b/ydb/core/fq/libs/private_client/internal_service.h index 2052ba46a439..a74d63d5058a 100644 --- a/ydb/core/fq/libs/private_client/internal_service.h +++ b/ydb/core/fq/libs/private_client/internal_service.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/fq/libs/private_client/private_client.cpp b/ydb/core/fq/libs/private_client/private_client.cpp index e44b28110141..9fa444ee2857 100644 --- a/ydb/core/fq/libs/private_client/private_client.cpp +++ b/ydb/core/fq/libs/private_client/private_client.cpp @@ -1,7 +1,7 @@ #include "private_client.h" #include #include -#include +#include namespace NFq { diff --git a/ydb/core/fq/libs/private_client/private_client.h b/ydb/core/fq/libs/private_client/private_client.h index e2084fb5c8b2..aa5142b455d3 100644 --- a/ydb/core/fq/libs/private_client/private_client.h +++ b/ydb/core/fq/libs/private_client/private_client.h @@ -2,7 +2,7 @@ #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/private_client/ya.make b/ydb/core/fq/libs/private_client/ya.make index 5d52f451bc87..d046664f2644 100644 --- a/ydb/core/fq/libs/private_client/ya.make +++ b/ydb/core/fq/libs/private_client/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( internal_service.cpp loopback_service.cpp @@ -13,7 +17,7 @@ PEERDIR( ydb/core/fq/libs/grpc ydb/core/fq/libs/shared_resources ydb/core/protos - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/quota_manager/quota_manager.cpp b/ydb/core/fq/libs/quota_manager/quota_manager.cpp index 960b13c8e1c1..d24a4b848fbd 100644 --- a/ydb/core/fq/libs/quota_manager/quota_manager.cpp +++ b/ydb/core/fq/libs/quota_manager/quota_manager.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -416,7 +416,7 @@ class TQuotaManagementService : public NActors::TActorBootstrapped& resultSets) { + [](TReadQuotaExecuter& executer, const std::vector& resultSets) { TResultSetParser parser(resultSets.front()); while (parser.TryNextRow()) { auto name = *parser.ColumnParser(METRIC_NAME_COLUMN_NAME).GetOptionalString(); @@ -570,7 +570,7 @@ class TQuotaManagementService : public NActors::TActorBootstrapped& resultSets) { + [](TSyncQuotaExecuter& executer, const std::vector& resultSets) { TResultSetParser parser(resultSets.front()); if (parser.TryNextRow()) { auto limitUpdatedAt = parser.ColumnParser(LIMIT_UPDATED_AT_COLUMN_NAME).GetOptionalTimestamp(); diff --git a/ydb/core/fq/libs/quota_manager/quota_proxy.cpp b/ydb/core/fq/libs/quota_manager/quota_proxy.cpp index 4b84a02535e9..8425647a2a72 100644 --- a/ydb/core/fq/libs/quota_manager/quota_proxy.cpp +++ b/ydb/core/fq/libs/quota_manager/quota_proxy.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/quota_manager/ya.make b/ydb/core/fq/libs/quota_manager/ya.make index 268e87a151d0..967bdcce88a4 100644 --- a/ydb/core/fq/libs/quota_manager/ya.make +++ b/ydb/core/fq/libs/quota_manager/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/core/fq/libs/shared_resources ydb/core/protos ydb/public/api/grpc/draft - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/rate_limiter/control_plane_service/rate_limiter_control_plane_service.cpp b/ydb/core/fq/libs/rate_limiter/control_plane_service/rate_limiter_control_plane_service.cpp index ad800293511e..82f23f7d44c3 100644 --- a/ydb/core/fq/libs/rate_limiter/control_plane_service/rate_limiter_control_plane_service.cpp +++ b/ydb/core/fq/libs/rate_limiter/control_plane_service/rate_limiter_control_plane_service.cpp @@ -14,6 +14,9 @@ #include #include +#include + + #include #include @@ -185,7 +188,7 @@ struct TRateLimiterRequestsQueue { originalRequest->Sender, new TEvRateLimiter::TEvCreateResourceResponse( RateLimiterPath, - ev->Get()->Result.GetIssues() + NYdb::NAdapters::ToYqlIssues(ev->Get()->Result.GetIssues()) ), 0, // flags originalRequest->Cookie @@ -194,7 +197,7 @@ struct TRateLimiterRequestsQueue { NActors::TActivationContext::AsActorContext().Send( originalRequest->Sender, new TEvRateLimiter::TEvCreateResourceResponse( - ev->Get()->Result.GetIssues() + NYdb::NAdapters::ToYqlIssues(ev->Get()->Result.GetIssues()) ), 0, // flags originalRequest->Cookie @@ -208,7 +211,7 @@ struct TRateLimiterRequestsQueue { originalRequest->Sender, new TEvRateLimiter::TEvDeleteResourceResponse( ev->Get()->Result.IsSuccess(), - ev->Get()->Result.GetIssues() + NYdb::NAdapters::ToYqlIssues(ev->Get()->Result.GetIssues()) ), 0, // flags originalRequest->Cookie @@ -238,7 +241,7 @@ struct TRateLimiterRequestsQueue { } // Special case: when resource os already created, but have its quota altered manually if (ev->Get()->Result.GetStatus() == NYdb::EStatus::BAD_REQUEST) { - for (const NYql::TIssue& i : ev->Get()->Result.GetIssues()) { + for (const auto& i : ev->Get()->Result.GetIssues()) { if (i.GetMessage().find("Resource already exists") != TString::npos) { return true; } diff --git a/ydb/core/fq/libs/rate_limiter/control_plane_service/update_limit_actor.cpp b/ydb/core/fq/libs/rate_limiter/control_plane_service/update_limit_actor.cpp index 15522c95d3f0..324758e4bdf9 100644 --- a/ydb/core/fq/libs/rate_limiter/control_plane_service/update_limit_actor.cpp +++ b/ydb/core/fq/libs/rate_limiter/control_plane_service/update_limit_actor.cpp @@ -34,7 +34,7 @@ class TUpdateCloudRateLimitActor : public NActors::TActorBootstrapped #include -#include +#include +#include #include #include @@ -144,7 +145,7 @@ class TSingleReadRuleCreator : public TActorBootstrapped LOG_D("Failed to add read rule to `" << TopicConsumer.topic_path() << "`: " << status.GetIssues().ToOneLineString() << ". Status: " << status.GetStatus() << ". Retry after: " << nextRetryDelay); if (!nextRetryDelay) { // Not retryable - Send(Owner, MakeHolder(status.GetIssues()), 0, Index); + Send(Owner, MakeHolder(NYdb::NAdapters::ToYqlIssues(status.GetIssues())), 0, Index); PassAway(); } else { if (!CheckFinish()) { diff --git a/ydb/core/fq/libs/read_rule/read_rule_creator.h b/ydb/core/fq/libs/read_rule/read_rule_creator.h index 51873fb98272..3ce5f28ba5a3 100644 --- a/ydb/core/fq/libs/read_rule/read_rule_creator.h +++ b/ydb/core/fq/libs/read_rule/read_rule_creator.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/ydb/core/fq/libs/read_rule/read_rule_deleter.cpp b/ydb/core/fq/libs/read_rule/read_rule_deleter.cpp index 7a3837fbbe02..3eacf0ed2bab 100644 --- a/ydb/core/fq/libs/read_rule/read_rule_deleter.cpp +++ b/ydb/core/fq/libs/read_rule/read_rule_deleter.cpp @@ -4,7 +4,8 @@ #include #include -#include +#include +#include #include #include @@ -138,7 +139,7 @@ class TSingleReadRuleDeleter : public TActorBootstrapped LOG_D("Failed to remove read rule from `" << Topic.topic_path() << "`: " << status.GetIssues().ToString() << ". Status: " << status.GetStatus() << ". Retry after: " << nextRetryDelay); if (!nextRetryDelay) { // Not retryable - Send(Owner, MakeHolder(status.GetIssues()), 0, Index); + Send(Owner, MakeHolder(NYdb::NAdapters::ToYqlIssues(status.GetIssues())), 0, Index); PassAway(); } else { Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup()); diff --git a/ydb/core/fq/libs/read_rule/read_rule_deleter.h b/ydb/core/fq/libs/read_rule/read_rule_deleter.h index bcf332b0f81e..6f446e91c65a 100644 --- a/ydb/core/fq/libs/read_rule/read_rule_deleter.h +++ b/ydb/core/fq/libs/read_rule/read_rule_deleter.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/ydb/core/fq/libs/read_rule/ya.make b/ydb/core/fq/libs/read_rule/ya.make index 95e2fd49c656..8a7cf731d6ca 100644 --- a/ydb/core/fq/libs/read_rule/ya.make +++ b/ydb/core/fq/libs/read_rule/ya.make @@ -13,8 +13,9 @@ PEERDIR( ydb/library/yql/providers/dq/api/protos ydb/library/yql/providers/pq/proto ydb/public/api/protos - ydb/public/lib/operation_id/protos - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/topic ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/result_formatter/result_formatter.cpp b/ydb/core/fq/libs/result_formatter/result_formatter.cpp index 17c1eefba0dc..9e116a27efaf 100644 --- a/ydb/core/fq/libs/result_formatter/result_formatter.cpp +++ b/ydb/core/fq/libs/result_formatter/result_formatter.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include @@ -208,7 +208,7 @@ TType MakeType(NYdb::TTypeParser& parser, TContext& env) if (!node) { return nullptr; } - items.push_back({colName, node}); + items.push_back({TString{colName}, node}); } parser.CloseStruct(); return MakeStructType(items, env); @@ -251,7 +251,7 @@ TType MakeType(NYdb::TTypeParser& parser, TContext& env) case NYdb::TTypeParser::ETypeKind::Tagged: { parser.OpenTagged(); auto tag = parser.GetTag(); - auto node = MakeTaggedType(tag, MakeType(parser, env), env); + auto node = MakeTaggedType(TString{tag}, MakeType(parser, env), env); parser.CloseTagged(); return node; } diff --git a/ydb/core/fq/libs/result_formatter/result_formatter.h b/ydb/core/fq/libs/result_formatter/result_formatter.h index 23de5e2afaea..0bf654c1f26b 100644 --- a/ydb/core/fq/libs/result_formatter/result_formatter.h +++ b/ydb/core/fq/libs/result_formatter/result_formatter.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include diff --git a/ydb/core/fq/libs/result_formatter/ya.make b/ydb/core/fq/libs/result_formatter/ya.make index bdb142f15094..47efa74d2d9b 100644 --- a/ydb/core/fq/libs/result_formatter/ya.make +++ b/ydb/core/fq/libs/result_formatter/ya.make @@ -12,9 +12,9 @@ PEERDIR( yql/essentials/minikql/computation yql/essentials/public/udf ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/value yql/essentials/providers/common/codec yql/essentials/providers/common/schema/expr yql/essentials/providers/common/schema/mkql diff --git a/ydb/core/fq/libs/row_dispatcher/actors_factory.h b/ydb/core/fq/libs/row_dispatcher/actors_factory.h index 9c7dc0fac19f..340d03cd81f9 100644 --- a/ydb/core/fq/libs/row_dispatcher/actors_factory.h +++ b/ydb/core/fq/libs/row_dispatcher/actors_factory.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include namespace NFq::NRowDispatcher { diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.h b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.h index 57da103b1a76..8771b96fd7d5 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.h +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.h @@ -15,7 +15,7 @@ class IFilteredDataConsumer : public IPurecalcFilterConsumer { public: virtual NActors::TActorId GetFilterId() const = 0; virtual const TVector& GetColumnIds() const = 0; - virtual TMaybe GetNextMessageOffset() const = 0; + virtual std::optional GetNextMessageOffset() const = 0; virtual void OnFilteredBatch(ui64 firstRow, ui64 lastRow) = 0; // inclusive interval [firstRow, lastRow] diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp index 3ea677e990fa..e798ae6f047c 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp @@ -186,7 +186,7 @@ class TTopicFormatHandler : public NActors::TActor, public return ColumnsIds; } - TMaybe GetNextMessageOffset() const override { + std::optional GetNextMessageOffset() const override { return Client->GetNextMessageOffset(); } @@ -348,10 +348,10 @@ class TTopicFormatHandler : public NActors::TActor, public } public: - void ParseMessages(const TVector& messages) override { + void ParseMessages(const std::vector& messages) override { LOG_ROW_DISPATCHER_TRACE("Send " << messages.size() << " messages to parser"); - if (messages) { + if (!messages.empty()) { CurrentOffset = messages.back().GetOffset(); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h index ca028a68885a..6e2fbb82d1a0 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h @@ -19,7 +19,7 @@ class IClientDataConsumer : public TThrRefBase { virtual const TString& GetWhereFilter() const = 0; virtual TPurecalcCompileSettings GetPurecalcSettings() const = 0; virtual NActors::TActorId GetClientId() const = 0; - virtual TMaybe GetNextMessageOffset() const = 0; + virtual std::optional GetNextMessageOffset() const = 0; virtual void OnClientError(TStatus status) = 0; @@ -45,7 +45,7 @@ class ITopicFormatHandler : public TNonCopyable { }; public: - virtual void ParseMessages(const TVector& messages) = 0; + virtual void ParseMessages(const std::vector& messages) = 0; // vector of (messages batch, [offsets]) virtual TQueue>> ExtractClientData(NActors::TActorId clientId) = 0; diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp index 89472c928877..f95c11387545 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp @@ -365,7 +365,7 @@ class TJsonParser : public TTopicParserBase { } public: - void ParseMessages(const TVector& messages) override { + void ParseMessages(const std::vector& messages) override { LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); Y_ENSURE(!Buffer.Finished, "Cannot parse messages with finished buffer"); diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_abstract.h b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_abstract.h index c952ba8e4b56..c74d974b4431 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_abstract.h +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_abstract.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include @@ -25,7 +25,7 @@ class ITopicParser : public TThrRefBase, public TNonCopyable { using TPtr = TIntrusivePtr; public: - virtual void ParseMessages(const TVector& messages) = 0; + virtual void ParseMessages(const std::vector& messages) = 0; virtual void Refresh(bool force = false) = 0; virtual const TVector& GetOffsets() const = 0; diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp index 32d4f4c4c17c..fe2ff8c927c8 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp @@ -50,7 +50,7 @@ class TRawParser : public TTopicParserBase { } public: - void ParseMessages(const TVector& messages) override { + void ParseMessages(const std::vector& messages) override { LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); for (const auto& message : messages) { diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make index ec51348a980c..e72dc3299fc8 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make @@ -16,7 +16,7 @@ PEERDIR( ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common - ydb/public/sdk/cpp/client/ydb_topic/include + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic yql/essentials/minikql yql/essentials/minikql/dom diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/ut/format_handler_ut.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/ut/format_handler_ut.cpp index dd3f068fe4ed..a7a79a666e89 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/ut/format_handler_ut.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/ut/format_handler_ut.cpp @@ -73,8 +73,8 @@ class TFormatHadlerFixture : public TBaseFixture { return ClientId; } - TMaybe GetNextMessageOffset() const override { - return Nothing(); + std::optional GetNextMessageOffset() const override { + return std::nullopt; } void OnClientError(TStatus status) override { diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/ut/topic_filter_ut.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/ut/topic_filter_ut.cpp index 7f9455985358..5448123d31af 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/ut/topic_filter_ut.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/ut/topic_filter_ut.cpp @@ -46,8 +46,8 @@ class TFiterFixture : public TBaseFixture { return ColumnIds; } - TMaybe GetNextMessageOffset() const override { - return Nothing(); + std::optional GetNextMessageOffset() const override { + return std::nullopt; } void OnFilterStarted() override { diff --git a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp index 2ba97f2cc45b..6e709b594ea5 100644 --- a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp +++ b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp @@ -446,7 +446,7 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { return; } const auto& session = description.GetOwners()[0]; - TString data = session.GetData(); + auto data = TString{session.GetData()}; auto generation = session.GetOrderId(); NActorsProto::TActorId protoId; if (!protoId.ParseFromString(data)) { diff --git a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp index 5aac6c57b130..5420ed0de64e 100644 --- a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp +++ b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp @@ -9,7 +9,9 @@ #include #include -#include +#include + +#include #include @@ -121,7 +123,7 @@ class TTopicSession : public TActorBootstrapped { return ReadActorId; } - TMaybe GetNextMessageOffset() const override { + std::optional GetNextMessageOffset() const override { return NextMessageOffset; } @@ -191,7 +193,7 @@ class TTopicSession : public TActorBootstrapped { ui64 UnreadRows = 0; ui64 UnreadBytes = 0; bool DataArrivedSent = false; - TMaybe NextMessageOffset; // offset to restart topic session + std::optional NextMessageOffset; // offset to restart topic session TMaybe ProcessedNextMessageOffset; // offset of fully processed data (to save to checkpoint) // Metrics @@ -284,7 +286,7 @@ class TTopicSession : public TActorBootstrapped { void CreateTopicSession(); void CloseTopicSession(); void SubscribeOnNextEvent(); - void SendToParsing(const TVector& messages); + void SendToParsing(const std::vector& messages); void SendData(TClientsInfo& info); void FatalError(TStatus status); void SendDataArrived(TClientsInfo& client); @@ -525,7 +527,7 @@ void TTopicSession::HandleNewEvents() { LOG_ROW_DISPATCHER_TRACE("Too much used memory (" << UnreadBytes << " bytes), stop reading from yds"); break; } - TMaybe event = ReadSession->GetEvent(false); + std::optional event = ReadSession->GetEvent(false); if (!event) { break; } @@ -567,14 +569,14 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TSessionClose Self.FatalError(TStatus::Fail( NYql::NDq::YdbStatusToDqStatus(static_cast(ev.GetStatus())), - ev.GetIssues() + NYdb::NAdapters::ToYqlIssues(ev.GetIssues()) ).AddParentIssue(message)); } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TStartPartitionSessionEvent& event) { LOG_ROW_DISPATCHER_DEBUG("StartPartitionSessionEvent received"); - TMaybe minOffset; + std::optional minOffset; for (const auto& [actorId, info] : Self.Clients) { if (!minOffset || (info->NextMessageOffset && *info->NextMessageOffset < *minOffset)) { minOffset = info->NextMessageOffset; @@ -598,10 +600,10 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } TString TTopicSession::GetSessionId() const { - return ReadSession ? ReadSession->GetSessionId() : TString{"empty"}; + return ReadSession ? TString{ReadSession->GetSessionId()} : TString{"empty"}; } -void TTopicSession::SendToParsing(const TVector& messages) { +void TTopicSession::SendToParsing(const std::vector& messages) { LOG_ROW_DISPATCHER_TRACE("SendToParsing, messages: " << messages.size()); for (const auto& [_, formatHandler] : FormatHandlers) { if (formatHandler->HasClients()) { @@ -751,7 +753,7 @@ void TTopicSession::RestartSessionIfOldestClient(const TClientsInfo& info) { if (!ReadSession || !info.NextMessageOffset) { return; } - TMaybe minMessageOffset; + std::optional minMessageOffset; for (auto& [readActorId, clientPtr] : Clients) { if (info.ReadActorId == readActorId || !clientPtr->NextMessageOffset) { continue; @@ -852,8 +854,8 @@ void TTopicSession::SendStatistics() { clientStatistic.Offset = info.ProcessedNextMessageOffset.GetOrElse(0); clientStatistic.FilteredReadBytes = info.FilteredStat.Bytes; clientStatistic.ReadBytes = Statistics.Bytes; - clientStatistic.IsWaiting = LastMessageOffset + 1 < info.NextMessageOffset.GetOrElse(0); - clientStatistic.ReadLagMessages = info.NextMessageOffset.GetOrElse(0) - LastMessageOffset - 1; + clientStatistic.IsWaiting = LastMessageOffset + 1 < info.NextMessageOffset.value_or(0); + clientStatistic.ReadLagMessages = info.NextMessageOffset.value_or(0) - LastMessageOffset - 1; clientStatistic.InitialOffset = info.InitialOffset; info.FilteredStat.Clear(); sessionStatistic.Clients.emplace_back(std::move(clientStatistic)); diff --git a/ydb/core/fq/libs/row_dispatcher/ut/topic_session_ut.cpp b/ydb/core/fq/libs/row_dispatcher/ut/topic_session_ut.cpp index 75af42af21bd..496cfc462602 100644 --- a/ydb/core/fq/libs/row_dispatcher/ut/topic_session_ut.cpp +++ b/ydb/core/fq/libs/row_dispatcher/ut/topic_session_ut.cpp @@ -263,7 +263,8 @@ class TFixture : public NTests::TBaseFixture { NActors::TActorId RowDispatcherActorId; NActors::TActorId CompileNotifier; NActors::TActorId PqGatewayNotifier; - NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr"))); + NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig() + .SetLog(std::unique_ptr(CreateLogBackend("cerr").Release()))); std::shared_ptr CredentialsProviderFactory; NActors::TActorId ReadActorId1; NActors::TActorId ReadActorId2; diff --git a/ydb/core/fq/libs/row_dispatcher/ya.make b/ydb/core/fq/libs/row_dispatcher/ya.make index d1039a939529..1fa6af603693 100644 --- a/ydb/core/fq/libs/row_dispatcher/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/ya.make @@ -29,8 +29,9 @@ PEERDIR( ydb/library/yql/dq/proto ydb/library/yql/providers/pq/provider - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/shared_resources/db_exec.h b/ydb/core/fq/libs/shared_resources/db_exec.h index 3e53969f63e7..d779cd8040aa 100644 --- a/ydb/core/fq/libs/shared_resources/db_exec.h +++ b/ydb/core/fq/libs/shared_resources/db_exec.h @@ -85,7 +85,7 @@ class TDbExecuter : public TDbExecutable { public: using TCallback = std::function&)>; using TBuildCallback = std::function&, TSqlQueryBuilder&)>; - using TResultCallback = std::function&, const TVector&)>; + using TResultCallback = std::function&, const std::vector&)>; private: struct TExecStep { @@ -99,7 +99,7 @@ class TDbExecuter : public TDbExecutable { ui32 CurrentStepIndex = 0; ui32 InsertStepIndex = 0; NActors::TActorId HandlerActorId; - TMaybe Transaction; + std::optional Transaction; NActors::TActorSystem* ActorSystem = nullptr; TCallback HandlerCallback; TCallback StateInitCallback; @@ -150,14 +150,14 @@ class TDbExecuter : public TDbExecutable { .Apply([selfHolder=SelfHolder, session=session](const TFuture& future) { auto self = Lock(selfHolder); if (!self) { - return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{"self has been deleted"}}}); + return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"self has been deleted"}}}); } TCommitTransactionResult result = future.GetValue(); auto status = static_cast(result); if (!status.IsSuccess()) { return MakeFuture(status); } else { - self->Transaction.Clear(); + self->Transaction.reset(); return self->NextStep(session); } }); @@ -170,7 +170,7 @@ class TDbExecuter : public TDbExecutable { })); } } - return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); + return MakeFuture(TStatus{EStatus::SUCCESS, NYdb::NIssue::TIssues{}}); } else { TSqlQueryBuilder builder(TablePathPrefix, Steps[CurrentStepIndex].Name); SkipStep_ = false; @@ -191,23 +191,23 @@ class TDbExecuter : public TDbExecutable { .Apply([selfHolder=SelfHolder, session=session](const TFuture& future) { auto self = Lock(selfHolder); if (!self) { - return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYql::TIssues{NYql::TIssue{"self has been deleted"}}}); + return MakeFuture(TStatus{EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue{"self has been deleted"}}}); } NYdb::NTable::TDataQueryResult result = future.GetValue(); auto status = static_cast(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist - self->Transaction.Clear(); - return MakeFuture(TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}); + self->Transaction.reset(); + return MakeFuture(TStatus{EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{status.GetIssues()}}); } if (!status.IsSuccess()) { - self->Transaction.Clear(); + self->Transaction.reset(); return MakeFuture(status); } if (self->Steps[self->CurrentStepIndex].Commit) { - self->Transaction.Clear(); + self->Transaction.reset(); } else if (!self->Transaction) { self->Transaction = result.GetTransaction(); } diff --git a/ydb/core/fq/libs/shared_resources/shared_resources.cpp b/ydb/core/fq/libs/shared_resources/shared_resources.cpp index 2e350130bf65..680728011a18 100644 --- a/ydb/core/fq/libs/shared_resources/shared_resources.cpp +++ b/ydb/core/fq/libs/shared_resources/shared_resources.cpp @@ -6,8 +6,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -84,7 +84,7 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes cfg.SetGrpcMemoryQuota(config.GetGrpcMemoryQuota()); } cfg.SetDiscoveryMode(NYdb::EDiscoveryMode::Async); // We are in actor system! - cfg.SetLog(MakeHolder(ActorSystemPtr, NKikimrServices::EServiceKikimr::YDB_SDK)); + cfg.SetLog(std::make_unique(ActorSystemPtr, NKikimrServices::EServiceKikimr::YDB_SDK)); return cfg; } diff --git a/ydb/core/fq/libs/shared_resources/shared_resources.h b/ydb/core/fq/libs/shared_resources/shared_resources.h index a08b0d8a49ef..7fbd70046290 100644 --- a/ydb/core/fq/libs/shared_resources/shared_resources.h +++ b/ydb/core/fq/libs/shared_resources/shared_resources.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/shared_resources/ya.make b/ydb/core/fq/libs/shared_resources/ya.make index 8087a4b4dc4d..89b86ad0547a 100644 --- a/ydb/core/fq/libs/shared_resources/ya.make +++ b/ydb/core/fq/libs/shared_resources/ya.make @@ -20,10 +20,10 @@ PEERDIR( ydb/library/logger ydb/library/security yql/essentials/utils - ydb/public/sdk/cpp/client/extensions/solomon_stats - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_extension - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/extensions/solomon_stats + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/extension_common + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/test_connection/test_object_storage.cpp b/ydb/core/fq/libs/test_connection/test_object_storage.cpp index f86c0ab19252..439f96edefc3 100644 --- a/ydb/core/fq/libs/test_connection/test_object_storage.cpp +++ b/ydb/core/fq/libs/test_connection/test_object_storage.cpp @@ -157,7 +157,7 @@ class TTestObjectStorageConnectionActor : public NActors::TActorBootstrappedCreateProvider()->GetAuthInfo(); TString requestId = CreateGuidAsString(); - NYql::IHTTPGateway::THeaders headers = NYql::IHTTPGateway::MakeYcHeaders(requestId, authToken, {}); + NYql::IHTTPGateway::THeaders headers = NYql::IHTTPGateway::MakeYcHeaders(requestId, TString{authToken}, {}); const auto retryPolicy = NYql::IHTTPGateway::TRetryPolicy::GetExponentialBackoffPolicy(RetryS3SlowDown); diff --git a/ydb/core/fq/libs/ydb/schema.cpp b/ydb/core/fq/libs/ydb/schema.cpp index 57cfe39de540..ff25ff42c700 100644 --- a/ydb/core/fq/libs/ydb/schema.cpp +++ b/ydb/core/fq/libs/ydb/schema.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -452,7 +452,7 @@ class TCreateRateLimiterResourceActor : public TRecursiveCreateActorBase::GetEntityName() << ". Attempt to create rate limiter resource root without limit"); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Internal error: attempt to create rate limiter resource root \"" << RequestsPath[0].Path << "\" without limit"); NYdb::TStatus status(NYdb::EStatus::INTERNAL_ERROR, std::move(issues)); ReplyStatusAndDie(status); @@ -474,7 +474,11 @@ class TCreateRateLimiterResourceActor : public TRecursiveCreateActorBaseRateLimiterClient.CreateResource(CoordinationNodePath, req.Path, NYdb::NRateLimiter::TCreateResourceSettings().MaxUnitsPerSecond(req.Limit)); + return Connection->RateLimiterClient.CreateResource( + CoordinationNodePath, + req.Path, + NYdb::NRateLimiter::TCreateResourceSettings() + .MaxUnitsPerSecond(req.Limit ? std::optional(req.Limit.GetRef()) : std::optional())); } private: @@ -537,7 +541,12 @@ class TUpdateRateLimiterResourceActor : public TUpdateActorBase { } NYdb::TAsyncStatus CallYdbSdk() override { - return Connection->RateLimiterClient.AlterResource(CoordinationNodePath, ResourcePath, NYdb::NRateLimiter::TAlterResourceSettings().MaxUnitsPerSecond(Limit)); + return Connection->RateLimiterClient.AlterResource( + CoordinationNodePath, + ResourcePath, + NYdb::NRateLimiter::TAlterResourceSettings() + .MaxUnitsPerSecond(Limit ? std::optional(Limit.GetRef()) : std::optional()) + ); } private: diff --git a/ydb/core/fq/libs/ydb/schema.h b/ydb/core/fq/libs/ydb/schema.h index e068013dac29..2391934ab99e 100644 --- a/ydb/core/fq/libs/ydb/schema.h +++ b/ydb/core/fq/libs/ydb/schema.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include diff --git a/ydb/core/fq/libs/ydb/ut/ydb_ut.cpp b/ydb/core/fq/libs/ydb/ut/ydb_ut.cpp index a0cc5edfdee6..c25da6eed02b 100644 --- a/ydb/core/fq/libs/ydb/ut/ydb_ut.cpp +++ b/ydb/core/fq/libs/ydb/ut/ydb_ut.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include diff --git a/ydb/core/fq/libs/ydb/util.h b/ydb/core/fq/libs/ydb/util.h index 55d5b2219890..c657915c4e7b 100644 --- a/ydb/core/fq/libs/ydb/util.h +++ b/ydb/core/fq/libs/ydb/util.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/ydb/core/fq/libs/ydb/ya.make b/ydb/core/fq/libs/ydb/ya.make index 6e392e79c13b..13072959191c 100644 --- a/ydb/core/fq/libs/ydb/ya.make +++ b/ydb/core/fq/libs/ydb/ya.make @@ -13,10 +13,11 @@ PEERDIR( ydb/core/fq/libs/config ydb/core/fq/libs/events ydb/library/security - ydb/public/sdk/cpp/client/ydb_coordination - ydb/public/sdk/cpp/client/ydb_rate_limiter - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/coordination + ydb/public/sdk/cpp/src/client/rate_limiter + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ) GENERATE_ENUM_SERIALIZATION(ydb.h) diff --git a/ydb/core/fq/libs/ydb/ydb.cpp b/ydb/core/fq/libs/ydb/ydb.cpp index 3bcdc01f5fbe..d03e2c71ecdb 100644 --- a/ydb/core/fq/libs/ydb/ydb.cpp +++ b/ydb/core/fq/libs/ydb/ydb.cpp @@ -2,6 +2,8 @@ #include "util.h" +#include + #include #include @@ -53,7 +55,7 @@ TFuture CheckGeneration( TResultSetParser parser(selectResult.GetResultSet(0)); if (parser.TryNextRow()) { - context->GenerationRead = parser.ColumnParser(context->GenerationColumn).GetOptionalUint64().GetOrElse(0); + context->GenerationRead = parser.ColumnParser(context->GenerationColumn).GetOptionalUint64().value_or(0); } bool isOk = false; @@ -82,6 +84,7 @@ TFuture CheckGeneration( } context->Transaction = selectResult.GetTransaction(); + selectResult.GetTransaction().reset(); if (!isOk) { RollbackTransaction(context); // don't care about result @@ -137,7 +140,7 @@ TFuture UpsertGeneration(const TGenerationContextPtr& context) { auto ttxControl = TTxControl::Tx(*context->Transaction); if (context->CommitTx) { ttxControl.CommitTx(); - context->Transaction.Clear(); + context->Transaction.reset(); } return context->Session.ExecuteDataQuery(query, ttxControl).Apply( @@ -196,13 +199,13 @@ TStatus MakeErrorStatus( auto& issue = issues.back(); issue.SetCode((ui32)code, severity); - return TStatus(code, std::move(issues)); + return TStatus(code, NYdb::NAdapters::ToSdkIssues(std::move(issues))); } NYql::TIssues StatusToIssues(const NYdb::TStatus& status) { TIssues issues; if (!status.IsSuccess()) { - issues = status.GetIssues(); + issues = NYdb::NAdapters::ToYqlIssues(status.GetIssues()); } return issues; } @@ -298,7 +301,7 @@ TFuture RollbackTransaction(const TGenerationContextPtr& context) { } auto future = context->Transaction->Rollback(); - context->Transaction.Clear(); + context->Transaction.reset(); return future; } diff --git a/ydb/core/fq/libs/ydb/ydb.h b/ydb/core/fq/libs/ydb/ydb.h index 200de1df1044..3856469ae698 100644 --- a/ydb/core/fq/libs/ydb/ydb.h +++ b/ydb/core/fq/libs/ydb/ydb.h @@ -1,12 +1,13 @@ #pragma once #include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -74,7 +75,7 @@ struct TGenerationContext : public TThrRefBase { // it with Transaction (must have CommitTx = false) const ui64 Generation; - TMaybe Transaction; + std::optional Transaction; // result of Select ui64 GenerationRead = 0; diff --git a/ydb/core/graph/service/ya.make b/ydb/core/graph/service/ya.make index f97b1c8743ce..65d54a4312da 100644 --- a/ydb/core/graph/service/ya.make +++ b/ydb/core/graph/service/ya.make @@ -8,6 +8,7 @@ SRCS( PEERDIR( ydb/core/base ydb/core/graph/api + ydb/public/sdk/cpp/src/client/params ) END() diff --git a/ydb/core/grpc_caching/cache_item.h b/ydb/core/grpc_caching/cache_item.h index 92bd0fc79116..6aaf4afeb541 100644 --- a/ydb/core/grpc_caching/cache_item.h +++ b/ydb/core/grpc_caching/cache_item.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace NKikimr { diff --git a/ydb/core/grpc_caching/cached_grpc_request_actor.h b/ydb/core/grpc_caching/cached_grpc_request_actor.h index dc70f46ef494..375bb159358c 100644 --- a/ydb/core/grpc_caching/cached_grpc_request_actor.h +++ b/ydb/core/grpc_caching/cached_grpc_request_actor.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/grpc_caching/ya.make b/ydb/core/grpc_caching/ya.make index e17c0df7ba0e..e35d1fdf1e83 100644 --- a/ydb/core/grpc_caching/ya.make +++ b/ydb/core/grpc_caching/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( ydb/library/actors/protos ydb/core/base + ydb/public/sdk/cpp/src/library/grpc/client ) END() diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 588fd7ae6d6c..7c9b2512228c 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/grpc_services/base/ya.make b/ydb/core/grpc_services/base/ya.make index d457dbdf97e1..9704d5dc3046 100644 --- a/ydb/core/grpc_services/base/ya.make +++ b/ydb/core/grpc_services/base/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/core/grpc_streaming ydb/core/jaeger_tracing ydb/public/api/protos - ydb/public/sdk/cpp/client/resources + ydb/public/sdk/cpp/src/client/resources yql/essentials/public/issue ) diff --git a/ydb/core/grpc_services/operation_helpers.cpp b/ydb/core/grpc_services/operation_helpers.cpp index 295cf6a829f6..63ec40aa4838 100644 --- a/ydb/core/grpc_services/operation_helpers.cpp +++ b/ydb/core/grpc_services/operation_helpers.cpp @@ -15,7 +15,7 @@ #include -#include +#include #include diff --git a/ydb/core/grpc_services/operation_helpers.h b/ydb/core/grpc_services/operation_helpers.h index a0fcaf40c5b6..0d0b84246fe1 100644 --- a/ydb/core/grpc_services/operation_helpers.h +++ b/ydb/core/grpc_services/operation_helpers.h @@ -1,7 +1,8 @@ #pragma once #include "defs.h" -#include +#include +#include namespace NKikimrIndexBuilder { class TIndexBuild; diff --git a/ydb/core/grpc_services/query/rpc_execute_script.cpp b/ydb/core/grpc_services/query/rpc_execute_script.cpp index 5f4159013715..6657d0f25aa1 100644 --- a/ydb/core/grpc_services/query/rpc_execute_script.cpp +++ b/ydb/core/grpc_services/query/rpc_execute_script.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/grpc_services/rpc_bsconfig_base.h b/ydb/core/grpc_services/rpc_bsconfig_base.h index 3dad9fd06f2d..9b5736757376 100644 --- a/ydb/core/grpc_services/rpc_bsconfig_base.h +++ b/ydb/core/grpc_services/rpc_bsconfig_base.h @@ -11,8 +11,8 @@ #include #include #include -#include -#include +#include +#include namespace NKikimr::NGRpcService { diff --git a/ydb/core/grpc_services/rpc_calls.h b/ydb/core/grpc_services/rpc_calls.h index c1180d127c73..622847a89e1b 100644 --- a/ydb/core/grpc_services/rpc_calls.h +++ b/ydb/core/grpc_services/rpc_calls.h @@ -19,7 +19,7 @@ #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_cancel_operation.cpp b/ydb/core/grpc_services/rpc_cancel_operation.cpp index b2db18932daf..dd280f189505 100644 --- a/ydb/core/grpc_services/rpc_cancel_operation.cpp +++ b/ydb/core/grpc_services/rpc_cancel_operation.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include @@ -54,7 +54,7 @@ class TCancelOperationRPC: public TRpcOperationRequestActor #include #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_deferrable.h b/ydb/core/grpc_services/rpc_deferrable.h index 540fa82394c5..9accfbdcc1a0 100644 --- a/ydb/core/grpc_services/rpc_deferrable.h +++ b/ydb/core/grpc_services/rpc_deferrable.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp index 6152ae5f14cb..f134adf39964 100644 --- a/ydb/core/grpc_services/rpc_execute_data_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_export_base.h b/ydb/core/grpc_services/rpc_export_base.h index d22086d4b0d7..0c15e2fc7a99 100644 --- a/ydb/core/grpc_services/rpc_export_base.h +++ b/ydb/core/grpc_services/rpc_export_base.h @@ -4,7 +4,8 @@ #include #include -#include +#include +#include #include diff --git a/ydb/core/grpc_services/rpc_forget_operation.cpp b/ydb/core/grpc_services/rpc_forget_operation.cpp index 2dca82f6a064..afa3631e01d1 100644 --- a/ydb/core/grpc_services/rpc_forget_operation.cpp +++ b/ydb/core/grpc_services/rpc_forget_operation.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include @@ -55,7 +55,7 @@ class TForgetOperationRPC: public TRpcOperationRequestActor #include #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_import_base.h b/ydb/core/grpc_services/rpc_import_base.h index e18253907346..0225e78ea026 100644 --- a/ydb/core/grpc_services/rpc_import_base.h +++ b/ydb/core/grpc_services/rpc_import_base.h @@ -4,7 +4,8 @@ #include #include -#include +#include +#include #include diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h index 698576763a7f..cd90e21a5b2b 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.h +++ b/ydb/core/grpc_services/rpc_kqp_base.h @@ -7,8 +7,8 @@ #include #include #include -#include -#include +#include +#include namespace NKikimr { namespace NGRpcService { diff --git a/ydb/core/grpc_services/rpc_list_operations.cpp b/ydb/core/grpc_services/rpc_list_operations.cpp index d2d6228e8853..50b953357c1a 100644 --- a/ydb/core/grpc_services/rpc_list_operations.cpp +++ b/ydb/core/grpc_services/rpc_list_operations.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/grpc_services/rpc_object_storage.cpp b/ydb/core/grpc_services/rpc_object_storage.cpp index 9fc10b3b82ac..29ad4d9d955c 100644 --- a/ydb/core/grpc_services/rpc_object_storage.cpp +++ b/ydb/core/grpc_services/rpc_object_storage.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include namespace NKikimr { namespace NGRpcService { diff --git a/ydb/core/grpc_services/rpc_prepare_data_query.cpp b/ydb/core/grpc_services/rpc_prepare_data_query.cpp index f49914e979bb..b706f8cd96e6 100644 --- a/ydb/core/grpc_services/rpc_prepare_data_query.cpp +++ b/ydb/core/grpc_services/rpc_prepare_data_query.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/grpc_services/rpc_read_rows.cpp b/ydb/core/grpc_services/rpc_read_rows.cpp index 4bfd31006552..83344b11b7a3 100644 --- a/ydb/core/grpc_services/rpc_read_rows.cpp +++ b/ydb/core/grpc_services/rpc_read_rows.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/grpc_services/rpc_whoami.cpp b/ydb/core/grpc_services/rpc_whoami.cpp index d0096a50e444..390ad5fdee58 100644 --- a/ydb/core/grpc_services/rpc_whoami.cpp +++ b/ydb/core/grpc_services/rpc_whoami.cpp @@ -70,7 +70,7 @@ class TWhoAmIRPC : public TActorBootstrapped { ReplyError("Empty token in Staff response"); } } else { - ReplyError(result.Error.Message); + ReplyError(TString{result.Error.Message}); } PassAway(); } diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 28240413250e..bd7d0a737703 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( audit_log.cpp audit_dml_operations.cpp @@ -145,8 +149,8 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/api/protos ydb/public/lib/fq - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/resources + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/resources ydb/services/ext_index/common ) diff --git a/ydb/core/grpc_streaming/ut/ya.make b/ydb/core/grpc_streaming/ut/ya.make index aea670910d38..d3f493adf1d6 100644 --- a/ydb/core/grpc_streaming/ut/ya.make +++ b/ydb/core/grpc_streaming/ut/ya.make @@ -9,7 +9,7 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/grpc_streaming/ut/grpc ydb/core/testlib/default ) diff --git a/ydb/core/health_check/health_check.cpp b/ydb/core/health_check/health_check.cpp index 54c7f3c64f02..4e398481d88f 100644 --- a/ydb/core/health_check/health_check.cpp +++ b/ydb/core/health_check/health_check.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include @@ -3124,7 +3124,7 @@ class TNodeCheckRequest : public TActorBootstrapped #include -#include +#include #include diff --git a/ydb/core/http_proxy/events.h b/ydb/core/http_proxy/events.h index 86d54b8a78cb..685abd98f1f1 100644 --- a/ydb/core/http_proxy/events.h +++ b/ydb/core/http_proxy/events.h @@ -6,9 +6,9 @@ #include -#include +#include -#include +#include diff --git a/ydb/core/http_proxy/grpc_service.cpp b/ydb/core/http_proxy/grpc_service.cpp index b310b55f152e..591a78ed43c3 100644 --- a/ydb/core/http_proxy/grpc_service.cpp +++ b/ydb/core/http_proxy/grpc_service.cpp @@ -93,8 +93,8 @@ class TGRpcRequestActor : public NActors::TActorBootstrapped } else if (ev->Get()->Status) { LOG_SP_INFO_S(ctx, NKikimrServices::GRPC_PROXY, "Replying " << ev->Get()->Status->GRpcStatusCode << " error " << ev->Get()->Status->Msg); if (ev->Get()->Status->GRpcStatusCode == grpc::StatusCode::NOT_FOUND) - return ReplyWithError(ctx, NYdb::EStatus::NOT_FOUND, ev->Get()->Status->Msg); - return ReplyWithError(ctx, NYdb::EStatus::INTERNAL_ERROR, ev->Get()->Status->Msg); + return ReplyWithError(ctx, NYdb::EStatus::NOT_FOUND, TString{ev->Get()->Status->Msg}); + return ReplyWithError(ctx, NYdb::EStatus::INTERNAL_ERROR, TString{ev->Get()->Status->Msg}); } else { LOG_SP_INFO_S(ctx, NKikimrServices::GRPC_PROXY, "Replying INTERNAL ERROR"); return ReplyWithError(ctx, NYdb::EStatus::INTERNAL_ERROR, "Error happened while discovering database endpoint"); @@ -112,7 +112,7 @@ class TGRpcRequestActor : public NActors::TActorBootstrapped auto deferred = resp->mutable_operation(); deferred->set_ready(true); deferred->set_status(Ydb::StatusIds::StatusCode(status)); - deferred->add_issues()->set_message(errorText); + deferred->add_issues()->set_message(TString{errorText}); ReqCtx->Reply(resp, Ydb::StatusIds::StatusCode(status)); TBase::Die(ctx); } diff --git a/ydb/core/http_proxy/http_req.cpp b/ydb/core/http_proxy/http_req.cpp index fac2e82412e7..e9378e3ecdd2 100644 --- a/ydb/core/http_proxy/http_req.cpp +++ b/ydb/core/http_proxy/http_req.cpp @@ -37,8 +37,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -66,6 +66,8 @@ #include +#include + #include #include @@ -306,7 +308,7 @@ namespace NKikimr::NHttpProxy { NYql::IssuesFromMessage(response.operation().issues(), issues); result->Status = MakeHolder( NYdb::EStatus(response.operation().status()), - std::move(issues) + NYdb::NAdapters::ToSdkIssues(std::move(issues)) ); Ydb::Ymq::V1::QueueTags queueTags; response.operation().metadata().UnpackTo(&queueTags); @@ -428,7 +430,7 @@ namespace NKikimr::NHttpProxy { ctx, get<1>(errorAndCode), get<0>(errorAndCode), - issues.begin()->GetMessage() + TString{issues.begin()->GetMessage()} ); } } @@ -690,7 +692,7 @@ namespace NKikimr::NHttpProxy { NYql::TIssues issues; NYql::IssuesFromMessage(response.operation().issues(), issues); result->Status = MakeHolder(NYdb::EStatus(response.operation().status()), - std::move(issues)); + NYdb::NAdapters::ToSdkIssues(std::move(issues))); actorSystem->Send(actorId, result.Release()); }); return; @@ -1490,7 +1492,7 @@ namespace NKikimr::NHttpProxy { void HandleTicketParser(const TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) { if (ev->Get()->Error) { - return ReplyWithError(ctx, ev->Get()->Error.Retryable ? NYdb::EStatus::UNAVAILABLE : NYdb::EStatus::UNAUTHORIZED, ev->Get()->Error.Message); + return ReplyWithError(ctx, ev->Get()->Error.Retryable ? NYdb::EStatus::UNAVAILABLE : NYdb::EStatus::UNAUTHORIZED, TString{ev->Get()->Error.Message}); } ctx.Send(Sender, new TEvServerlessProxy::TEvToken(ev->Get()->Token->GetUserSID(), "", ev->Get()->SerializedToken, {"", DatabaseId, DatabasePath, CloudId, FolderId})); diff --git a/ydb/core/http_proxy/http_req.h b/ydb/core/http_proxy/http_req.h index 2248d911593a..9509eb080318 100644 --- a/ydb/core/http_proxy/http_req.h +++ b/ydb/core/http_proxy/http_req.h @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/http_proxy/http_service.h b/ydb/core/http_proxy/http_service.h index 5eb9eb063b8d..5b5d1601cdcb 100644 --- a/ydb/core/http_proxy/http_service.h +++ b/ydb/core/http_proxy/http_service.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/ydb/core/http_proxy/json_proto_conversion.h b/ydb/core/http_proxy/json_proto_conversion.h index 61e3ff14c20f..1a3e3a96f0d9 100644 --- a/ydb/core/http_proxy/json_proto_conversion.h +++ b/ydb/core/http_proxy/json_proto_conversion.h @@ -7,11 +7,12 @@ #include #include #include -#include +#include #include #include #include +#include #include diff --git a/ydb/core/http_proxy/ut/datastreams_fixture.h b/ydb/core/http_proxy/ut/datastreams_fixture.h index 7ce923d4e454..8b35136bb07f 100644 --- a/ydb/core/http_proxy/ut/datastreams_fixture.h +++ b/ydb/core/http_proxy/ut/datastreams_fixture.h @@ -24,10 +24,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -461,7 +461,7 @@ class THttpProxyTestMock : public NUnitTest::TBaseFixture { TString endpoint = TStringBuilder() << "localhost:" << KikimrGrpcPort; auto driverConfig = NYdb::TDriverConfig() .SetEndpoint(endpoint) - .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(driverConfig); auto tableClient = NYdb::NTable::TTableClient(driver); diff --git a/ydb/core/http_proxy/ut/inside_ydb_ut/ya.make b/ydb/core/http_proxy/ut/inside_ydb_ut/ya.make index 09771e4c90d0..b935196f3853 100644 --- a/ydb/core/http_proxy/ut/inside_ydb_ut/ya.make +++ b/ydb/core/http_proxy/ut/inside_ydb_ut/ya.make @@ -14,8 +14,8 @@ PEERDIR( ydb/core/testlib/default ydb/library/aclib ydb/library/persqueue/tests - ydb/public/sdk/cpp/client/ydb_discovery - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/src/client/discovery + ydb/public/sdk/cpp/src/client/types ydb/services/ydb ) diff --git a/ydb/core/http_proxy/ut/ya.make b/ydb/core/http_proxy/ut/ya.make index 8b17e711155b..a689f09e0e9b 100644 --- a/ydb/core/http_proxy/ut/ya.make +++ b/ydb/core/http_proxy/ut/ya.make @@ -19,8 +19,8 @@ PEERDIR( ydb/library/testlib/service_mocks yql/essentials/public/udf/service/exception_policy yql/essentials/sql/pg_dummy - ydb/public/sdk/cpp/client/ydb_discovery - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/src/client/discovery + ydb/public/sdk/cpp/src/client/types ydb/services/datastreams ydb/services/kesus ydb/services/persqueue_cluster_discovery diff --git a/ydb/core/http_proxy/ya.make b/ydb/core/http_proxy/ya.make index 55ed496c64e0..744421cbd121 100644 --- a/ydb/core/http_proxy/ya.make +++ b/ydb/core/http_proxy/ya.make @@ -1,5 +1,8 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) SRCS( auth_factory.cpp @@ -22,6 +25,7 @@ SRCS( ) PEERDIR( + contrib/libs/grpc contrib/restricted/nlohmann_json ydb/library/actors/http ydb/library/actors/core @@ -36,10 +40,11 @@ PEERDIR( ydb/library/ycloud/api ydb/library/ycloud/impl ydb/library/naming_conventions - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_persqueue_core - ydb/public/sdk/cpp/client/ydb_topic/codecs - ydb/public/sdk/cpp/client/iam_private + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/iam_private ydb/services/datastreams ydb/services/persqueue_v1/actors ydb/services/ymq diff --git a/ydb/core/kafka_proxy/actors/control_plane_common.h b/ydb/core/kafka_proxy/actors/control_plane_common.h index b5539d528b60..fcc534ecc52c 100644 --- a/ydb/core/kafka_proxy/actors/control_plane_common.h +++ b/ydb/core/kafka_proxy/actors/control_plane_common.h @@ -142,8 +142,8 @@ class TAlterTopicActor : public NKikimr::NGRpcProxy::V1::TUpdateSchemeActorSendResult(status, message); + [this](const EKafkaErrors status, const std::string& message) { + this->SendResult(status,TString{message}); }) ) , TopicPath(topicPath) @@ -184,7 +184,7 @@ class TKafkaTopicModificationRequest : public NKikimr::NGRpcService::IRequestOpC TIntrusiveConstPtr userToken, TString topicPath, TString databaseName, - const std::function sendResultCallback) + const std::function sendResultCallback) : UserToken(userToken) , TopicPath(topicPath) , DatabaseName(databaseName) @@ -371,7 +371,7 @@ class TKafkaTopicModificationRequest : public NKikimr::NGRpcService::IRequestOpC const NKikimr::NGRpcService::TAuditLogParts DummyAuditLogParts; const TString TopicPath; const TString DatabaseName; - const std::function SendResultCallback; + const std::function SendResultCallback; NYql::TIssue Issue; void ProcessYdbStatusCode(Ydb::StatusIds::StatusCode& status) { diff --git a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp index 55600858669f..4bdca062a829 100644 --- a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp @@ -17,7 +17,7 @@ class TKafkaAlterConfigsRequest: public TKafkaTopicModificationRequest { TIntrusiveConstPtr userToken, TString topicPath, TString databaseName, - const std::function sendResultCallback) + const std::function sendResultCallback) : TKafkaTopicModificationRequest(userToken, topicPath, databaseName, sendResultCallback) { }; diff --git a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp index 2c6d5632325f..4150e3ecd27a 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp @@ -19,7 +19,7 @@ class TKafkaCreatePartitionsRequest : public NKikimr::NGRpcService::IRequestOpCt TIntrusiveConstPtr userToken, TString topicPath, TString databaseName, - const std::function sendResultCallback) + const std::function sendResultCallback) : UserToken(userToken) , TopicPath(topicPath) , DatabaseName(databaseName) @@ -199,7 +199,7 @@ class TKafkaCreatePartitionsRequest : public NKikimr::NGRpcService::IRequestOpCt const NKikimr::NGRpcService::TAuditLogParts DummyAuditLogParts; const TString TopicPath; const TString DatabaseName; - const std::function SendResultCallback; + const std::function SendResultCallback; NYql::TIssue Issue; void ProcessYdbStatusCode(Ydb::StatusIds::StatusCode& status) { diff --git a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp index 44af7beb5ff7..606332f51314 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp @@ -27,8 +27,8 @@ class TCreateTopicActor : public NKikimr::NGRpcProxy::V1::TPQGrpcSchemaBaseSendResult(status, message); + [this](EKafkaErrors status, const std::string& message) { + this->SendResult(status, TString{message}); }) ) , Requester(requester) diff --git a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp index 7cdcc4a5c444..10bd5607e945 100644 --- a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp @@ -49,7 +49,7 @@ void TKafkaSaslAuthActor::StartPlainAuth(const NActors::TActorContext& ctx) { void TKafkaSaslAuthActor::Handle(NKikimr::TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const NActors::TActorContext& ctx) { if (ev->Get()->Error) { - SendResponseAndDie(EKafkaErrors::SASL_AUTHENTICATION_FAILED, "", ev->Get()->Error.Message, ctx); + SendResponseAndDie(EKafkaErrors::SASL_AUTHENTICATION_FAILED, "", TString{ev->Get()->Error.Message}, ctx); return; } UserToken = ev->Get()->Token; diff --git a/ydb/core/kafka_proxy/ut/metarequest_ut.cpp b/ydb/core/kafka_proxy/ut/metarequest_ut.cpp index 2c41f8474d46..f9f2808a0014 100644 --- a/ydb/core/kafka_proxy/ut/metarequest_ut.cpp +++ b/ydb/core/kafka_proxy/ut/metarequest_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/ydb/core/kafka_proxy/ut/port_discovery_ut.cpp b/ydb/core/kafka_proxy/ut/port_discovery_ut.cpp index bd8fbfe0aebc..643b32d8dfca 100644 --- a/ydb/core/kafka_proxy/ut/port_discovery_ut.cpp +++ b/ydb/core/kafka_proxy/ut/port_discovery_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/ydb/core/kafka_proxy/ut/ut_protocol.cpp b/ydb/core/kafka_proxy/ut/ut_protocol.cpp index 9d9f12c78b80..cd6639a4f450 100644 --- a/ydb/core/kafka_proxy/ut/ut_protocol.cpp +++ b/ydb/core/kafka_proxy/ut/ut_protocol.cpp @@ -9,12 +9,12 @@ #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -154,7 +154,9 @@ class TTestServer { ui16 grpc = KikimrServer->GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driverConfig = TDriverConfig().SetEndpoint(location).SetLog(CreateLogBackend("cerr", TLOG_DEBUG)); + auto driverConfig = TDriverConfig() + .SetEndpoint(location) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", TLOG_DEBUG).Release())); if (secure) { driverConfig.UseSecureConnection(TString(NYdbSslTestData::CaCrt)); driverConfig.SetAuthToken("root@builtin"); @@ -1806,7 +1808,7 @@ Y_UNIT_TEST_SUITE(KafkaProtocol) { auto result993 = pqClient.DescribeTopic("/Root/topic-993-test", describeTopicSettings).GetValueSync(); UNIT_ASSERT(result993.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result993.GetTopicDescription().GetRetentionPeriod().MilliSeconds(), retentionMs); - UNIT_ASSERT_VALUES_EQUAL(result993.GetTopicDescription().GetRetentionStorageMb(), retentionBytes / 1_MB); + UNIT_ASSERT_VALUES_EQUAL(result993.GetTopicDescription().GetRetentionStorageMb().value(), retentionBytes / 1_MB); } { @@ -2112,12 +2114,12 @@ Y_UNIT_TEST_SUITE(KafkaProtocol) { auto result0 = pqClient.DescribeTopic(shortTopic0Name, describeTopicSettings).GetValueSync(); UNIT_ASSERT(result0.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result0.GetTopicDescription().GetRetentionPeriod().MilliSeconds(), retentionMs); - UNIT_ASSERT_VALUES_EQUAL(result0.GetTopicDescription().GetRetentionStorageMb(), retentionBytes / (1024 * 1024)); + UNIT_ASSERT_VALUES_EQUAL(result0.GetTopicDescription().GetRetentionStorageMb().value(), retentionBytes / (1024 * 1024)); auto result1 = pqClient.DescribeTopic(shortTopic0Name, describeTopicSettings).GetValueSync(); UNIT_ASSERT(result1.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result1.GetTopicDescription().GetRetentionPeriod().MilliSeconds(), retentionMs); - UNIT_ASSERT_VALUES_EQUAL(result1.GetTopicDescription().GetRetentionStorageMb(), retentionBytes / (1024 * 1024)); + UNIT_ASSERT_VALUES_EQUAL(result1.GetTopicDescription().GetRetentionStorageMb().value(), retentionBytes / (1024 * 1024)); } { @@ -2140,9 +2142,8 @@ Y_UNIT_TEST_SUITE(KafkaProtocol) { initialTopicDescription.GetRetentionPeriod().MilliSeconds(), resultingTopicDescription.GetRetentionPeriod().MilliSeconds() ); - UNIT_ASSERT_VALUES_EQUAL( - initialTopicDescription.GetRetentionStorageMb(), - resultingTopicDescription.GetRetentionStorageMb() + UNIT_ASSERT( + initialTopicDescription.GetRetentionStorageMb() == resultingTopicDescription.GetRetentionStorageMb() ); } diff --git a/ydb/core/kafka_proxy/ut/ya.make b/ydb/core/kafka_proxy/ut/ya.make index 0ad4df04ab5f..27b495412f24 100644 --- a/ydb/core/kafka_proxy/ut/ya.make +++ b/ydb/core/kafka_proxy/ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/core/kafka_proxy) +ADDINCL( + ydb/public/sdk/cpp +) + SIZE(medium) SRCS( ut_kafka_functions.cpp @@ -13,7 +17,7 @@ PEERDIR( ydb/core/kafka_proxy ydb/core/persqueue/ut/common ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/common/events/script_executions.h b/ydb/core/kqp/common/events/script_executions.h index bfc620a711c3..70ee87658fe0 100644 --- a/ydb/core/kqp/common/events/script_executions.h +++ b/ydb/core/kqp/common/events/script_executions.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/kqp/common/events/ya.make b/ydb/core/kqp/common/events/ya.make index a442cb07ea7c..568009363da9 100644 --- a/ydb/core/kqp/common/events/ya.make +++ b/ydb/core/kqp/common/events/ya.make @@ -19,7 +19,7 @@ PEERDIR( ydb/library/yql/dq/actors ydb/public/api/protos - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ydb/library/actors/core ) diff --git a/ydb/core/kqp/common/kqp_script_executions.cpp b/ydb/core/kqp/common/kqp_script_executions.cpp index 97ab89195675..8273e2ca7d78 100644 --- a/ydb/core/kqp/common/kqp_script_executions.cpp +++ b/ydb/core/kqp/common/kqp_script_executions.cpp @@ -1,6 +1,6 @@ #include "kqp_script_executions.h" -#include +#include namespace NKikimr::NKqp { @@ -17,7 +17,7 @@ TMaybe ScriptExecutionIdFromOperation(const TString& operationId) { } TMaybe ScriptExecutionIdFromOperation(const NOperationId::TOperationId& operationId) { - if (operationId.GetKind() != Ydb::TOperationId::SCRIPT_EXECUTION) { + if (operationId.GetKind() != NOperationId::TOperationId::SCRIPT_EXECUTION) { return Nothing(); } @@ -25,7 +25,7 @@ TMaybe ScriptExecutionIdFromOperation(const NOperationId::TOperationId& if (values.empty() || !values[0]) { return Nothing(); } - return *values[0]; + return TString{*values[0]}; } } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/common/kqp_script_executions.h b/ydb/core/kqp/common/kqp_script_executions.h index 3f8491a6e51b..2b397e0083bc 100644 --- a/ydb/core/kqp/common/kqp_script_executions.h +++ b/ydb/core/kqp/common/kqp_script_executions.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include diff --git a/ydb/core/kqp/common/ya.make b/ydb/core/kqp/common/ya.make index 8e724e0d72be..992571fe4a77 100644 --- a/ydb/core/kqp/common/ya.make +++ b/ydb/core/kqp/common/ya.make @@ -43,8 +43,8 @@ PEERDIR( ydb/library/yql/dq/common yql/essentials/core/dq_integration yql/essentials/parser/pg_wrapper/interface - ydb/public/lib/operation_id - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/library/operation_id/protos ydb/core/grpc_services/cancelation library/cpp/lwtrace #library/cpp/lwtrace/protos diff --git a/ydb/core/kqp/counters/ya.make b/ydb/core/kqp/counters/ya.make index 8b44b02641cf..67155627777c 100644 --- a/ydb/core/kqp/counters/ya.make +++ b/ydb/core/kqp/counters/ya.make @@ -9,6 +9,7 @@ SRCS( PEERDIR( ydb/core/base ydb/core/protos + ydb/core/tx/tx_proxy ydb/core/sys_view/service ydb/library/yql/dq/actors/spilling yql/essentials/minikql diff --git a/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp index 178c86cb2c4a..24da9943ba3d 100644 --- a/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/executer_actor/ut/ya.make b/ydb/core/kqp/executer_actor/ut/ya.make index 8fe5732d9690..0acdef260a4b 100644 --- a/ydb/core/kqp/executer_actor/ut/ya.make +++ b/ydb/core/kqp/executer_actor/ut/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/core/kqp/common ydb/core/kqp/host ydb/core/kqp/ut/common - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/proto ydb/library/yql/providers/common/http_gateway yql/essentials/sql/pg_dummy ) diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp index e3d0a39fc15c..2dce397c501f 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp +++ b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp @@ -114,12 +114,12 @@ class TRanksCheckerActor : public NKikimr::TQueryBase { if (!RanksToCheck.empty()) { NYdb::TResultSetParser result(ResultSets[resultSetId++]); while (result.TryNextRow()) { - TMaybe rank = result.ColumnParser("rank").GetOptionalInt64(); + std::optional rank = result.ColumnParser("rank").GetOptionalInt64(); if (!rank) { continue; } - TMaybe name = result.ColumnParser("name").GetOptionalUtf8(); + std::optional name = result.ColumnParser("name").GetOptionalUtf8(); if (!name) { continue; } @@ -138,7 +138,7 @@ class TRanksCheckerActor : public NKikimr::TQueryBase { return; } - MaxRank = result.ColumnParser("MaxRank").GetOptionalInt64().GetOrElse(0); + MaxRank = result.ColumnParser("MaxRank").GetOptionalInt64().value_or(0); NumberClassifiers = result.ColumnParser("NumberClassifiers").GetUint64(); } diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index 5297cf0138b9..5f24e76cfcef 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index d436c81bd7d4..c7260f314003 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -2422,8 +2422,8 @@ class TKqpGatewayProxy : public IKikimrGateway { auto& params = *config.MutableSrcConnectionParams(); if (const auto& connectionString = settings.Settings.ConnectionString) { const auto parseResult = NYdb::ParseConnectionString(*connectionString); - params.SetEndpoint(parseResult.Endpoint); - params.SetDatabase(parseResult.Database); + params.SetEndpoint(TString{parseResult.Endpoint}); + params.SetDatabase(TString{parseResult.Database}); params.SetEnableSsl(parseResult.EnableSsl); } if (const auto& endpoint = settings.Settings.Endpoint) { @@ -2505,8 +2505,8 @@ class TKqpGatewayProxy : public IKikimrGateway { auto& params = *config.MutableSrcConnectionParams(); if (const auto& connectionString = settings.Settings.ConnectionString) { const auto parseResult = NYdb::ParseConnectionString(*connectionString); - params.SetEndpoint(parseResult.Endpoint); - params.SetDatabase(parseResult.Database); + params.SetEndpoint(TString{parseResult.Endpoint}); + params.SetDatabase(TString{parseResult.Database}); } if (const auto& endpoint = settings.Settings.Endpoint) { params.SetEndpoint(*endpoint); diff --git a/ydb/core/kqp/host/ya.make b/ydb/core/kqp/host/ya.make index a8b0dfb59c88..dfff95bae554 100644 --- a/ydb/core/kqp/host/ya.make +++ b/ydb/core/kqp/host/ya.make @@ -33,7 +33,7 @@ PEERDIR( yql/essentials/providers/pg/provider yql/essentials/providers/result/provider ydb/library/yql/providers/s3/expr_nodes - ydb/public/sdk/cpp/client/impl/ydb_internal/common + ydb/public/sdk/cpp/src/client/impl/ydb_internal/common ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/provider/read_attributes_utils.cpp b/ydb/core/kqp/provider/read_attributes_utils.cpp index e226c2a19e19..adc932e8f77e 100644 --- a/ydb/core/kqp/provider/read_attributes_utils.cpp +++ b/ydb/core/kqp/provider/read_attributes_utils.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace NYql { diff --git a/ydb/core/kqp/provider/ya.make b/ydb/core/kqp/provider/ya.make index db17672c235b..da0e3b8515be 100644 --- a/ydb/core/kqp/provider/ya.make +++ b/ydb/core/kqp/provider/ya.make @@ -38,7 +38,7 @@ PEERDIR( yql/essentials/minikql yql/essentials/public/decimal ydb/public/lib/scheme_types - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic yql/essentials/core/expr_nodes yql/essentials/core/peephole_opt yql/essentials/parser/pg_wrapper/interface diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index 2725e87f18cc..fa29120dcdd6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 99aab9eedad6..537cbdcae67a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 3852b9f7b406..1ce0a8e8c526 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp index fd9df173447e..305e35e0dcca 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include @@ -100,7 +100,7 @@ std::optional TryDecodeYdbSessionId(const TString& sessionId) { TString EncodeSessionId(ui32 nodeId, const TString& id) { Ydb::TOperationId opId; - opId.SetKind(NOperationId::TOperationId::SESSION_YQL); + opId.SetKind(Ydb::TOperationId::SESSION_YQL); NOperationId::AddOptionalValue(opId, "node_id", ToString(nodeId)); NOperationId::AddOptionalValue(opId, "id", Base64Encode(id)); return NOperationId::ProtoToString(opId); diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp index 17a8f6c99264..6cca0e5d6eba 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp @@ -10,10 +10,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -572,7 +572,7 @@ Y_UNIT_TEST_SUITE(KqpProxy) { auto executeScrptsResult = client.ExecuteScript("SELECT 42").ExtractValueSync(); scriptStatus = executeScrptsResult.Status().GetStatus(); UNIT_ASSERT_C(scriptStatus == NYdb::EStatus::UNAVAILABLE || scriptStatus == NYdb::EStatus::SUCCESS, executeScrptsResult.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptStatus == NYdb::EStatus::UNAVAILABLE || executeScrptsResult.Metadata().ExecutionId); + UNIT_ASSERT(scriptStatus == NYdb::EStatus::UNAVAILABLE || !executeScrptsResult.Metadata().ExecutionId.empty()); Sleep(TDuration::MilliSeconds(10)); } while (scriptStatus == NYdb::EStatus::UNAVAILABLE); diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp index 1755c133c7e1..20592f2be0d5 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp +++ b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp @@ -14,9 +14,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include @@ -670,7 +670,7 @@ class TCheckLeaseStatusQueryActor : public TQueryBase { result.TryNextRow(); - const TMaybe runScriptActorId = result.ColumnParser("run_script_actor_id").GetOptionalUtf8(); + const std::optional runScriptActorId = result.ColumnParser("run_script_actor_id").GetOptionalUtf8(); if (!runScriptActorId) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Unexpected database response"); return; @@ -680,14 +680,14 @@ class TCheckLeaseStatusQueryActor : public TQueryBase { return; } - TMaybe operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); + std::optional operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); - const TMaybe finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); + const std::optional finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); if (finalizationStatus) { FinalizationStatus = static_cast(*finalizationStatus); } - TMaybe leaseDeadline; + std::optional leaseDeadline; NYdb::TResultSetParser result2(ResultSets[1]); @@ -706,11 +706,11 @@ class TCheckLeaseStatusQueryActor : public TQueryBase { } } else if (operationStatus) { OperationStatus = static_cast(*operationStatus); - TMaybe executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); + std::optional executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); if (executionStatus) { ExecutionStatus = static_cast(*executionStatus); } - const TMaybe issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); + const std::optional issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); if (issuesSerialized) { OperationIssues = DeserializeIssues(*issuesSerialized); } @@ -898,14 +898,14 @@ class TForgetScriptExecutionOperationQueryActor : public TQueryBase { result.TryNextRow(); - TMaybe maxResultSetId = result.ColumnParser("max_result_set_id").GetOptionalInt32(); + std::optional maxResultSetId = result.ColumnParser("max_result_set_id").GetOptionalInt32(); if (!maxResultSetId) { Finish(); return; } NumberRowsInBatch = std::max(MAX_NUMBER_ROWS_IN_BATCH / (*maxResultSetId + 1), 1); - TMaybe maxRowId = result.ColumnParser("max_row_id").GetOptionalInt64(); + std::optional maxRowId = result.ColumnParser("max_row_id").GetOptionalInt64(); if (!maxRowId) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Result set row id is not specified"); return; @@ -1121,54 +1121,54 @@ class TGetScriptExecutionOperationQueryActor : public TQueryBase { result.TryNextRow(); - const TMaybe operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); + const std::optional operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); if (operationStatus) { OperationStatus = static_cast(*operationStatus); } - const TMaybe finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); + const std::optional finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); if (finalizationStatus) { FinalizationStatus = static_cast(*finalizationStatus); } Metadata.set_execution_id(ExecutionId); - const TMaybe executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); + const std::optional executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); if (executionStatus) { Metadata.set_exec_status(static_cast(*executionStatus)); } - const TMaybe sql = result.ColumnParser("query_text").GetOptionalUtf8(); + const std::optional sql = result.ColumnParser("query_text").GetOptionalUtf8(); if (sql) { Metadata.mutable_script_content()->set_text(*sql); } - const TMaybe syntax = result.ColumnParser("syntax").GetOptionalInt32(); + const std::optional syntax = result.ColumnParser("syntax").GetOptionalInt32(); if (syntax) { Metadata.mutable_script_content()->set_syntax(static_cast(*syntax)); } - const TMaybe executionMode = result.ColumnParser("execution_mode").GetOptionalInt32(); + const std::optional executionMode = result.ColumnParser("execution_mode").GetOptionalInt32(); if (executionMode) { Metadata.set_exec_mode(static_cast(*executionMode)); } - const TMaybe serializedStats = result.ColumnParser("stats").GetOptionalJsonDocument(); + const std::optional serializedStats = result.ColumnParser("stats").GetOptionalJsonDocument(); if (serializedStats) { NJson::TJsonValue statsJson; NJson::ReadJsonTree(*serializedStats, &statsJson); NProtobufJson::Json2Proto(statsJson, *Metadata.mutable_exec_stats(), NProtobufJson::TJson2ProtoConfig()); } - const TMaybe plan = result.ColumnParser("plan").GetOptionalJsonDocument(); + const std::optional plan = result.ColumnParser("plan").GetOptionalJsonDocument(); if (plan) { Metadata.mutable_exec_stats()->set_query_plan(*plan); } - TMaybe ast; - const TMaybe astCompressionMethod = result.ColumnParser("ast_compression_method").GetOptionalUtf8(); + std::optional ast; + const std::optional astCompressionMethod = result.ColumnParser("ast_compression_method").GetOptionalUtf8(); if (astCompressionMethod) { - const TMaybe astCompressed = result.ColumnParser("ast_compressed").GetOptionalString(); + const std::optional astCompressed = result.ColumnParser("ast_compressed").GetOptionalString(); if (astCompressed) { const NFq::TCompressor compressor(*astCompressionMethod); ast = compressor.Decompress(*astCompressed); @@ -1180,12 +1180,12 @@ class TGetScriptExecutionOperationQueryActor : public TQueryBase { Metadata.mutable_exec_stats()->set_query_ast(*ast); } - const TMaybe issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); + const std::optional issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); if (issuesSerialized) { Issues = DeserializeIssues(*issuesSerialized); } - const TMaybe serializedMetas = result.ColumnParser("result_set_metas").GetOptionalJsonDocument(); + const std::optional serializedMetas = result.ColumnParser("result_set_metas").GetOptionalJsonDocument(); if (serializedMetas) { NJson::TJsonValue value; if (!NJson::ReadJsonTree(*serializedMetas, &value) || value.GetType() != NJson::JSON_ARRAY) { @@ -1200,7 +1200,7 @@ class TGetScriptExecutionOperationQueryActor : public TQueryBase { } } - const TMaybe runScriptActorIdString = result.ColumnParser("run_script_actor_id").GetOptionalUtf8(); + const std::optional runScriptActorIdString = result.ColumnParser("run_script_actor_id").GetOptionalUtf8(); if (runScriptActorIdString) { ScriptExecutionRunnerActorIdFromString(*runScriptActorIdString, RunScriptActorId); } @@ -1215,7 +1215,7 @@ class TGetScriptExecutionOperationQueryActor : public TQueryBase { deadlineResult.TryNextRow(); - TMaybe leaseDeadline = deadlineResult.ColumnParser(0).GetOptionalTimestamp(); + std::optional leaseDeadline = deadlineResult.ColumnParser(0).GetOptionalTimestamp(); if (!leaseDeadline) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Unexpected operation state"); return; @@ -1430,13 +1430,13 @@ class TListScriptExecutionOperationsQuery : public TQueryBase { Operations.reserve(result.RowsCount()); while (result.TryNextRow()) { - const TMaybe executionId = result.ColumnParser("execution_id").GetOptionalUtf8(); + const std::optional executionId = result.ColumnParser("execution_id").GetOptionalUtf8(); if (!executionId) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "NULL execution id"); return; } - const TMaybe creationTs = result.ColumnParser("start_ts").GetOptionalTimestamp(); + const std::optional creationTs = result.ColumnParser("start_ts").GetOptionalTimestamp(); if (!creationTs) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "NULL creation ts"); return; @@ -1447,32 +1447,32 @@ class TListScriptExecutionOperationsQuery : public TQueryBase { break; } - const TMaybe operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); + const std::optional operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); Ydb::Query::ExecuteScriptMetadata metadata; metadata.set_execution_id(*executionId); - const TMaybe executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); + const std::optional executionStatus = result.ColumnParser("execution_status").GetOptionalInt32(); if (executionStatus) { metadata.set_exec_status(static_cast(*executionStatus)); } - const TMaybe sql = result.ColumnParser("query_text").GetOptionalUtf8(); + const std::optional sql = result.ColumnParser("query_text").GetOptionalUtf8(); if (sql) { metadata.mutable_script_content()->set_text(*sql); } - const TMaybe syntax = result.ColumnParser("syntax").GetOptionalInt32(); + const std::optional syntax = result.ColumnParser("syntax").GetOptionalInt32(); if (syntax) { metadata.mutable_script_content()->set_syntax(static_cast(*syntax)); } - const TMaybe executionMode = result.ColumnParser("execution_mode").GetOptionalInt32(); + const std::optional executionMode = result.ColumnParser("execution_mode").GetOptionalInt32(); if (executionMode) { metadata.set_exec_mode(static_cast(*executionMode)); } - const TMaybe issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); + const std::optional issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); NYql::TIssues issues; if (issuesSerialized) { issues = DeserializeIssues(*issuesSerialized); @@ -1480,7 +1480,7 @@ class TListScriptExecutionOperationsQuery : public TQueryBase { Ydb::Operations::Operation op; op.set_id(ScriptExecutionOperationFromExecutionId(*executionId)); - op.set_ready(operationStatus.Defined()); + op.set_ready(operationStatus.has_value()); if (operationStatus) { op.set_status(static_cast(*operationStatus)); } @@ -1751,7 +1751,7 @@ class TSaveScriptExecutionResultMetaQuery : public TQueryBase { class TSaveScriptExecutionResultQuery : public TQueryBase { public: TSaveScriptExecutionResultQuery(const TString& database, const TString& executionId, i32 resultSetId, - TMaybe expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet resultSet) + std::optional expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet resultSet) : TQueryBase(__func__, executionId) , Database(database) , ExecutionId(executionId) @@ -1836,7 +1836,7 @@ class TSaveScriptExecutionResultQuery : public TQueryBase { const TString Database; const TString ExecutionId; const i32 ResultSetId; - const TMaybe ExpireAt; + const std::optional ExpireAt; const i64 FirstRow; const i64 AccumulatedSize; const Ydb::ResultSet ResultSet; @@ -1850,7 +1850,7 @@ class TSaveScriptExecutionResultActor : public TActorBootstrapped expireAt, + const TString& executionId, i32 resultSetId, std::optional expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet&& resultSet) : ReplyActorId(replyActorId) , Database(database) @@ -1870,7 +1870,7 @@ class TSaveScriptExecutionResultActor : public TActorBootstrapped, i64, i64, Ydb::ResultSet>(SelfId(), Database, ExecutionId, ResultSetId, ExpireAt, FirstRow, AccumulatedSize, ResultSets.back())); + Register(new TQueryRetryActor, i64, i64, Ydb::ResultSet>(SelfId(), Database, ExecutionId, ResultSetId, ExpireAt, FirstRow, AccumulatedSize, ResultSets.back())); FirstRow += numberRows; ResultSets.pop_back(); @@ -1918,7 +1918,7 @@ class TSaveScriptExecutionResultActor : public TActorBootstrapped ExpireAt; + const std::optional ExpireAt; i64 FirstRow; i64 AccumulatedSize; NFq::TRowsProtoSplitter RowsSplitter; @@ -1986,7 +1986,7 @@ class TGetScriptExecutionResultQueryActor : public TQueryBase { return; } - const TMaybe operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); + const std::optional operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); if (!operationStatus) { Finish(Ydb::StatusIds::BAD_REQUEST, "Results are not ready"); return; @@ -2004,7 +2004,7 @@ class TGetScriptExecutionResultQueryActor : public TQueryBase { return; } - const auto ttl = GetTtlFromSerializedMeta(*serializedMeta); + const auto ttl = GetTtlFromSerializedMeta(TString{*serializedMeta}); if (!ttl) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Metainformation is corrupted"); return; @@ -2017,7 +2017,7 @@ class TGetScriptExecutionResultQueryActor : public TQueryBase { Ydb::StatusIds::StatusCode operationStatusCode = static_cast(*operationStatus); if (operationStatusCode != Ydb::StatusIds::SUCCESS) { - const TMaybe issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); + const std::optional issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); if (issuesSerialized) { Finish(operationStatusCode, DeserializeIssues(*issuesSerialized)); } else { @@ -2026,7 +2026,7 @@ class TGetScriptExecutionResultQueryActor : public TQueryBase { return; } - const TMaybe serializedMetas = result.ColumnParser("result_set_metas").GetOptionalJsonDocument(); + const std::optional serializedMetas = result.ColumnParser("result_set_metas").GetOptionalJsonDocument(); if (!serializedMetas) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Result meta is empty"); return; @@ -2099,7 +2099,7 @@ class TGetScriptExecutionResultQueryActor : public TQueryBase { void OnStreamResult(NYdb::TResultSet&& resultSet) override { NYdb::TResultSetParser result(resultSet); while (result.TryNextRow()) { - const TMaybe serializedRow = result.ColumnParser("result_set").GetOptionalString(); + const std::optional serializedRow = result.ColumnParser("result_set").GetOptionalString(); if (!serializedRow) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Result set row is null"); return; @@ -2361,7 +2361,7 @@ class TSaveScriptFinalStatusActor : public TQueryBase { result.TryNextRow(); - TMaybe finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); + std::optional finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); if (finalizationStatus) { if (Request.FinalizationStatus != *finalizationStatus) { Finish(Ydb::StatusIds::PRECONDITION_FAILED, "Execution already have different finalization status"); @@ -2370,7 +2370,7 @@ class TSaveScriptFinalStatusActor : public TQueryBase { Response->ApplicateScriptExternalEffectRequired = true; } - TMaybe operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); + std::optional operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); if (Request.LeaseGeneration && !operationStatus) { NYdb::TResultSetParser leaseResult(ResultSets[1]); @@ -2381,7 +2381,7 @@ class TSaveScriptFinalStatusActor : public TQueryBase { leaseResult.TryNextRow(); - TMaybe leaseGenerationInDatabase = leaseResult.ColumnParser("lease_generation").GetOptionalInt64(); + std::optional leaseGenerationInDatabase = leaseResult.ColumnParser("lease_generation").GetOptionalInt64(); if (!leaseGenerationInDatabase) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Unknown lease generation"); return; @@ -2393,12 +2393,12 @@ class TSaveScriptFinalStatusActor : public TQueryBase { } } - TMaybe customerSuppliedId = result.ColumnParser("customer_supplied_id").GetOptionalUtf8(); + std::optional customerSuppliedId = result.ColumnParser("customer_supplied_id").GetOptionalUtf8(); if (customerSuppliedId) { Response->CustomerSuppliedId = *customerSuppliedId; } - TMaybe userToken = result.ColumnParser("user_token").GetOptionalUtf8(); + std::optional userToken = result.ColumnParser("user_token").GetOptionalUtf8(); if (userToken) { Response->UserToken = *userToken; } @@ -2443,7 +2443,7 @@ class TSaveScriptFinalStatusActor : public TQueryBase { return; } - const auto ttl = GetTtlFromSerializedMeta(*serializedMeta); + const auto ttl = GetTtlFromSerializedMeta(TString{*serializedMeta}); if (!ttl) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Metainformation is corrupted"); return; @@ -2514,9 +2514,9 @@ class TSaveScriptFinalStatusActor : public TQueryBase { serializedStats = NJson::WriteJson(statsJson); } - TMaybe ast; - TMaybe astCompressed; - TMaybe astCompressionMethod; + std::optional ast; + std::optional astCompressed; + std::optional astCompressionMethod; if (Request.QueryAst && Request.QueryAstCompressionMethod) { astCompressed = *Request.QueryAst; astCompressionMethod = *Request.QueryAstCompressionMethod; @@ -2611,8 +2611,8 @@ class TSaveScriptFinalStatusActor : public TQueryBase { bool FinalStatusAlreadySaved = false; TDuration OperationTtl; - TMaybe SerializedSinks; - TMaybe SerializedSecretNames; + std::optional SerializedSinks; + std::optional SerializedSecretNames; }; class TScriptFinalizationFinisherActor : public TQueryBase { @@ -2664,7 +2664,7 @@ class TScriptFinalizationFinisherActor : public TQueryBase { result.TryNextRow(); - TMaybe finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); + std::optional finalizationStatus = result.ColumnParser("finalization_status").GetOptionalInt32(); if (!finalizationStatus) { Finish(Ydb::StatusIds::PRECONDITION_FAILED, "Already finished"); return; @@ -2851,7 +2851,7 @@ NActors::IActor* CreateSaveScriptExecutionResultMetaActor(const NActors::TActorI return new TQueryRetryActor(runScriptActorId, database, executionId, serializedMeta); } -NActors::IActor* CreateSaveScriptExecutionResultActor(const NActors::TActorId& runScriptActorId, const TString& database, const TString& executionId, i32 resultSetId, TMaybe expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet&& resultSet) { +NActors::IActor* CreateSaveScriptExecutionResultActor(const NActors::TActorId& runScriptActorId, const TString& database, const TString& executionId, i32 resultSetId, std::optional expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet&& resultSet) { return new TSaveScriptExecutionResultActor(runScriptActorId, database, executionId, resultSetId, expireAt, firstRow, accumulatedSize, std::move(resultSet)); } diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions.h b/ydb/core/kqp/proxy_service/kqp_script_executions.h index d8dd61c40576..fc0a6c5da8ea 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions.h +++ b/ydb/core/kqp/proxy_service/kqp_script_executions.h @@ -28,7 +28,7 @@ NActors::IActor* CreateScriptLeaseUpdateActor(const TActorId& runScriptActorId, // Store and fetch results. NActors::IActor* CreateSaveScriptExecutionResultMetaActor(const NActors::TActorId& runScriptActorId, const TString& database, const TString& executionId, const TString& serializedMeta); -NActors::IActor* CreateSaveScriptExecutionResultActor(const NActors::TActorId& runScriptActorId, const TString& database, const TString& executionId, i32 resultSetId, TMaybe expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet&& resultSet); +NActors::IActor* CreateSaveScriptExecutionResultActor(const NActors::TActorId& runScriptActorId, const TString& database, const TString& executionId, i32 resultSetId, std::optional expireAt, i64 firstRow, i64 accumulatedSize, Ydb::ResultSet&& resultSet); NActors::IActor* CreateGetScriptExecutionResultActor(const NActors::TActorId& replyActorId, const TString& database, const TString& executionId, i32 resultSetIndex, i64 offset, i64 rowsLimit, i64 sizeLimit, TInstant operationDeadline); // Compute external effects and updates status in database diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp b/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp index 113b4cfee997..2b8ba4e746f0 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp +++ b/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp @@ -5,8 +5,8 @@ #include #include #include -#include -#include +#include +#include #include @@ -200,7 +200,7 @@ struct TScriptExecutionsYdbSetup { return reply; } - void CheckLeaseExistance(const TString& executionId, bool expectedExistance, TMaybe expectedStatus) { + void CheckLeaseExistance(const TString& executionId, bool expectedExistance, std::optional expectedStatus) { TStringBuilder sql; sql << R"( @@ -237,7 +237,7 @@ struct TScriptExecutionsYdbSetup { NYdb::TResultSetParser rs2 = result.GetResultSetParser(1); UNIT_ASSERT(rs2.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(rs2.ColumnParser("operation_status").GetOptionalInt32(), expectedStatus); + UNIT_ASSERT(rs2.ColumnParser("operation_status").GetOptionalInt32() == expectedStatus); } THolder UpdateLease(const TString& executionId, TDuration leaseDuration) { @@ -269,12 +269,12 @@ Y_UNIT_TEST_SUITE(ScriptExecutionsTest) { const TString executionId = ydb.CreateQueryInDb(); UNIT_ASSERT(executionId); const TInstant startLeaseTime = TInstant::Now(); - ydb.CheckLeaseExistance(executionId, true, Nothing()); + ydb.CheckLeaseExistance(executionId, true, std::nullopt); auto checkResult1 = ydb.CheckLeaseStatus(executionId); const TDuration checkTime = TInstant::Now() - startLeaseTime; if (checkTime < TestLeaseDuration) { UNIT_ASSERT_VALUES_EQUAL(checkResult1->Get()->OperationStatus, Nothing()); - ydb.CheckLeaseExistance(executionId, true, Nothing()); + ydb.CheckLeaseExistance(executionId, true, std::nullopt); SleepUntil(startLeaseTime + TestLeaseDuration); } @@ -291,7 +291,7 @@ Y_UNIT_TEST_SUITE(ScriptExecutionsTest) { TInstant startLeaseTime = TInstant::Now(); - ydb.CheckLeaseExistance(executionId, true, Nothing()); + ydb.CheckLeaseExistance(executionId, true, std::nullopt); SleepUntil(startLeaseTime + TestLeaseDuration); startLeaseTime = TInstant::Now(); @@ -300,7 +300,7 @@ Y_UNIT_TEST_SUITE(ScriptExecutionsTest) { UNIT_ASSERT_C(updateResponse->Status == Ydb::StatusIds::SUCCESS, updateResponse->Issues.ToString()); UNIT_ASSERT(updateResponse->ExecutionEntryExists); - ydb.CheckLeaseExistance(executionId, true, Nothing()); + ydb.CheckLeaseExistance(executionId, true, std::nullopt); auto checkResult = ydb.CheckLeaseStatus(executionId); if (TInstant::Now() - startLeaseTime < leaseDuration) { @@ -313,7 +313,7 @@ Y_UNIT_TEST_SUITE(ScriptExecutionsTest) { const TString executionId = ydb.CreateQueryInDb(); UNIT_ASSERT(executionId); - ydb.CheckLeaseExistance(executionId, true, Nothing()); + ydb.CheckLeaseExistance(executionId, true, std::nullopt); Sleep(TestLeaseDuration); diff --git a/ydb/core/kqp/proxy_service/ut/ya.make b/ydb/core/kqp/proxy_service/ut/ya.make index ae98f67640d8..397f19e2d038 100644 --- a/ydb/core/kqp/proxy_service/ut/ya.make +++ b/ydb/core/kqp/proxy_service/ut/ya.make @@ -15,8 +15,8 @@ PEERDIR( ydb/core/kqp/ut/common ydb/core/kqp/workload_service/ut/common yql/essentials/sql/pg_dummy - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/driver ydb/services/ydb ) diff --git a/ydb/core/kqp/proxy_service/ya.make b/ydb/core/kqp/proxy_service/ya.make index f57778956993..54aece76b5e5 100644 --- a/ydb/core/kqp/proxy_service/ya.make +++ b/ydb/core/kqp/proxy_service/ya.make @@ -36,9 +36,9 @@ PEERDIR( yql/essentials/public/issue ydb/library/yql/dq/actors/spilling ydb/public/api/protos - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ydb/public/lib/scheme_types - ydb/public/sdk/cpp/client/ydb_params + ydb/public/sdk/cpp/src/client/params ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/query_data/kqp_query_data.cpp b/ydb/core/kqp/query_data/kqp_query_data.cpp index 5aee836444f4..08ed96d0d944 100644 --- a/ydb/core/kqp/query_data/kqp_query_data.cpp +++ b/ydb/core/kqp/query_data/kqp_query_data.cpp @@ -7,10 +7,10 @@ #include #include #include -#include +#include #include -#include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/query_data/ya.make b/ydb/core/kqp/query_data/ya.make index cdce57eb57d4..5ea68ac02114 100644 --- a/ydb/core/kqp/query_data/ya.make +++ b/ydb/core/kqp/query_data/ya.make @@ -8,7 +8,7 @@ SRCS( PEERDIR( ydb/public/api/grpc/draft - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id/protos ydb/library/actors/core ydb/core/actorlib_impl ydb/core/base diff --git a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp index a10a25720191..0b5543a63d7d 100644 --- a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp @@ -686,7 +686,7 @@ class TRunScriptActor : public NActors::TActorBootstrapped { // Result std::vector ResultSetInfos; TMap StreamChannels; - TMaybe ExpireAt; + std::optional ExpireAt; NJson::TJsonValue ResultSetMetas; ui32 SaveResultInflight = 0; ui64 SaveResultInflightBytes = 0; diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index acf9dc27b462..991684611022 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include @@ -72,7 +72,7 @@ std::optional TryDecodeYdbSessionId(const TString& sessionId) { return std::nullopt; } - return *ids[0]; + return TString{*ids[0]}; } catch (...) { return std::nullopt; } diff --git a/ydb/core/kqp/session_actor/ya.make b/ydb/core/kqp/session_actor/ya.make index b09fb343deff..5332615f3841 100644 --- a/ydb/core/kqp/session_actor/ya.make +++ b/ydb/core/kqp/session_actor/ya.make @@ -14,7 +14,7 @@ PEERDIR( ydb/core/docapi ydb/core/kqp/common ydb/core/kqp/federated_query - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ydb/core/tx/schemeshard ) diff --git a/ydb/core/kqp/tests/kikimr_tpch/kqp_tpch_ut.cpp b/ydb/core/kqp/tests/kikimr_tpch/kqp_tpch_ut.cpp index 7de1b6463619..16ad1c33ace9 100644 --- a/ydb/core/kqp/tests/kikimr_tpch/kqp_tpch_ut.cpp +++ b/ydb/core/kqp/tests/kikimr_tpch/kqp_tpch_ut.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/tests/kikimr_tpch/ya.make b/ydb/core/kqp/tests/kikimr_tpch/ya.make index 8a57e8dce002..480a9f038151 100644 --- a/ydb/core/kqp/tests/kikimr_tpch/ya.make +++ b/ydb/core/kqp/tests/kikimr_tpch/ya.make @@ -22,7 +22,7 @@ PEERDIR( ydb/core/kqp/tests/tpch/lib library/cpp/testing/unittest ydb/core/protos - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client yql/essentials/public/udf/service/stub ydb/public/lib/yson_value ) diff --git a/ydb/core/kqp/tests/tpch/cmd_run_bench.cpp b/ydb/core/kqp/tests/tpch/cmd_run_bench.cpp index 0109f4ea8966..ddde4d93970d 100644 --- a/ydb/core/kqp/tests/tpch/cmd_run_bench.cpp +++ b/ydb/core/kqp/tests/tpch/cmd_run_bench.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/tests/tpch/lib/tpch_runner.cpp b/ydb/core/kqp/tests/tpch/lib/tpch_runner.cpp index 9c84da3bd2df..d98fabf3e888 100644 --- a/ydb/core/kqp/tests/tpch/lib/tpch_runner.cpp +++ b/ydb/core/kqp/tests/tpch/lib/tpch_runner.cpp @@ -2,7 +2,7 @@ #include "tpch_tables.h" #include -#include +#include #include @@ -29,7 +29,7 @@ void ThrowOnError(const TStatus& status) { } template -void BuildRow(const TString& line, const TVector& columns, TValueBuilderBase& row) { +void BuildRow(const std::string& line, const std::vector& columns, TValueBuilderBase& row) { TVector data = StringSplitter(line).Split('|').SkipEmpty(); Y_ENSURE(data.size() == columns.size()); diff --git a/ydb/core/kqp/tests/tpch/lib/tpch_runner.h b/ydb/core/kqp/tests/tpch/lib/tpch_runner.h index 5fd5e8dcc1c6..7c2a06f6f39d 100644 --- a/ydb/core/kqp/tests/tpch/lib/tpch_runner.h +++ b/ydb/core/kqp/tests/tpch/lib/tpch_runner.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include diff --git a/ydb/core/kqp/tests/tpch/lib/tpch_tables.h b/ydb/core/kqp/tests/tpch/lib/tpch_tables.h index 3989b357c25c..8806e8f8ba8e 100644 --- a/ydb/core/kqp/tests/tpch/lib/tpch_tables.h +++ b/ydb/core/kqp/tests/tpch/lib/tpch_tables.h @@ -1,7 +1,8 @@ #pragma once -#include +#include +#include namespace NYdb::NTpch { extern const THashMap TABLES; diff --git a/ydb/core/kqp/tests/tpch/lib/ya.make b/ydb/core/kqp/tests/tpch/lib/ya.make index c84e8b6b10ed..1b69bc91f79d 100644 --- a/ydb/core/kqp/tests/tpch/lib/ya.make +++ b/ydb/core/kqp/tests/tpch/lib/ya.make @@ -13,8 +13,8 @@ SRCS( PEERDIR( library/cpp/resource ydb/core/protos - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ) RESOURCE( diff --git a/ydb/core/kqp/topics/ya.make b/ydb/core/kqp/topics/ya.make index fc785296cd18..89f17253a819 100644 --- a/ydb/core/kqp/topics/ya.make +++ b/ydb/core/kqp/topics/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( ydb/core/base + ydb/core/tx/scheme_cache ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/arrow/kqp_arrow_in_channels_ut.cpp b/ydb/core/kqp/ut/arrow/kqp_arrow_in_channels_ut.cpp index 46fc16dda6d2..876b4da25d54 100644 --- a/ydb/core/kqp/ut/arrow/kqp_arrow_in_channels_ut.cpp +++ b/ydb/core/kqp/ut/arrow/kqp_arrow_in_channels_ut.cpp @@ -63,26 +63,26 @@ void InsertAllColumnsAndCheckSelectAll(TKikimrRunner* runner) { UNIT_ASSERT_C(columns.size() == 19, "Wrong columns count"); NYdb::TResultSetParser parser(resultSet); UNIT_ASSERT_C(parser.TryNextRow(), "Row is missing"); - UNIT_ASSERT(*parser.ColumnParser(0).GetOptionalUint64().Get() == 42); - UNIT_ASSERT(*parser.ColumnParser(1).GetOptionalBool().Get() == true); - UNIT_ASSERT(*parser.ColumnParser(2).GetOptionalInt32().Get() == -1); - UNIT_ASSERT(*parser.ColumnParser(3).GetOptionalUint32().Get() == 1); - UNIT_ASSERT(*parser.ColumnParser(4).GetOptionalInt64().Get() == -2); - UNIT_ASSERT(*parser.ColumnParser(5).GetOptionalUint64().Get() == 2); - UNIT_ASSERT(*parser.ColumnParser(6).GetOptionalFloat().Get() == 3.0); - UNIT_ASSERT(*parser.ColumnParser(7).GetOptionalDouble().Get() == 4.0); - UNIT_ASSERT(*parser.ColumnParser(8).GetOptionalString().Get() == TString("five")); - UNIT_ASSERT(*parser.ColumnParser(9).GetOptionalUtf8().Get() == TString("six")); - UNIT_ASSERT(*parser.ColumnParser(10).GetOptionalDate().Get() == TInstant::ParseIso8601("2007-07-07")); - UNIT_ASSERT(*parser.ColumnParser(11).GetOptionalDatetime().Get() == TInstant::ParseIso8601("2008-08-08T08:08:08Z")); - UNIT_ASSERT(*parser.ColumnParser(12).GetOptionalTimestamp().Get() == TInstant::ParseIso8601("2009-09-09T09:09:09.09Z")); + UNIT_ASSERT(parser.ColumnParser(0).GetOptionalUint64().value() == 42); + UNIT_ASSERT(parser.ColumnParser(1).GetOptionalBool().value() == true); + UNIT_ASSERT(parser.ColumnParser(2).GetOptionalInt32().value() == -1); + UNIT_ASSERT(parser.ColumnParser(3).GetOptionalUint32().value() == 1); + UNIT_ASSERT(parser.ColumnParser(4).GetOptionalInt64().value() == -2); + UNIT_ASSERT(parser.ColumnParser(5).GetOptionalUint64().value() == 2); + UNIT_ASSERT(parser.ColumnParser(6).GetOptionalFloat().value() == 3.0); + UNIT_ASSERT(parser.ColumnParser(7).GetOptionalDouble().value() == 4.0); + UNIT_ASSERT(parser.ColumnParser(8).GetOptionalString().value() == "five"); + UNIT_ASSERT(parser.ColumnParser(9).GetOptionalUtf8().value() == "six"); + UNIT_ASSERT(parser.ColumnParser(10).GetOptionalDate().value() == TInstant::ParseIso8601("2007-07-07")); + UNIT_ASSERT(parser.ColumnParser(11).GetOptionalDatetime().value() == TInstant::ParseIso8601("2008-08-08T08:08:08Z")); + UNIT_ASSERT(parser.ColumnParser(12).GetOptionalTimestamp().value() == TInstant::ParseIso8601("2009-09-09T09:09:09.09Z")); Cerr << TInstant::Days(10).MicroSeconds() << Endl; - UNIT_ASSERT(*parser.ColumnParser(13).GetOptionalInterval().Get() == TInstant::Days(10).MicroSeconds()); - UNIT_ASSERT(parser.ColumnParser(14).GetOptionalDecimal().Get()->ToString() == TString("11.11")); - UNIT_ASSERT(*parser.ColumnParser(15).GetOptionalJson().Get() == TString("[12]")); - UNIT_ASSERT(*parser.ColumnParser(16).GetOptionalYson().Get() == TString("[13]")); - UNIT_ASSERT(*parser.ColumnParser(17).GetOptionalJsonDocument().Get() == TString("[14]")); - UNIT_ASSERT(*parser.ColumnParser(18).GetOptionalDyNumber().Get() == TString(".1515e2")); + UNIT_ASSERT(parser.ColumnParser(13).GetOptionalInterval().value() == TInstant::Days(10).MicroSeconds()); + UNIT_ASSERT(parser.ColumnParser(14).GetOptionalDecimal().value().ToString() == "11.11"); + UNIT_ASSERT(parser.ColumnParser(15).GetOptionalJson().value() == "[12]"); + UNIT_ASSERT(parser.ColumnParser(16).GetOptionalYson().value() == "[13]"); + UNIT_ASSERT(parser.ColumnParser(17).GetOptionalJsonDocument().value() == "[14]"); + UNIT_ASSERT(parser.ColumnParser(18).GetOptionalDyNumber().value() == ".1515e2"); } } diff --git a/ydb/core/kqp/ut/arrow/kqp_types_arrow_ut.cpp b/ydb/core/kqp/ut/arrow/kqp_types_arrow_ut.cpp index 655e0a2fae12..55492178d546 100644 --- a/ydb/core/kqp/ut/arrow/kqp_types_arrow_ut.cpp +++ b/ydb/core/kqp/ut/arrow/kqp_types_arrow_ut.cpp @@ -70,26 +70,26 @@ void InsertAllColumnsAndCheckSelectAll(TKikimrRunner* runner) { UNIT_ASSERT_C(columns.size() == 20, "Wrong columns count"); NYdb::TResultSetParser parser(resultSet); UNIT_ASSERT_C(parser.TryNextRow(), "Row is missing"); - UNIT_ASSERT(*parser.ColumnParser(0).GetOptionalUint64().Get() == 42); - UNIT_ASSERT(*parser.ColumnParser(1).GetOptionalBool().Get() == true); - UNIT_ASSERT(*parser.ColumnParser(2).GetOptionalInt32().Get() == -1); - UNIT_ASSERT(*parser.ColumnParser(3).GetOptionalUint32().Get() == 1); - UNIT_ASSERT(*parser.ColumnParser(4).GetOptionalInt64().Get() == -2); - UNIT_ASSERT(*parser.ColumnParser(5).GetOptionalUint64().Get() == 2); - UNIT_ASSERT(*parser.ColumnParser(6).GetOptionalFloat().Get() == 3.0); - UNIT_ASSERT(*parser.ColumnParser(7).GetOptionalDouble().Get() == 4.0); - UNIT_ASSERT(*parser.ColumnParser(8).GetOptionalString().Get() == TString("five")); - UNIT_ASSERT(*parser.ColumnParser(9).GetOptionalUtf8().Get() == TString("six")); - UNIT_ASSERT(*parser.ColumnParser(10).GetOptionalDate().Get() == TInstant::ParseIso8601("2007-07-07")); - UNIT_ASSERT(*parser.ColumnParser(11).GetOptionalDatetime().Get() == TInstant::ParseIso8601("2008-08-08T08:08:08Z")); - UNIT_ASSERT(*parser.ColumnParser(12).GetOptionalTimestamp().Get() == TInstant::ParseIso8601("2009-09-09T09:09:09.09Z")); + UNIT_ASSERT(parser.ColumnParser(0).GetOptionalUint64().value() == 42); + UNIT_ASSERT(parser.ColumnParser(1).GetOptionalBool().value() == true); + UNIT_ASSERT(parser.ColumnParser(2).GetOptionalInt32().value() == -1); + UNIT_ASSERT(parser.ColumnParser(3).GetOptionalUint32().value() == 1); + UNIT_ASSERT(parser.ColumnParser(4).GetOptionalInt64().value() == -2); + UNIT_ASSERT(parser.ColumnParser(5).GetOptionalUint64().value() == 2); + UNIT_ASSERT(parser.ColumnParser(6).GetOptionalFloat().value() == 3.0); + UNIT_ASSERT(parser.ColumnParser(7).GetOptionalDouble().value() == 4.0); + UNIT_ASSERT(parser.ColumnParser(8).GetOptionalString().value() == "five"); + UNIT_ASSERT(parser.ColumnParser(9).GetOptionalUtf8().value() == "six"); + UNIT_ASSERT(parser.ColumnParser(10).GetOptionalDate().value() == TInstant::ParseIso8601("2007-07-07")); + UNIT_ASSERT(parser.ColumnParser(11).GetOptionalDatetime().value() == TInstant::ParseIso8601("2008-08-08T08:08:08Z")); + UNIT_ASSERT(parser.ColumnParser(12).GetOptionalTimestamp().value() == TInstant::ParseIso8601("2009-09-09T09:09:09.09Z")); Cerr << TInstant::Days(10).MicroSeconds() << Endl; - UNIT_ASSERT(*parser.ColumnParser(13).GetOptionalInterval().Get() == TInstant::Days(10).MicroSeconds()); - UNIT_ASSERT(parser.ColumnParser(14).GetOptionalDecimal().Get()->ToString() == TString("11.11")); - UNIT_ASSERT(*parser.ColumnParser(15).GetOptionalJson().Get() == TString("[12]")); - UNIT_ASSERT(*parser.ColumnParser(16).GetOptionalYson().Get() == TString("[13]")); - UNIT_ASSERT(*parser.ColumnParser(17).GetOptionalJsonDocument().Get() == TString("[14]")); - UNIT_ASSERT(*parser.ColumnParser(18).GetOptionalDyNumber().Get() == TString(".1515e2")); + UNIT_ASSERT(parser.ColumnParser(13).GetOptionalInterval().value() == TInstant::Days(10).MicroSeconds()); + UNIT_ASSERT(parser.ColumnParser(14).GetOptionalDecimal().value().ToString() == "11.11"); + UNIT_ASSERT(parser.ColumnParser(15).GetOptionalJson().value() == "[12]"); + UNIT_ASSERT(parser.ColumnParser(16).GetOptionalYson().value() == "[13]"); + UNIT_ASSERT(parser.ColumnParser(17).GetOptionalJsonDocument().value() == "[14]"); + UNIT_ASSERT(parser.ColumnParser(18).GetOptionalDyNumber().value() == ".1515e2"); UNIT_ASSERT(parser.ColumnParser(19).GetInt32() == 123); } diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h index ba85f6686162..394d61cc4047 100644 --- a/ydb/core/kqp/ut/common/columnshard.h +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -9,8 +9,8 @@ #include #include #include -#include -#include +#include +#include #include diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 45601cb9d42c..c05e2d249ff4 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include @@ -588,8 +588,29 @@ bool HasIssue(const NYql::TIssues& issues, ui32 code, return hasIssue; } +bool HasIssue(const NYdb::NIssue::TIssues& issues, ui32 code, + std::function predicate) +{ + bool hasIssue = false; + + for (auto& issue : issues) { + NYdb::NIssue::WalkThroughIssues(issue, false, [code, predicate, &hasIssue] (const NYdb::NIssue::TIssue& issue, int level) { + Y_UNUSED(level); + if (issue.GetCode() == code) { + bool issueMatch = predicate + ? predicate(issue) + : true; + + hasIssue = hasIssue || issueMatch; + } + }); + } + + return hasIssue; +} + void PrintQueryStats(const TDataQueryResult& result) { - if (!result.GetStats().Defined()) { + if (!result.GetStats().has_value()) { return; } @@ -800,7 +821,7 @@ TString StreamResultToYsonImpl(TIterator& it, TVector* profiles, bool t if (!streamPart.IsSuccess()) { if (opStatus != NYdb::EStatus::SUCCESS) { UNIT_ASSERT_VALUES_EQUAL_C(streamPart.GetStatus(), opStatus, streamPart.GetIssues().ToString()); - UNIT_ASSERT_C(streamPart.GetIssues().ToString().Contains(issueMessageSubString), TStringBuilder() << "Issue should contain '" << issueMessageSubString << "'. " << streamPart.GetIssues().ToString()); + UNIT_ASSERT_C(streamPart.GetIssues().ToString().contains(issueMessageSubString), TStringBuilder() << "Issue should contain '" << issueMessageSubString << "'. " << streamPart.GetIssues().ToString()); break; } if (throwOnTimeout && IsTimeoutError(streamPart.GetStatus())) { diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index ff5e68e4420a..5e392069872a 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -6,10 +6,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -302,6 +302,9 @@ void CreateManyShardsTable(TKikimrRunner& kikimr, ui32 totalRows = 1000, ui32 sh bool HasIssue(const NYql::TIssues& issues, ui32 code, std::function predicate = {}); +bool HasIssue(const NYdb::NIssue::TIssues& issues, ui32 code, + std::function predicate = {}); + void PrintQueryStats(const NYdb::NTable::TDataQueryResult& result); struct TExpectedTableStats { diff --git a/ydb/core/kqp/ut/common/ya.make b/ydb/core/kqp/ut/common/ya.make index 47db133340da..e7453d2297a0 100644 --- a/ydb/core/kqp/ut/common/ya.make +++ b/ydb/core/kqp/ut/common/ya.make @@ -20,12 +20,12 @@ PEERDIR( yql/essentials/utils/backtrace ydb/public/lib/yson_value ydb/core/tx/columnshard/test_helper - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp index 81318f827d29..7cc8678ccde4 100644 --- a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp +++ b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/kqp/ut/data/kqp_read_null_ut.cpp b/ydb/core/kqp/ut/data/kqp_read_null_ut.cpp index e6cbaf81fadf..529080dc8968 100644 --- a/ydb/core/kqp/ut/data/kqp_read_null_ut.cpp +++ b/ydb/core/kqp/ut/data/kqp_read_null_ut.cpp @@ -9,7 +9,7 @@ #include -#include +#include namespace NKikimr { diff --git a/ydb/core/kqp/ut/data/ya.make b/ydb/core/kqp/ut/data/ya.make index 60aefef6d00e..b1836bc0f15b 100644 --- a/ydb/core/kqp/ut/data/ya.make +++ b/ydb/core/kqp/ut/data/ya.make @@ -14,7 +14,7 @@ PEERDIR( ydb/core/testlib/default ydb/core/tx ydb/core/tx/datashard/ut_common - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/src/client/types ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp b/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp index 5bb12bc60a5a..6474a18b0bf2 100644 --- a/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp +++ b/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -41,8 +41,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { ("foo", 10u), ("bar", 11u), ("baz", 10u) )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Duplicated keys found."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Duplicated keys found."); })); result = session.ExecuteDataQuery(R"( @@ -63,8 +63,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { ("foo", 1u), ("bar", 11u) )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Conflict with existing key."); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Conflict with existing key."); }), result.GetIssues().ToString()); result = session.ExecuteDataQuery(R"( @@ -144,8 +144,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in) )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Duplicated keys found."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Duplicated keys found."); })); result = session.ExecuteDataQuery(R"( @@ -183,8 +183,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in) )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Conflict with existing key."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Conflict with existing key."); })); result = session.ExecuteDataQuery(R"( @@ -260,8 +260,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { INSERT INTO `/Root/TwoShard` SELECT Value3 as Key, Value1, Value2 FROM `/Root/Foo` )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Duplicated keys found."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Duplicated keys found."); })); result = session.ExecuteDataQuery(R"( @@ -299,8 +299,8 @@ Y_UNIT_TEST_SUITE(KqpEffects) { INSERT INTO `/Root/TwoShard` SELECT * FROM `/Root/Foo` )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Conflict with existing key."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Conflict with existing key."); })); result = session.ExecuteDataQuery(R"( diff --git a/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp b/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp index 95f47eca1a52..94ef94ccab1a 100644 --- a/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp +++ b/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -403,8 +403,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { SELECT * FROM TestImmediateEffects; )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Duplicated keys found."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Duplicated keys found."); })); } } @@ -429,8 +429,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { SELECT * FROM TestImmediateEffects; )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Conflict with existing key."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const auto& issue) { + return issue.GetMessage().contains("Conflict with existing key."); })); } } @@ -921,7 +921,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx; + std::optional tx; { auto result = session1.ExecuteDataQuery(R"( --!syntax_v1 @@ -992,7 +992,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session); - TMaybe tx; + std::optional tx; { auto result = session.ExecuteDataQuery(R"( --!syntax_v1 @@ -1070,7 +1070,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); - TMaybe tx; + std::optional tx; { auto result = session.ExecuteDataQuery(R"( --!syntax_v1 @@ -1174,7 +1174,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); - TMaybe tx; + std::optional tx; { auto result = session.ExecuteDataQuery(R"( --!syntax_v1 @@ -1232,7 +1232,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session); - TMaybe tx1; + std::optional tx1; auto session1 = db.CreateSession().GetValueSync().GetSession(); { auto result = session1.ExecuteDataQuery(R"( @@ -1246,7 +1246,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { UNIT_ASSERT(tx1); } - TMaybe tx2; + std::optional tx2; auto session2 = db.CreateSession().GetValueSync().GetSession(); { auto result = session2.ExecuteDataQuery(R"( @@ -1310,7 +1310,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session); - TMaybe tx1; + std::optional tx1; auto session1 = db.CreateSession().GetValueSync().GetSession(); { auto result = session1.ExecuteDataQuery(R"( @@ -1326,7 +1326,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { UNIT_ASSERT(tx1); } - TMaybe tx2; + std::optional tx2; auto session2 = db.CreateSession().GetValueSync().GetSession(); { // This just establishes a snapshot that is before tx1 commit @@ -1400,7 +1400,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); auto session1 = db.CreateSession().GetValueSync().GetSession(); - TMaybe tx1; + std::optional tx1; { auto result = session.ExecuteDataQuery(R"( --!syntax_v1 @@ -1448,8 +1448,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 auto result = session1.ExecuteDataQuery(R"( @@ -1505,8 +1505,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 auto result = session1.ExecuteDataQuery(R"( @@ -1566,8 +1566,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 auto result = session1.ExecuteDataQuery(R"( @@ -1627,8 +1627,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // write1 auto result = session1.ExecuteDataQuery(R"( @@ -1684,8 +1684,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // write1 auto result = session1.ExecuteDataQuery(R"( @@ -1739,8 +1739,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // write1 auto result = session1.ExecuteDataQuery(R"( @@ -1795,8 +1795,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // write1 auto result = session1.ExecuteDataQuery(R"( @@ -1851,8 +1851,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 + write1 auto result = session1.ExecuteDataQuery(R"( @@ -1912,8 +1912,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 + write1 auto result = session1.ExecuteDataQuery(R"( @@ -1968,8 +1968,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 + write1 auto result = session1.ExecuteDataQuery(R"( @@ -2025,8 +2025,8 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session1); - TMaybe tx1; - TMaybe tx2; + std::optional tx1; + std::optional tx2; { // read1 + write1 auto result = session1.ExecuteDataQuery(R"( @@ -2084,7 +2084,7 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CreateShardedTestTable(session); - TMaybe tx; + std::optional tx; NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); diff --git a/ydb/core/kqp/ut/effects/kqp_inplace_update_ut.cpp b/ydb/core/kqp/ut/effects/kqp_inplace_update_ut.cpp index 87488940d18c..da6187a2eef2 100644 --- a/ydb/core/kqp/ut/effects/kqp_inplace_update_ut.cpp +++ b/ydb/core/kqp/ut/effects/kqp_inplace_update_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/effects/kqp_write_ut.cpp b/ydb/core/kqp/ut/effects/kqp_write_ut.cpp index 4d4f0e0700fa..8a7e369a605c 100644 --- a/ydb/core/kqp/ut/effects/kqp_write_ut.cpp +++ b/ydb/core/kqp/ut/effects/kqp_write_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/federated_query/common/common.h b/ydb/core/kqp/ut/federated_query/common/common.h index a06d044a3e6d..06be0e9a5a41 100644 --- a/ydb/core/kqp/ut/federated_query/common/common.h +++ b/ydb/core/kqp/ut/federated_query/common/common.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include namespace NKikimr::NKqp::NFederatedQueryTest { diff --git a/ydb/core/kqp/ut/federated_query/common/ya.make b/ydb/core/kqp/ut/federated_query/common/ya.make index 61e24c1e10ef..93a828077ede 100644 --- a/ydb/core/kqp/ut/federated_query/common/ya.make +++ b/ydb/core/kqp/ut/federated_query/common/ya.make @@ -9,8 +9,8 @@ STYLE_CPP() PEERDIR( ydb/core/kqp/ut/common ydb/library/yql/providers/s3/actors_factory - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_query + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/query ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp index 480ad5f29912..34647372530b 100644 --- a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp @@ -5,9 +5,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -161,7 +161,7 @@ namespace NKikimr::NKqp { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(query).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_C(readyOp.Metadata().ExecStatus == EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp index 650492f7175d..3fcf790121e9 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp @@ -3,11 +3,11 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include @@ -73,7 +73,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -208,7 +208,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql, settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -251,7 +251,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -324,7 +324,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { , "external_source"_a = externalDataSourceName , "ydb_table"_a = ydbTable)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -386,7 +386,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { SELECT * FROM `{external_table}` )", "external_table"_a=externalTableName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_C(readyOp.Metadata().ExecStatus == EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -464,7 +464,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { , "external_source"_a = externalDataSourceName , "ydb_table"_a = ydbTable)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_C(readyOp.Metadata().ExecStatus == EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -515,7 +515,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { ) )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -577,7 +577,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); TFetchScriptResultsResult results(TStatus(EStatus::SUCCESS, {})); @@ -1363,7 +1363,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto scriptResult = yqlScriptClient.ExplainYqlScript(sql, settings).GetValueSync(); UNIT_ASSERT_C(scriptResult.IsSuccess(), scriptResult.GetIssues().ToString()); - UNIT_ASSERT(scriptResult.GetPlan()); + UNIT_ASSERT(!scriptResult.GetPlan().empty()); } Y_UNIT_TEST(ReadFromDataSourceWithoutTable) { @@ -1399,7 +1399,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Failed); @@ -1424,7 +1424,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -1681,7 +1681,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto rs = result.GetResultSetParser(0); UNIT_ASSERT_VALUES_EQUAL(rs.RowsCount(), 1); rs.TryNextRow(); - TMaybe sum = rs.ColumnParser(0).GetOptionalDouble(); + std::optional sum = rs.ColumnParser(0).GetOptionalDouble(); UNIT_ASSERT(!sum); } } @@ -1753,7 +1753,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("data").GetString(), rowContent); } - if (!results.GetNextFetchToken()) { + if (results.GetNextFetchToken().empty()) { break; } @@ -1935,7 +1935,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { void RunGenericScript(const TString& script, TQueryClient& client, const TDriver& driver) { auto scriptExecutionOperation = client.ExecuteScript(script).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), driver); UNIT_ASSERT_VALUES_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToOneLineString()); @@ -2205,7 +2205,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto db = kikimr->GetQueryClient(); auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -2375,7 +2375,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -2390,7 +2390,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -2405,7 +2405,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -2420,7 +2420,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp index f8f19e11418c..6e138cea69ce 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp index bc5a6ce27256..26e08db12b8f 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp @@ -3,11 +3,11 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include diff --git a/ydb/core/kqp/ut/federated_query/s3/ya.make b/ydb/core/kqp/ut/federated_query/s3/ya.make index 84a73ef09873..2b2f48e24111 100644 --- a/ydb/core/kqp/ut/federated_query/s3/ya.make +++ b/ydb/core/kqp/ut/federated_query/s3/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/library/yql/providers/s3/actors yql/essentials/sql/pg_dummy ydb/library/testlib/s3_recipe_helper - ydb/public/sdk/cpp/client/ydb_types/operation + ydb/public/sdk/cpp/src/client/types/operation ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/idx_test/ya.make b/ydb/core/kqp/ut/idx_test/ya.make index 42bbc8322b21..9f274e76d363 100644 --- a/ydb/core/kqp/ut/idx_test/ya.make +++ b/ydb/core/kqp/ut/idx_test/ya.make @@ -22,8 +22,8 @@ PEERDIR( ydb/core/kqp/ut/common ydb/public/lib/idx_test ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table yql/essentials/sql/pg_dummy ) diff --git a/ydb/core/kqp/ut/idx_test/ydb_index_ut.cpp b/ydb/core/kqp/ut/idx_test/ydb_index_ut.cpp index 766cc12e33ac..bac85976bb5c 100644 --- a/ydb/core/kqp/ut/idx_test/ydb_index_ut.cpp +++ b/ydb/core/kqp/ut/idx_test/ydb_index_ut.cpp @@ -33,10 +33,10 @@ static void RunTest(ui32 shardsCount, ui32 rowsCount, ui32 indexCount, const TRu auto builder = TTableBuilder() .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); - TVector pks; + std::vector pks; pks.push_back(keyColumnName); - TVector dataColumn; + std::vector dataColumn; if (withDataColumn) { dataColumn.push_back("value"); } @@ -53,15 +53,15 @@ static void RunTest(ui32 shardsCount, ui32 rowsCount, ui32 indexCount, const TRu if (!pkOverlap) { if (uniqIndex) { - builder.AddUniqueSecondaryIndex(ss.Str() + "_name", TVector{ss.Str()}, dataColumn); + builder.AddUniqueSecondaryIndex(ss.Str() + "_name", std::vector{ss.Str()}, dataColumn); } else { - builder.AddSecondaryIndex(ss.Str() + "_name", TVector{ss.Str()}, dataColumn); + builder.AddSecondaryIndex(ss.Str() + "_name", std::vector{ss.Str()}, dataColumn); } } else { if (uniqIndex) { - builder.AddUniqueSecondaryIndex(ss.Str() + "_name", TVector{ss.Str(), keyColumnName}, dataColumn); + builder.AddUniqueSecondaryIndex(ss.Str() + "_name", std::vector{ss.Str(), keyColumnName}, dataColumn); } else { - builder.AddSecondaryIndex(ss.Str() + "_name", TVector{ss.Str(), keyColumnName}, dataColumn); + builder.AddSecondaryIndex(ss.Str() + "_name", std::vector{ss.Str(), keyColumnName}, dataColumn); } } if (indexOverlap) { diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp index d4c7cb2b97c4..a98fecc1e3c3 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp @@ -1,7 +1,8 @@ #include -#include -#include +#include +#include +#include #include #include @@ -1561,7 +1562,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { auto result = ExecuteDataQuery(session, query); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, + UNIT_ASSERT_C(HasIssue(NYdb::NAdapters::ToYqlIssues(result.GetIssues()), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, [](const NYql::TIssue& issue) { return issue.GetMessage().Contains("Given predicate is not suitable for used index: index"); }), result.GetIssues().ToString()); @@ -1577,7 +1578,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { auto result = ExecuteDataQuery(session, query); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, + UNIT_ASSERT_C(HasIssue(NYdb::NAdapters::ToYqlIssues(result.GetIssues()), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, [](const NYql::TIssue& issue) { return issue.GetMessage().Contains("Given predicate is not suitable for used index: index"); }), result.GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 9773c391fbe7..93fd36ab74e5 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -6,8 +6,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -238,8 +238,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); Cerr << explainResult.GetAst() << Endl; - UNIT_ASSERT_C(explainResult.GetAst().Contains("'('\"Reverse\")"), explainResult.GetAst()); - UNIT_ASSERT_C(explainResult.GetAst().Contains("'('\"Sorted\")"), explainResult.GetAst()); + UNIT_ASSERT_C(explainResult.GetAst().contains("'('\"Reverse\")"), explainResult.GetAst()); + UNIT_ASSERT_C(explainResult.GetAst().contains("'('\"Sorted\")"), explainResult.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(explainResult.GetPlan(), &plan, true); @@ -390,12 +390,12 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { auto session = db.CreateSession().GetValueSync().GetSession(); { - const TString keyColumnName = "key"; + const std::string keyColumnName = "key"; auto builder = TTableBuilder() .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); builder.AddNullableColumn("index_0", EPrimitiveType::Utf8); - builder.AddSecondaryIndex("index_0_name", TVector{"index_0", keyColumnName}); + builder.AddSecondaryIndex("index_0_name", std::vector{"index_0", keyColumnName}); builder.AddNullableColumn("value", EPrimitiveType::Uint32); builder.SetPrimaryKeyColumns({keyColumnName}); @@ -452,8 +452,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -511,7 +511,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { // Continue tx1, select from index table only auto result2 = session.ExecuteDataQuery( query2, - TTxControl::Tx(result1.GetTransaction().GetRef()).CommitTx()) + TTxControl::Tx(result1.GetTransaction().value()).CommitTx()) .ExtractValueSync(); // read only tx should succeed in MVCC case UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), NYdb::EStatus::SUCCESS, result2.GetIssues().ToString()); @@ -531,8 +531,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -591,7 +591,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { // Continue tx1, write to table should fail in both MVCC and non-MVCC scenarios auto result2 = session.ExecuteDataQuery( query2, - TTxControl::Tx(result1.GetTransaction().GetRef()).CommitTx()) + TTxControl::Tx(result1.GetTransaction().value()).CommitTx()) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), NYdb::EStatus::ABORTED, result2.GetIssues().ToString().c_str()); } @@ -613,8 +613,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("fk2", EPrimitiveType::Int32) .AddNullableColumn("fk3", EPrimitiveType::Uint64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"k1", "k2"}); - tableBuilder.AddUniqueSecondaryIndex("Index", TVector{"fk1", "fk2", "fk3", "k2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"k1", "k2"}); + tableBuilder.AddUniqueSecondaryIndex("Index", std::vector{"fk1", "fk2", "fk3", "k2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -796,9 +796,9 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("fk2", EPrimitiveType::Int32) .AddNullableColumn("fk3", EPrimitiveType::Uint64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"k1"}); - tableBuilder.AddUniqueSecondaryIndex("Index12", TVector{"fk1", "fk2"}, TVector{"Value"}); - tableBuilder.AddUniqueSecondaryIndex("Index3", TVector{"fk3"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"k1"}); + tableBuilder.AddUniqueSecondaryIndex("Index12", std::vector{"fk1", "fk2"}, std::vector{"Value"}); + tableBuilder.AddUniqueSecondaryIndex("Index3", std::vector{"fk3"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -915,11 +915,11 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("fk2", EPrimitiveType::Int32) .AddNullableColumn("fk3", EPrimitiveType::Uint64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); if (uniq) { - tableBuilder.AddUniqueSecondaryIndex("Index", TVector{"fk1", "fk2", "fk3"}); + tableBuilder.AddUniqueSecondaryIndex("Index", std::vector{"fk1", "fk2", "fk3"}); } else { - tableBuilder.AddSecondaryIndex("Index", TVector{"fk1", "fk2", "fk3"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"fk1", "fk2", "fk3"}); } auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1031,8 +1031,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1244,7 +1244,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("IndexColumn", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1340,8 +1340,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("IndexColumn1", EPrimitiveType::String) .AddNullableColumn("IndexColumn2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "IndexColumn1", "IndexColumn2"}); - tableBuilder.AddSecondaryIndex("IndexName1", TVector{"IndexColumn1"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key", "IndexColumn1", "IndexColumn2"}); + tableBuilder.AddSecondaryIndex("IndexName1", std::vector{"IndexColumn1"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1474,8 +1474,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1666,8 +1666,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Index2A", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2", "Index2A"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2", "Index2A"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1755,7 +1755,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index", "Index2"); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -1860,8 +1860,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::Int64) .AddNullableColumn("Index2", EPrimitiveType::Int64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1894,7 +1894,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -1920,8 +1920,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -1946,8 +1946,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -1972,7 +1972,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -1997,7 +1997,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2021,8 +2021,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); } { @@ -2047,8 +2047,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { query) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2073,8 +2073,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { query) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2100,8 +2100,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { query) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2127,8 +2127,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { query) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2271,7 +2271,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNonNullableColumn("emb", EPrimitiveType::String) .AddNonNullableColumn("data", EPrimitiveType::String); } - tableBuilder.SetPrimaryKeyColumns(TVector{"pk"}); + tableBuilder.SetPrimaryKeyColumns({"pk"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -2652,8 +2652,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Key", EPrimitiveType::Int64) .AddNullableColumn("Index2", EPrimitiveType::Int64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -2674,7 +2674,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); UNIT_ASSERT_C(!result.GetDiagnostics().empty(), "Query result diagnostics is empty"); @@ -2707,7 +2707,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); UNIT_ASSERT_C(result.GetDiagnostics().empty(), "Query result diagnostics should be empty, but it's not"); } @@ -2729,10 +2729,10 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("customer", EPrimitiveType::Utf8) .AddNullableColumn("created", EPrimitiveType::Datetime) .AddNullableColumn("processed", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"id"}); - tableBuilder.AddSecondaryIndex("ix_cust", TVector{"customer"}); - tableBuilder.AddSecondaryIndex("ix_cust2", TVector{"customer", "created"}); - tableBuilder.AddSecondaryIndex("ix_cust3", TVector{"customer", "created"}, TVector{"processed"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"id"}); + tableBuilder.AddSecondaryIndex("ix_cust", std::vector{"customer"}); + tableBuilder.AddSecondaryIndex("ix_cust2", std::vector{"customer", "created"}); + tableBuilder.AddSecondaryIndex("ix_cust3", std::vector{"customer", "created"}, std::vector{"processed"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -2770,7 +2770,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2817,8 +2817,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"ItemsLimit"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2864,8 +2864,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"ItemsLimit"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'('\"Reverse\")"), result.GetAst()); } { @@ -2903,8 +2903,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Index2A", EPrimitiveType::Uint8) .AddNullableColumn("Value", EPrimitiveType::Utf8); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2", "Index2A"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2", "Index2A"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -2978,8 +2978,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -3022,7 +3022,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value1", EPrimitiveType::String) .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index1", "Value1"); tableBuilder.AddSecondaryIndex("Index2", "Value2"); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); @@ -3085,7 +3085,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value1", EPrimitiveType::String) .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddUniqueSecondaryIndex("Index1Uniq", {"Value1"}); tableBuilder.AddSecondaryIndex("Index2NotUniq", "Value2"); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); @@ -3275,7 +3275,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value1", EPrimitiveType::String) .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddUniqueSecondaryIndex("Index1Uniq", {"Value1"}, {"Value2"}); tableBuilder.AddSecondaryIndex("Index2NotUniq", {"Value2"}, {"Value1"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); @@ -3399,9 +3399,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Value1", EPrimitiveType::String) .AddNullableColumn("Value2", EPrimitiveType::String) .AddNullableColumn("Value3", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", TVector{"Value1", "Value3"}); - tableBuilder.AddSecondaryIndex("Index2", TVector{"Value2", "Value3"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", std::vector{"Value1", "Value3"}); + tableBuilder.AddSecondaryIndex("Index2", std::vector{"Value2", "Value3"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -3735,9 +3735,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("KeyA", EPrimitiveType::Int64) .AddNullableColumn("Value1", EPrimitiveType::String) .AddNullableColumn("Payload", EPrimitiveType::Utf8); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index1", TVector{"Value1", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index2", TVector{"Key", "Value1"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index1", std::vector{"Value1", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index2", std::vector{"Key", "Value1"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4008,7 +4008,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index1", "Value"); auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -4019,7 +4019,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index1", "Value"); auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -4249,7 +4249,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index1", "Value"); auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -4261,7 +4261,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::Int64) .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); tableBuilder.AddSecondaryIndex("Index1", "Value"); auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -4407,8 +4407,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4434,7 +4434,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) .ExtractValueSync(); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); } @@ -4447,7 +4447,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) .ExtractValueSync(); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); } @@ -4460,7 +4460,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) .ExtractValueSync(); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); } @@ -4473,7 +4473,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) .ExtractValueSync(); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); } @@ -4493,7 +4493,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4504,7 +4504,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Value", EPrimitiveType::String) .AddNullableColumn("Key", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Value"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Value"}); auto result = session.CreateTable("/Root/TestTable/Index", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); @@ -4521,7 +4521,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder .AddNullableColumn("Value", EPrimitiveType::String) .AddNullableColumn("Key", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Value"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Value"}); auto result = session.CreateTable("/Root/TestTable/Index/indexImplTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); @@ -4541,8 +4541,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4577,8 +4577,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4663,8 +4663,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .AddNullableColumn("Key", EPrimitiveType::String) .AddNullableColumn("Index2", EPrimitiveType::String) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector{"Index2"}); + tableBuilder.SetPrimaryKeyColumns(std::vector{"Key"}); + tableBuilder.AddSecondaryIndex("Index", std::vector{"Index2"}); auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -4954,7 +4954,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_C(!result.GetAst().Contains("EquiJoin"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("EquiJoin"), result.GetAst()); auto params = TParamsBuilder() .AddParam("$targets") @@ -5315,7 +5315,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda )").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(schemeResult.GetStatus(), EStatus::SUCCESS, schemeResult.GetIssues().ToString()); - auto checkPlan = [](const TString& planJson, ui32 tableReads, ui32 tableWrites, TMaybe indexWrites) { + auto checkPlan = [](const std::string& planJson, ui32 tableReads, ui32 tableWrites, TMaybe indexWrites) { NJson::TJsonValue plan; NJson::ReadJsonTree(planJson, &plan, true); const auto& tables = plan["tables"]; diff --git a/ydb/core/kqp/ut/indexes/ya.make b/ydb/core/kqp/ut/indexes/ya.make index c058edef3f41..ca7535eca6ac 100644 --- a/ydb/core/kqp/ut/indexes/ya.make +++ b/ydb/core/kqp/ut/indexes/ya.make @@ -22,6 +22,7 @@ PEERDIR( ydb/library/yql/providers/common/http_gateway ydb/library/yql/udfs/common/knn yql/essentials/sql/pg_dummy + ydb/public/sdk/cpp/adapters/issue ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp index 3d03a0cae641..5a513a6c5480 100644 --- a/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index c36e8bc527a3..70dd5ee56971 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -205,7 +205,7 @@ class TChainTester { auto result = Session.ExplainDataQuery(joinRequest).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - PrintPlan(result.GetPlan()); + PrintPlan(TString{result.GetPlan()}); } TKikimrRunner Kikimr; @@ -234,7 +234,7 @@ void ExplainJoinOrderTestDataQueryWithStats(const TString& queryPath, const TStr ).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - PrintPlan(*result.GetStats()->GetPlan()); + PrintPlan(TString{*result.GetStats()->GetPlan()}); } } @@ -411,7 +411,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) ).ExtractValueSync(); - PrintPlan(*result.GetStats()->GetPlan()); + PrintPlan(TString{*result.GetStats()->GetPlan()}); NJson::TJsonValue plan; NJson::ReadJsonTree(*result.GetStats()->GetPlan(), &plan, true); @@ -696,7 +696,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { ).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); - PrintPlan(*result.GetStats()->GetPlan()); + PrintPlan(TString{*result.GetStats()->GetPlan()}); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); if (useStreamLookupJoin) { @@ -707,7 +707,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { correctJoinOrderPath = correctJoinOrderPath.substr(0, correctJoinOrderPath.find(".json")) + "_column_store.json"; } - auto currentJoinOrder = GetPrettyJSON(GetDetailedJoinOrder(*result.GetStats()->GetPlan())); + auto currentJoinOrder = GetPrettyJSON(GetDetailedJoinOrder(TString{*result.GetStats()->GetPlan()})); /* to canonize the tests use --test-param CANONIZE_JOIN_ORDER_TESTS=TRUE */ TString canonize = GetTestParam("CANONIZE_JOIN_ORDER_TESTS"); canonize.to_lower(); @@ -718,9 +718,9 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } TString ref = GetStatic(correctJoinOrderPath); - Cout << "actual\n" << GetJoinOrder(*result.GetStats()->GetPlan()).GetStringRobust() << Endl; + Cout << "actual\n" << GetJoinOrder(TString{*result.GetStats()->GetPlan()}).GetStringRobust() << Endl; Cout << "expected\n" << GetJoinOrderFromDetailedJoinOrder(ref).GetStringRobust() << Endl; - UNIT_ASSERT(JoinOrderAndAlgosMatch(*result.GetStats()->GetPlan(), ref)); + UNIT_ASSERT(JoinOrderAndAlgosMatch(TString{*result.GetStats()->GetPlan()}, ref)); } } diff --git a/ydb/core/kqp/ut/join/kqp_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_ut.cpp index c81dc50d3527..47cfbba286eb 100644 --- a/ydb/core/kqp/ut/join/kqp_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -215,7 +215,7 @@ static TDataQueryResult ExecQuery(TSession& session, const TString& query, const if (checkRewrite) { auto explain = session.ExplainDataQuery(query).GetValueSync(); - UNIT_ASSERT_C(explain.GetAst().Contains("PartitionByKey"), explain.GetAst()); + UNIT_ASSERT_C(explain.GetAst().contains("PartitionByKey"), explain.GetAst()); } return result; @@ -700,7 +700,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddListItem().BeginStruct().AddMember("v").OptionalString("1.One").EndStruct() // dup .AddListItem().BeginStruct().AddMember("v").OptionalString("1.Two").EndStruct() .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("v").OptionalString(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])", false); @@ -742,7 +742,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])"); @@ -790,12 +790,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddMember("k2").OptionalString("Two").EndStruct() .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(42) .AddMember("k2").OptionalString("FortyTwo").EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing()) + .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(std::nullopt) .AddMember("k2").OptionalString("One").EndStruct() // null .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1) - .AddMember("k2").OptionalString(Nothing()).EndStruct() // null - .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing()) - .AddMember("k2").OptionalString(Nothing()).EndStruct() // null + .AddMember("k2").OptionalString(std::nullopt).EndStruct() // null + .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(std::nullopt) + .AddMember("k2").OptionalString(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])"); @@ -906,7 +906,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])"); @@ -969,7 +969,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct() // dup .AddListItem().BeginStruct().AddMember("v").OptionalString("2.Five").EndStruct() .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("v").OptionalString(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])"); @@ -1022,8 +1022,8 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddMember("v").OptionalString("2.Five").EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42) .AddMember("v").OptionalString("Any").EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()) - .AddMember("v").OptionalString(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(std::nullopt) + .AddMember("v").OptionalString(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])"); @@ -1073,7 +1073,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup .AddListItem().BeginStruct().AddMember("k").OptionalInt32(5).EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(std::nullopt).EndStruct() // null .EndList().Build().Build(); auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])"); @@ -1666,7 +1666,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { [[105u];["One"];[105u];["One"];["Name2"]] ])", FormatResultSetYson(result.GetResultSet(0))); AssertTableReads(result, "/Root/Join2", 5); - UNIT_ASSERT(result.GetQueryPlan().Contains("Lookup")); + UNIT_ASSERT(result.GetQueryPlan().contains("Lookup")); } { @@ -1684,7 +1684,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { [[101u];["Two"];[101u];["Two"];["Name1"]] ])", FormatResultSetYson(result.GetResultSet(0))); AssertTableReads(result, "/Root/Join2", 2); - UNIT_ASSERT(result.GetQueryPlan().Contains("Lookup")); + UNIT_ASSERT(result.GetQueryPlan().contains("Lookup")); } { @@ -1703,7 +1703,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { [[103u];["One"];[103u];["One"];["Name1"]] ])", FormatResultSetYson(result.GetResultSet(0))); AssertTableReads(result, "/Root/Join2", 3); - UNIT_ASSERT(result.GetQueryPlan().Contains("Lookup")); + UNIT_ASSERT(result.GetQueryPlan().contains("Lookup")); } { @@ -1722,7 +1722,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { [[105u];["One"];[105u];["One"];["Name2"]] ])", FormatResultSetYson(result.GetResultSet(0))); AssertTableReads(result, "/Root/Join2", 3); - UNIT_ASSERT(result.GetQueryPlan().Contains("Lookup")); + UNIT_ASSERT(result.GetQueryPlan().contains("Lookup")); } { @@ -1740,7 +1740,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { [[105u];["One"];[105u];["One"];["Name2"]] ])", FormatResultSetYson(result.GetResultSet(0))); AssertTableReads(result, "/Root/Join2", 2); - UNIT_ASSERT(result.GetQueryPlan().Contains("Lookup")); + UNIT_ASSERT(result.GetQueryPlan().contains("Lookup")); } } diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index 4aadd5af93c8..4db334e774fa 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -11,8 +11,8 @@ #include #include -#include -#include +#include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/ut/olap/datatime64_ut.cpp b/ydb/core/kqp/ut/olap/datatime64_ut.cpp index 85aa88b5a76b..aee332a2f460 100644 --- a/ydb/core/kqp/ut/olap/datatime64_ut.cpp +++ b/ydb/core/kqp/ut/olap/datatime64_ut.cpp @@ -3,10 +3,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include diff --git a/ydb/core/kqp/ut/olap/decimal_ut.cpp b/ydb/core/kqp/ut/olap/decimal_ut.cpp index 472c5ad44d34..2e68ae64c0f0 100644 --- a/ydb/core/kqp/ut/olap/decimal_ut.cpp +++ b/ydb/core/kqp/ut/olap/decimal_ut.cpp @@ -8,10 +8,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/core/kqp/ut/olap/helpers/get_value.cpp b/ydb/core/kqp/ut/olap/helpers/get_value.cpp index 67fe905d8e44..542b1fd9a81e 100644 --- a/ydb/core/kqp/ut/olap/helpers/get_value.cpp +++ b/ydb/core/kqp/ut/olap/helpers/get_value.cpp @@ -86,7 +86,7 @@ TString GetUtf8(const NYdb::TValue& v) { if (value.GetKind() == NYdb::TTypeParser::ETypeKind::Optional) { return *value.GetOptionalUtf8(); } else { - return value.GetUtf8(); + return TString{value.GetUtf8()}; } } diff --git a/ydb/core/kqp/ut/olap/helpers/get_value.h b/ydb/core/kqp/ut/olap/helpers/get_value.h index 7902b816caef..d74f894a59ba 100644 --- a/ydb/core/kqp/ut/olap/helpers/get_value.h +++ b/ydb/core/kqp/ut/olap/helpers/get_value.h @@ -1,5 +1,5 @@ #pragma once -#include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/ut/olap/helpers/query_executor.cpp b/ydb/core/kqp/ut/olap/helpers/query_executor.cpp index 11ef90591b5e..7dbf61dd2f8f 100644 --- a/ydb/core/kqp/ut/olap/helpers/query_executor.cpp +++ b/ydb/core/kqp/ut/olap/helpers/query_executor.cpp @@ -1,7 +1,7 @@ #include "query_executor.h" #include "get_value.h" #include -#include +#include namespace NKikimr::NKqp { @@ -31,7 +31,7 @@ TVector> CollectRows(NYdb::NTable::TScanQueryPar } if (streamPart.HasDiagnostics()) { - TString diagnosticsString = streamPart.GetDiagnostics(); + auto diagnosticsString = TString{streamPart.GetDiagnostics()}; if (!diagnosticsString.empty() && diagnostics) { UNIT_ASSERT(NJson::ReadJsonFastTree(diagnosticsString, diagnostics)); } diff --git a/ydb/core/kqp/ut/olap/helpers/query_executor.h b/ydb/core/kqp/ut/olap/helpers/query_executor.h index 18dad7f72f34..b5545e817ebd 100644 --- a/ydb/core/kqp/ut/olap/helpers/query_executor.h +++ b/ydb/core/kqp/ut/olap/helpers/query_executor.h @@ -1,7 +1,7 @@ #pragma once #include -#include -#include +#include +#include #include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/ut/olap/helpers/typed_local.h b/ydb/core/kqp/ut/olap/helpers/typed_local.h index 9feb72ea3a63..dafcf55dde2a 100644 --- a/ydb/core/kqp/ut/olap/helpers/typed_local.h +++ b/ydb/core/kqp/ut/olap/helpers/typed_local.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index 0f4595691198..f86d7f7fc9bd 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/kqp/ut/olap/ya.make b/ydb/core/kqp/ut/olap/ya.make index 813d7d5f9153..f59f37004ec4 100644 --- a/ydb/core/kqp/ut/olap/ya.make +++ b/ydb/core/kqp/ut/olap/ya.make @@ -37,7 +37,7 @@ PEERDIR( ydb/core/tx/columnshard ydb/core/kqp/ut/olap/helpers ydb/core/tx/datashard/ut_common - ydb/public/sdk/cpp/client/ydb_operation + ydb/public/sdk/cpp/src/client/operation ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp b/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp index f8a15dc9f6b6..fac1d7e3731c 100644 --- a/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -175,7 +175,7 @@ void Test( CompareYson(answer, FormatResultSetYson(result.GetResultSet(0))); auto explain = session.ExplainDataQuery(query).ExtractValueSync(); - UNIT_ASSERT(explain.GetPlan().Contains("Lookup")); + UNIT_ASSERT(explain.GetPlan().contains("Lookup")); Cerr << explain.GetPlan(); NJson::TJsonValue plan; diff --git a/ydb/core/kqp/ut/opt/kqp_kv_ut.cpp b/ydb/core/kqp/ut/opt/kqp_kv_ut.cpp index 95fd9f378f00..04d4adde9853 100644 --- a/ydb/core/kqp/ut/opt/kqp_kv_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_kv_ut.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include @@ -101,7 +101,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { Cout << res << Endl; CompareYson(R"( [[[0u];[0];["abcde"]];[[137u];[1];["abcde"]];[[274u];[2];["abcde"]];[[411u];[3];["abcde"]];[[548u];[4];["abcde"]];[[685u];[5];["abcde"]];[[822u];[6];["abcde"]];[[959u];[7];["abcde"]];[[1096u];[8];["abcde"]];[[1233u];[9];["abcde"]]] - )", res); + )", TString{res}); } Y_UNIT_TEST(ReadRows_SpecificKey) { @@ -157,7 +157,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { [5765290426629915225u;3u;"abcde"]; [7687053901553772359u;4u;"abcde"] ] - )", res); + )", TString{res}); } Y_UNIT_TEST(ReadRows_UnknownTable) { @@ -191,7 +191,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { UNIT_ASSERT_C(selectResult.GetIssues().ToString().size(), "Expect non-empty issue in case of error"); UNIT_ASSERT_EQUAL(selectResult.GetStatus(), EStatus::SCHEME_ERROR); auto res = FormatResultSetYson(selectResult.GetResultSet()); - CompareYson("[]", res); + CompareYson("[]", TString{res}); } Y_UNIT_TEST(ReadRows_NonExistentKeys) { @@ -243,7 +243,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { UNIT_ASSERT_C(selectResult.IsSuccess(), selectResult.GetIssues().ToString()); auto res = FormatResultSetYson(selectResult.GetResultSet()); Cerr << res << Endl; - CompareYson("[]", res); + CompareYson("[]", TString{res}); } { NYdb::TValueBuilder keys; @@ -267,7 +267,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { [12u;2u;"abcde"]; [13u;3u;"abcde"]; [14u;4u;"abcde"] - ])", res); + ])", TString{res}); } { NYdb::TValueBuilder keys; @@ -363,7 +363,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { UNIT_ASSERT_C(selectResult.IsSuccess(), selectResult.GetIssues().ToString()); auto res = FormatResultSetYson(selectResult.GetResultSet()); - CompareYson(Sprintf("[[%du;%du]]", valueToReturn_1, valueToReturn_2), res); + CompareYson(Sprintf("[[%du;%du]]", valueToReturn_1, valueToReturn_2), TString{res}); } TVector<::ReadRowsPgParam> readRowsPgParams @@ -733,7 +733,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { ["1.123456789";"1000.123456789";"10.123456789";"1000000.123456789";1u]; ["2.123456789";"2000.123456789";"20.123456789";"2000000.123456789";2u] ] - )", res); + )", TString{res}); } // Good case: lookup overflowed decimal @@ -749,7 +749,7 @@ Y_UNIT_TEST_SUITE(KqpKv) { auto selectResult = db.ReadRows("/Root/TestTable", keys.Build()).GetValueSync(); UNIT_ASSERT_C(selectResult.IsSuccess(), selectResult.GetIssues().ToString()); auto res = FormatResultSetYson(selectResult.GetResultSet()); - CompareYson(R"([["inf";"inf";"inf";"inf";999999999u];])", res); + CompareYson(R"([["inf";"inf";"inf";"inf";999999999u];])", TString{res}); } } diff --git a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp index d35014e05109..58398038fe5d 100644 --- a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { auto explainResult = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(explainResult.GetStatus(), EStatus::SUCCESS, explainResult.GetIssues().ToString()); - UNIT_ASSERT_C(explainResult.GetAst().Contains("KqpReadRangesSource"), explainResult.GetAst()); + UNIT_ASSERT_C(explainResult.GetAst().contains("KqpReadRangesSource"), explainResult.GetAst()); auto params = kikimr.GetTableClient().GetParamsBuilder() .AddParam("$key").Uint64(302).Build() @@ -2199,7 +2199,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, TStringBuilder() << query << Endl << "Failed with: " << result.GetIssues().ToString()); - UNIT_ASSERT_C(result.GetAst().Contains("('('\"ItemsLimit\""), + UNIT_ASSERT_C(result.GetAst().contains("('('\"ItemsLimit\""), TStringBuilder() << query << Endl << "Failed with AST: " << result.GetAst()); NJson::TJsonValue plan; @@ -2620,7 +2620,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync(); AssertSuccessResult(result); auto resultYson = FormatResultSetYson(result.GetResultSet(0)); - CompareYson(item.second, resultYson); + CompareYson(item.second, TString{resultYson}); } for (auto& item: testData) { diff --git a/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp b/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp index 5252d6f1e6de..0238be6546e3 100644 --- a/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include namespace NKikimr { namespace NKqp { @@ -2024,7 +2024,7 @@ Y_UNIT_TEST_SUITE(KqpNotNullColumns) { proto.mutable_columns()->begin()->set_not_null(true); auto result = session.CreateTable("/Root/NotNullCheck2", TTableDescription(std::move(proto), {})).GetValueSync(); UNIT_ASSERT_C(!result.GetIssues().Empty(), "ok with faulty protobuf"); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Error: Not consistent column type and not_null option for column: 1")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Error: Not consistent column type and not_null option for column: 1")); } } diff --git a/ydb/core/kqp/ut/opt/kqp_ranges_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ranges_ut.cpp index f9068704f9da..87457cf4e2e6 100644 --- a/ydb/core/kqp/ut/opt/kqp_ranges_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_ranges_ut.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include @@ -1508,7 +1508,7 @@ Y_UNIT_TEST_SUITE(KqpRanges) { TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - CompareYson(expectedYson, FormatResultSetYson(result.GetResultSet(0))); + CompareYson(TString{expectedYson}, TString{FormatResultSetYson(result.GetResultSet(0))}); } } diff --git a/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp b/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp index 29d2d742e0f5..457ca8ace7db 100644 --- a/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp b/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp index 4ffed8ba2409..d2645e072340 100644 --- a/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp @@ -21,7 +21,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -57,7 +57,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -92,7 +92,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -130,7 +130,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -167,7 +167,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -205,9 +205,9 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -248,7 +248,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -260,7 +260,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { UNIT_ASSERT(read.IsDefined()); UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); - UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'\"ItemsLimit\""), result.GetAst()); } { @@ -288,9 +288,9 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("'\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("('\"Reverse\")"), result.GetAst()); NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); @@ -449,7 +449,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(result.GetAst().Contains("ItemsLimit"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().contains("ItemsLimit"), result.GetAst()); } { @@ -713,7 +713,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("KiPartialTake"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("KiPartialTake"), result.GetAst()); } { @@ -756,7 +756,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { auto result = session.ExplainDataQuery(query).GetValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_C(!result.GetAst().Contains("KiPartialTake"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().contains("KiPartialTake"), result.GetAst()); } { @@ -786,7 +786,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { .AddNullableColumn("Value1", EPrimitiveType::Int32) .AddNullableColumn("Value2", EPrimitiveType::String) .AddNullableColumn("Value3", EPrimitiveType::Uint32) - .SetPrimaryKeyColumns(TVector{"Key"}) + .SetPrimaryKeyColumns({"Key"}) .Build(); auto createSettings = TCreateTableSettings() diff --git a/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp b/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp index 6d620f79d827..876c4318023c 100644 --- a/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -91,7 +91,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { // empty parameters { - TMap paramsType; + std::map paramsType; paramsType.emplace("$in", TTypeBuilder().BeginList().Primitive(EPrimitiveType::Uint64).EndList().Build()); auto params = TParamsBuilder(paramsType); @@ -140,7 +140,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { } } if (optionalParam) { - pl.AddListItem().OptionalUint64(Nothing()); + pl.AddListItem().OptionalUint64(std::nullopt); } pl.EndList().Build(); @@ -183,7 +183,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalUint64(1) .AddListItem().OptionalUint64(2) .AddListItem().OptionalUint64(42) - .AddListItem().OptionalUint64(Nothing()) + .AddListItem().OptionalUint64(std::nullopt) .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[[3u];["Three"]]; @@ -221,7 +221,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { } } if (optionalParam) { - pl.AddListItem().OptionalString(Nothing()); + pl.AddListItem().OptionalString(std::nullopt); } pl.EndList().Build(); @@ -267,7 +267,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalString("Tony") .AddListItem().OptionalString("Hugo") .AddListItem().OptionalString("Logan") - .AddListItem().OptionalString(Nothing()) + .AddListItem().OptionalString(std::nullopt) .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[[1u];["Anna"];[3500u];["None"]]; @@ -296,7 +296,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalString("Tony") .AddListItem().OptionalString("Harry") .AddListItem().OptionalString("Hugo") - .AddListItem().OptionalString(Nothing()) + .AddListItem().OptionalString(std::nullopt) .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[[2u];["Tony"];[7200u];["None"]]; @@ -332,7 +332,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalString("Tony") .AddListItem().OptionalString("Jack") .AddListItem().OptionalString("Hugo") - .AddListItem().OptionalString(Nothing()) + .AddListItem().OptionalString(std::nullopt) .EndList().Build() .Build(); @@ -564,13 +564,13 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalInt32(1) .AddListItem().OptionalInt32(2) .AddListItem().OptionalInt32(42) - .AddListItem().OptionalInt32(Nothing()) + .AddListItem().OptionalInt32(std::nullopt) .EndList().Build() .AddParam("$in2").BeginList() .AddListItem().OptionalString("Payload1") .AddListItem().OptionalString("Payload2") .AddListItem().OptionalString("Payload0") - .AddListItem().OptionalString(Nothing()) + .AddListItem().OptionalString(std::nullopt) .EndList().Build() .Build(); @@ -634,7 +634,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalInt32(1) .AddListItem().OptionalInt32(2) .AddListItem().OptionalInt32(42) - .AddListItem().OptionalInt32(Nothing()) + .AddListItem().OptionalInt32(std::nullopt) .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])"); @@ -672,7 +672,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().OptionalInt32(1) .AddListItem().OptionalInt32(2) .AddListItem().OptionalInt32(42) - .AddListItem().OptionalInt32(Nothing()) + .AddListItem().OptionalInt32(std::nullopt) .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[[1];[1];["Payload1"]]])"); @@ -843,7 +843,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().BeginTuple().AddElement().OptionalInt32(1).AddElement().String("Fk1").EndTuple() .AddListItem().BeginTuple().AddElement().OptionalInt32(2).AddElement().String("Fk2").EndTuple() .AddListItem().BeginTuple().AddElement().OptionalInt32(42).AddElement().String("Fk5").EndTuple() - .AddListItem().BeginTuple().AddElement().OptionalInt32(Nothing()).AddElement().String("FkNull").EndTuple() + .AddListItem().BeginTuple().AddElement().OptionalInt32(std::nullopt).AddElement().String("FkNull").EndTuple() .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])"); @@ -911,7 +911,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).AddMember("v").String("Fk2").EndStruct() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).AddMember("v").String("Fk5").EndStruct() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).AddMember("v").String("FkNull").EndStruct() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(std::nullopt).AddMember("v").String("FkNull").EndStruct() .EndList().Build().Build(); auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])"); diff --git a/ydb/core/kqp/ut/perf/kqp_query_perf_ut.cpp b/ydb/core/kqp/ut/perf/kqp_query_perf_ut.cpp index d92319c3f335..b7ef8f640c64 100644 --- a/ydb/core/kqp/ut/perf/kqp_query_perf_ut.cpp +++ b/ydb/core/kqp/ut/perf/kqp_query_perf_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/ut/perf/kqp_workload_ut.cpp b/ydb/core/kqp/ut/perf/kqp_workload_ut.cpp index 24279f2841bd..9f94735d6f0f 100644 --- a/ydb/core/kqp/ut/perf/kqp_workload_ut.cpp +++ b/ydb/core/kqp/ut/perf/kqp_workload_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index d5460cd164ff..36c5dd41044b 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -2295,7 +2295,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { UNIT_ASSERT_C(resultSelect.IsSuccess(), resultSelect.GetIssues().ToString()); bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -2347,7 +2347,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { UNIT_ASSERT_C(resultSelect.IsSuccess(), resultSelect.GetIssues().ToString()); bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -3025,7 +3025,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { UNIT_ASSERT_C(resultSelect.IsSuccess(), resultSelect.GetIssues().ToString()); bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); @@ -3099,7 +3099,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { } bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); @@ -3320,7 +3320,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { } bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -3600,14 +3600,14 @@ Y_UNIT_TEST_SUITE(KqpPg) { UPDATE test SET key = key, value = 121 WHERE key = 123; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot update primary key column: key")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot update primary key column: key")); } { auto result = db.ExecuteQuery(R"( UPDATE test SET key = 12 WHERE key = 123; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot update primary key column: key")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot update primary key column: key")); } { auto result = db.ExecuteQuery(R"( @@ -3656,8 +3656,8 @@ Y_UNIT_TEST_SUITE(KqpPg) { UPDATE test SET key1 = 1, key2 = 2, value = 2 WHERE key1 = 1; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot update primary key column: key1")); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot update primary key column: key2")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot update primary key column: key1")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot update primary key column: key2")); } { kikimr.GetTestClient().CreateTable("/Root", R"( @@ -3752,7 +3752,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { SELECT * FROM test; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot find table 'db.[/Root/test]'")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot find table 'db.[/Root/test]'")); } } @@ -3804,7 +3804,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { SELECT * FROM test; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot find table 'db.[/Root/test]'")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot find table 'db.[/Root/test]'")); } } @@ -4067,7 +4067,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { SELECT * FROM test; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Cannot find table 'db.[/Root/test]'")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Cannot find table 'db.[/Root/test]'")); } } @@ -4301,21 +4301,21 @@ Y_UNIT_TEST_SUITE(KqpPg) { INSERT INTO t VALUES (1, 'a', 'a'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("values have 3 columns, INSERT INTO expects: 2")); + UNIT_ASSERT(result.GetIssues().ToString().contains("values have 3 columns, INSERT INTO expects: 2")); } { auto result = db.ExecuteQuery(R"( INSERT INTO t VALUES ('a', 1); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("invalid input syntax for type integer: \"a\"")); + UNIT_ASSERT(result.GetIssues().ToString().contains("invalid input syntax for type integer: \"a\"")); } { auto result = db.ExecuteQuery(R"( INSERT INTO nopg VALUES ('a'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Failed to convert 'id': pgunknown to Optional")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Failed to convert 'id': pgunknown to Optional")); } } @@ -4868,7 +4868,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { )"); auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("alternative is not implemented yet : 34")); + UNIT_ASSERT(result.GetIssues().ToString().contains("alternative is not implemented yet : 34")); } } @@ -4888,7 +4888,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { )"); auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unable to find an overload for operator = with given argument type(s): (text,int4)")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unable to find an overload for operator = with given argument type(s): (text,int4)")); } { @@ -4897,7 +4897,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { )"); auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unable to find an overload for operator = with given argument type(s): (text,int4)")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unable to find an overload for operator = with given argument type(s): (text,int4)")); } { @@ -4906,7 +4906,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { )"); auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("invalid input syntax for type integer: \"a\"")); + UNIT_ASSERT(result.GetIssues().ToString().contains("invalid input syntax for type integer: \"a\"")); } } @@ -4927,21 +4927,21 @@ Y_UNIT_TEST_SUITE(KqpPg) { INSERT INTO PgTable1 VALUES (1, 'a', 'a'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("values have 3 columns, INSERT INTO expects: 2")); + UNIT_ASSERT(result.GetIssues().ToString().contains("values have 3 columns, INSERT INTO expects: 2")); } { auto result = db.ExecuteQuery(R"( INSERT INTO PgTable2 VALUES ('a'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Failed to convert 'id': pgunknown to Optional")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Failed to convert 'id': pgunknown to Optional")); } { auto result = db.ExecuteQuery(R"( INSERT INTO PgTable1 VALUES ('a', 1); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("invalid input syntax for type integer: \"a\"")); + UNIT_ASSERT(result.GetIssues().ToString().contains("invalid input syntax for type integer: \"a\"")); } } @@ -5014,7 +5014,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { INSERT INTO t (id, data2) VALUES ($1, $2); )", NYdb::NQuery::TTxControl::NoTx(), params, settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("invalid byte sequence for encoding \"UTF8\": 0x00")); + UNIT_ASSERT(result.GetIssues().ToString().contains("invalid byte sequence for encoding \"UTF8\": 0x00")); } } diff --git a/ydb/core/kqp/ut/pg/pg_catalog_ut.cpp b/ydb/core/kqp/ut/pg/pg_catalog_ut.cpp index 55427a205331..76e450bab1a2 100644 --- a/ydb/core/kqp/ut/pg/pg_catalog_ut.cpp +++ b/ydb/core/kqp/ut/pg/pg_catalog_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -98,7 +98,7 @@ Y_UNIT_TEST_SUITE(PgCatalog) { )"); result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unsupported table: pgtable")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unsupported table: pgtable")); query = Q_(R"( select set_config('search_path', 'public', false); @@ -317,7 +317,7 @@ Y_UNIT_TEST_SUITE(PgCatalog) { )"); result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), settings).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Unsupported table: pgtable")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Unsupported table: pgtable")); query = Q_(R"( select set_config('search_path', 'public', false); @@ -511,7 +511,7 @@ Y_UNIT_TEST_SUITE(PgCatalog) { select tablename from pg_tables where hasindexes='pg_proc'; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("invalid input syntax for type boolean: \"pg_proc\"")); + UNIT_ASSERT(result.GetIssues().ToString().contains("invalid input syntax for type boolean: \"pg_proc\"")); } } } diff --git a/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp index 0da871fcb2b1..f737a3d42e88 100644 --- a/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include diff --git a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp index 3eefbcc91116..e66afd503af1 100644 --- a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include @@ -876,7 +876,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); auto strPlan = res.GetPlan(); - UNIT_ASSERT(strPlan); + UNIT_ASSERT(!strPlan.empty()); Cerr << strPlan << Endl; diff --git a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp index 27447ff6e637..ac78ebc00c3b 100644 --- a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp @@ -80,7 +80,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { )", NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); - UNIT_ASSERT(!to_lower(result.GetIssues().ToString()).Contains("query result")); + UNIT_ASSERT(!to_lower(TString{result.GetIssues().ToString()}).Contains("query result")); } Y_UNIT_TEST(KqpMkqlMemoryLimitException) { @@ -185,7 +185,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Mkql memory limit exceeded"), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains("Mkql memory limit exceeded"), result.GetIssues().ToString()); } Y_UNIT_TEST(ComputeActorMemoryAllocationFailureQueryService) { @@ -214,8 +214,8 @@ Y_UNIT_TEST_SUITE(KqpLimits) { auto stats = result.GetStats(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Mkql memory limit exceeded"), result.GetIssues().ToString()); - UNIT_ASSERT(stats.Defined()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains("Mkql memory limit exceeded"), result.GetIssues().ToString()); + UNIT_ASSERT(stats.has_value()); Cerr << stats->ToString(true) << Endl; } @@ -418,9 +418,9 @@ Y_UNIT_TEST_SUITE(KqpLimits) { if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNAVAILABLE, result.GetIssues().ToString()); - if (result.GetIssues().ToString().Contains("OUT_OF_SPACE")) { + if (result.GetIssues().ToString().contains("OUT_OF_SPACE")) { getOutOfSpace = true; - } else if (result.GetIssues().ToString().Contains("WRONG_SHARD_STATE")) { + } else if (result.GetIssues().ToString().contains("WRONG_SHARD_STATE")) { // shards are allowed to split continue; } @@ -544,8 +544,8 @@ Y_UNIT_TEST_SUITE(KqpLimits) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("exceeds limit"); + [] (const auto& issue) { + return issue.GetMessage().contains("exceeds limit"); })); } @@ -569,8 +569,8 @@ Y_UNIT_TEST_SUITE(KqpLimits) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("larger than the allowed threshold"); + [] (const auto& issue) { + return issue.GetMessage().contains("larger than the allowed threshold"); })); } @@ -667,8 +667,8 @@ Y_UNIT_TEST_SUITE(KqpLimits) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Memory limit exceeded"); + [] (const auto& issue) { + return issue.GetMessage().contains("Memory limit exceeded"); })); } @@ -739,7 +739,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { if (result.IsSuccess()) { auto yson = FormatResultSetYson(result.GetResultSet(0)); - CompareYson(TString("[") + expected + "]", yson); + CompareYson(TString("[") + expected + "]", TString{yson}); expected += createExpectedRow(createKey(i)); if (i != maxTimeoutMs) expected += ";"; @@ -1228,7 +1228,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { SELECT * FROM `/Root/TableTest`; )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("result size limit"), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains("result size limit"), result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(counters.GetTxReplySizeExceededError()->Val(), 1); } } @@ -1374,7 +1374,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { )", NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); - UNIT_ASSERT(!to_lower(result.GetIssues().ToString()).Contains("query result")); + UNIT_ASSERT(!to_lower(TString{result.GetIssues().ToString()}).Contains("query result")); } } diff --git a/ydb/core/kqp/ut/query/kqp_params_ut.cpp b/ydb/core/kqp/ut/query/kqp_params_ut.cpp index 454f0088210b..3d88ae41b272 100644 --- a/ydb/core/kqp/ut/query/kqp_params_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_params_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -501,7 +501,7 @@ Y_UNIT_TEST_SUITE(KqpParams) { SELECT * FROM `/Root/Test` WHERE Group = $group; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), true); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), i); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -531,7 +531,7 @@ Y_UNIT_TEST_SUITE(KqpParams) { SELECT * FROM `/Root/Test` WHERE Group = $group; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), true); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), i); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -561,7 +561,7 @@ Y_UNIT_TEST_SUITE(KqpParams) { SELECT * FROM `/Root/Test` WHERE Group = $group; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), true); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/query/kqp_query_ut.cpp b/ydb/core/kqp/ut/query/kqp_query_ut.cpp index 14750787df5f..89ffb23005fa 100644 --- a/ydb/core/kqp/ut/query/kqp_query_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_query_ut.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -192,8 +192,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { "erase_row", }; - TVector grantPermissions; - TVector revokePermissions; + std::vector grantPermissions; + std::vector revokePermissions; for (const auto& permission : allPermissions) { if (permissions.contains(permission)) { @@ -226,11 +226,11 @@ Y_UNIT_TEST_SUITE(KqpQuery) { auto session = db.CreateSession().GetValueSync().GetSession(); const auto result = params - ? session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), *params).ExtractValueSync() - : session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + ? session.ExecuteDataQuery(std::string{query}, TTxControl::BeginTx().CommitTx(), *params).ExtractValueSync() + : session.ExecuteDataQuery(std::string{query}, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), expectedStatus, result.GetIssues().ToString()); if (expectedIssue) { - UNIT_ASSERT_C(result.GetIssues().ToString().Contains(*expectedIssue), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains(*expectedIssue), result.GetIssues().ToString()); } }; @@ -633,8 +633,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { const TString query(Q_(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)")); auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("ATOM evaluation is not supported in YDB queries."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const auto& issue) { + return issue.GetMessage().contains("ATOM evaluation is not supported in YDB queries."); })); } @@ -1040,8 +1040,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { DO $hello() )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("EVALUATE IF is not supported in YDB queries."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const auto& issue) { + return issue.GetMessage().contains("EVALUATE IF is not supported in YDB queries."); })); } @@ -1052,8 +1052,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { END DO; )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("EVALUATE is not supported in YDB queries."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const auto& issue) { + return issue.GetMessage().contains("EVALUATE is not supported in YDB queries."); })); } @@ -1090,8 +1090,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("ATOM evaluation is not supported in YDB queries."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const auto& issue) { + return issue.GetMessage().contains("ATOM evaluation is not supported in YDB queries."); })); } } @@ -1107,8 +1107,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); if (result.GetStatus() == EStatus::GENERIC_ERROR) { - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Execution failed"); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const auto& issue) { + return issue.GetMessage().contains("Execution failed"); })); } else { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); @@ -1143,8 +1143,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); result.GetIssues().PrintTo(Cerr); - const auto issueChecker = [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("can't be performed in data query"); + const auto issueChecker = [](const auto& issue) { + return issue.GetMessage().contains("can't be performed in data query"); }; UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); @@ -1451,7 +1451,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT(!prepareResult.IsSuccess()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("AS VALUES statement is not supported for CreateTableAs."), + prepareResult.GetIssues().ToString().contains("AS VALUES statement is not supported for CreateTableAs."), prepareResult.GetIssues().ToString()); } @@ -1468,7 +1468,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT(!prepareResult.IsSuccess()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("Column types are not supported for CREATE TABLE AS"), + prepareResult.GetIssues().ToString().contains("Column types are not supported for CREATE TABLE AS"), prepareResult.GetIssues().ToString()); } @@ -1483,7 +1483,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT(!prepareResult.IsSuccess()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("CTAS statement can be executed only in NoTx mode."), + prepareResult.GetIssues().ToString().contains("CTAS statement can be executed only in NoTx mode."), prepareResult.GetIssues().ToString()); } @@ -1500,7 +1500,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT(!prepareResult.IsSuccess()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("CREATE TABLE AS with columns is not supported"), + prepareResult.GetIssues().ToString().contains("CREATE TABLE AS with columns is not supported"), prepareResult.GetIssues().ToString()); } } @@ -1601,7 +1601,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_C(!prepareResult.IsSuccess(), prepareResult.GetIssues().ToString()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("Creating table with data is not supported."), + prepareResult.GetIssues().ToString().contains("Creating table with data is not supported."), prepareResult.GetIssues().ToString()); } } diff --git a/ydb/core/kqp/ut/query/kqp_stats_ut.cpp b/ydb/core/kqp/ut/query/kqp_stats_ut.cpp index a19b05397aeb..5fb900800270 100644 --- a/ydb/core/kqp/ut/query/kqp_stats_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_stats_ut.cpp @@ -1,11 +1,11 @@ #include #include -#include -#include -#include +#include +#include +#include #include -#include +#include #include diff --git a/ydb/core/kqp/ut/query/kqp_types_ut.cpp b/ydb/core/kqp/ut/query/kqp_types_ut.cpp index 768e555a02d5..fb76940b58be 100644 --- a/ydb/core/kqp/ut/query/kqp_types_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_types_ut.cpp @@ -228,7 +228,7 @@ Y_UNIT_TEST_SUITE(KqpTypes) { } else { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToString().Contains("Type 'Datetime64' specified for column 'DatetimePK', but support for new date/time 64 types is disabled (EnableTableDatetime64 feature flag is off)")); + UNIT_ASSERT(result.GetIssues().ToString().contains("Type 'Datetime64' specified for column 'DatetimePK', but support for new date/time 64 types is disabled (EnableTableDatetime64 feature flag is off)")); return; } } diff --git a/ydb/core/kqp/ut/query/ya.make b/ydb/core/kqp/ut/query/ya.make index 4dc660948089..f8c5a4a9ca58 100644 --- a/ydb/core/kqp/ut/query/ya.make +++ b/ydb/core/kqp/ut/query/ya.make @@ -22,7 +22,7 @@ SRCS( PEERDIR( ydb/core/statistics/ut_common - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/proto ydb/core/kqp ydb/core/kqp/ut/common yql/essentials/sql/pg_dummy diff --git a/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp b/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp index db5dbc523927..031841a54636 100644 --- a/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp @@ -253,8 +253,8 @@ Y_UNIT_TEST_SUITE(KqpScan) { auto describeResult = session.DescribeTable("/Root/DecimalTest" , TDescribeTableSettings().WithKeyShardBoundary(true)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetStatus(), NYdb::EStatus::SUCCESS); const NYdb::NTable::TTableDescription& tableDescription = describeResult.GetTableDescription(); - const TVector& keyRanges = tableDescription.GetKeyRanges(); - const TVector& columns = tableDescription.GetTableColumns(); + const std::vector& keyRanges = tableDescription.GetKeyRanges(); + const std::vector& columns = tableDescription.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 4); UNIT_ASSERT_STRINGS_EQUAL(columns[0].Type.ToString(), "Decimal(22,9)?"); UNIT_ASSERT_STRINGS_EQUAL(columns[1].Type.ToString(), "Decimal(35,10)?"); @@ -1351,8 +1351,8 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetStatus()); part.GetIssues().PrintTo(Cerr); UNIT_ASSERT(HasIssue(part.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, - [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Requested too many execution units"); + [](const auto& issue) { + return issue.GetMessage().contains("Requested too many execution units"); })); part = it.ReadNext().GetValueSync(); @@ -1735,7 +1735,7 @@ Y_UNIT_TEST_SUITE(KqpScan) { for (auto v : {1, 2, 3, 42, 50, 100}) { pl.AddListItem().OptionalUint64(v); } - pl.AddListItem().OptionalUint64(Nothing()); + pl.AddListItem().OptionalUint64(std::nullopt); pl.EndList().Build(); auto it = db.StreamExecuteScanQuery(query, params.Build()).GetValueSync(); @@ -1792,9 +1792,9 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_C(!streamPart.IsSuccess(), streamPart.GetIssues().ToString()); UNIT_ASSERT_C(!streamPart.EOS(), streamPart.GetIssues().ToString()); UNIT_ASSERT( - HasIssue(streamPart.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Terminate was called") // general termination prefix - && issue.GetMessage().Contains("Bad filter value."); // test specific UDF exception + HasIssue(streamPart.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const auto& issue) { + return issue.GetMessage().contains("Terminate was called") // general termination prefix + && issue.GetMessage().contains("Bad filter value."); // test specific UDF exception })); }; @@ -2108,8 +2108,8 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); auto result = it.ReadNext().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("ATOM evaluation is not supported in YDB queries."); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const auto& issue) { + return issue.GetMessage().contains("ATOM evaluation is not supported in YDB queries."); })); } diff --git a/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp index 915f2de448e7..7a39a4be9c90 100644 --- a/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp @@ -1,7 +1,7 @@ #include -#include -#include +#include +#include namespace NKikimr { namespace NKqp { @@ -11,7 +11,7 @@ using namespace NYdb::NTable; const TString UserName = "user0@builtin"; -void AddPermissions(const TKikimrRunner& kikimr, const TString& path, const TString& subject, const TVector& permissionNames) { +void AddPermissions(const TKikimrRunner& kikimr, const TString& path, const TString& subject, const std::vector& permissionNames) { auto driver = NYdb::TDriver(NYdb::TDriverConfig() .SetEndpoint(kikimr.GetEndpoint()) .SetDatabase("/Root") @@ -130,7 +130,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { Cerr << result.GetIssues().ToString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); const auto expectedIssueMessage = "Cannot find table 'db.[/Root/TwoShard]' because it does not exist or you do not have access permissions."; - UNIT_ASSERT_VALUES_EQUAL(result.GetIssues().ToString().Contains(expectedIssueMessage), true); + UNIT_ASSERT_VALUES_EQUAL(result.GetIssues().ToString().contains(expectedIssueMessage), true); driver.Stop(true); } @@ -173,7 +173,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ABORTED); const auto expectedIssueMessage = "Failed to resolve table `/Root/TwoShard` status: AccessDenied."; - UNIT_ASSERT_VALUES_EQUAL(result.GetIssues().ToString().Contains(expectedIssueMessage), true); + UNIT_ASSERT_VALUES_EQUAL(result.GetIssues().ToString().contains(expectedIssueMessage), true); driver.Stop(true); } @@ -253,13 +253,13 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!result.IsSuccess(), result.GetIssues().ToString()); const auto expectedIssueMessage = "Cannot find table 'db.[/Root/test_acl]' because it does not exist or you do not have access permissions."; - UNIT_ASSERT_C(result.GetIssues().ToString().Contains(expectedIssueMessage), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains(expectedIssueMessage), result.GetIssues().ToString()); auto resultWrite = client.ExecuteQuery(R"( REPLACE INTO `/Root/test_acl` (id, name) VALUES (1, 'test'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultWrite.IsSuccess(), resultWrite.GetIssues().ToString()); - UNIT_ASSERT_C(resultWrite.GetIssues().ToString().Contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); + UNIT_ASSERT_C(resultWrite.GetIssues().ToString().contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); driver.Stop(true); } @@ -285,13 +285,13 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!result.IsSuccess(), result.GetIssues().ToString()); const auto expectedIssueMessage = "Failed to resolve table `/Root/test_acl` status: AccessDenied., code: 2028"; - UNIT_ASSERT_C(result.GetIssues().ToString().Contains(expectedIssueMessage), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains(expectedIssueMessage), result.GetIssues().ToString()); auto resultWrite = client.ExecuteQuery(R"( REPLACE INTO `/Root/test_acl` (id, name) VALUES (1, 'test'); )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultWrite.IsSuccess(), resultWrite.GetIssues().ToString()); - UNIT_ASSERT_C(resultWrite.GetIssues().ToString().Contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); + UNIT_ASSERT_C(resultWrite.GetIssues().ToString().contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); driver.Stop(true); } @@ -322,7 +322,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultWrite.IsSuccess(), resultWrite.GetIssues().ToString()); const auto expectedIssueMessage = "Failed to resolve table `/Root/test_acl` status: AccessDenied., code: 2028"; - UNIT_ASSERT_C(resultWrite.GetIssues().ToString().Contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); + UNIT_ASSERT_C(resultWrite.GetIssues().ToString().contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); driver.Stop(true); } @@ -358,7 +358,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultDelete.IsSuccess(), resultDelete.GetIssues().ToString()); const auto expectedIssueMessage = "Failed to resolve table `/Root/test_acl` status: AccessDenied., code: 2028"; - UNIT_ASSERT_C(resultDelete.GetIssues().ToString().Contains(expectedIssueMessage), resultDelete.GetIssues().ToString()); + UNIT_ASSERT_C(resultDelete.GetIssues().ToString().contains(expectedIssueMessage), resultDelete.GetIssues().ToString()); driver.Stop(true); } @@ -392,7 +392,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultWrite.IsSuccess(), resultWrite.GetIssues().ToString()); const auto expectedIssueMessage = "Failed to resolve table `/Root/test_acl` status: AccessDenied., code: 2028"; - UNIT_ASSERT_C(resultWrite.GetIssues().ToString().Contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); + UNIT_ASSERT_C(resultWrite.GetIssues().ToString().contains(expectedIssueMessage), resultWrite.GetIssues().ToString()); auto resultDelete = client.ExecuteQuery(R"( DELETE FROM `/Root/test_acl` WHERE 1=1; @@ -458,7 +458,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!result.IsSuccess(), result.GetIssues().ToString()); const auto expectedIssueMessage = "Failed to resolve table `/Root/test_acl` status: AccessDenied., code: 2028"; - UNIT_ASSERT_C(result.GetIssues().ToString().Contains(expectedIssueMessage), result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetIssues().ToString().contains(expectedIssueMessage), result.GetIssues().ToString()); auto resultWrite = client.ExecuteQuery(R"( REPLACE INTO `/Root/test_acl` (id, name) VALUES (1, 'test'); @@ -469,7 +469,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { DELETE FROM `/Root/test_acl` WHERE 1=1; )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(!resultDelete.IsSuccess(), resultDelete.GetIssues().ToString()); - UNIT_ASSERT_C(resultDelete.GetIssues().ToString().Contains(expectedIssueMessage), resultDelete.GetIssues().ToString()); + UNIT_ASSERT_C(resultDelete.GetIssues().ToString().contains(expectedIssueMessage), resultDelete.GetIssues().ToString()); driver.Stop(true); } diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp index 92d6ea0c18ac..6bbb1d42ad01 100644 --- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp @@ -3,9 +3,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 920335e49293..c833737e0020 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -7,10 +7,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include @@ -876,10 +876,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); } } @@ -1059,10 +1059,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } @@ -1072,10 +1072,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); } } @@ -1794,13 +1794,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto parser = TValueParser(val); parser.OpenTuple(); UNIT_ASSERT(parser.TryNextElement()); - return parser.GetOptionalUint64().GetRef(); + return parser.GetOptionalUint64().value(); }; - const TVector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); + const std::vector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); size_t n = 0; - const TVector expectedRanges = { 10ul, 100ul, 1000ul, 10000ul }; + const std::vector expectedRanges = { 10ul, 100ul, 1000ul, 10000ul }; for (const auto& range : keyRanges) { if (n == 0) { @@ -1860,13 +1860,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto parser = TValueParser(val); parser.OpenTuple(); UNIT_ASSERT(parser.TryNextElement()); - return parser.GetOptionalInt64().GetRef(); + return parser.GetOptionalInt64().value(); }; - const TVector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); + const std::vector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); size_t n = 0; - const TVector expectedRanges = { 0l, 10l, 10000l }; + const std::vector expectedRanges = { 0l, 10l, 10000l }; for (const auto& range : keyRanges) { if (n == 0) { @@ -1915,16 +1915,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto parser = TValueParser(val); parser.OpenTuple(); UNIT_ASSERT(parser.TryNextElement()); - ui64 pk1 = parser.GetOptionalUint64().GetRef(); + ui64 pk1 = parser.GetOptionalUint64().value(); UNIT_ASSERT(parser.TryNextElement()); auto pk2 = parser.GetOptionalString(); - return std::pair>(pk1, pk2); + return std::pair>(pk1, pk2); }; - const TVector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); + const std::vector& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); size_t n = 0; - const TVector> expectedRanges = { + const std::vector> expectedRanges = { { 10ul, "" }, { 100ul, "123" }, { 1000ul, "cde" } @@ -1939,8 +1939,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& [expectedPk1, expectedPk2] = expectedRanges[n - 1]; UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); - if (pk2.Defined()) { - UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); + if (pk2.has_value()) { + UNIT_ASSERT_VALUES_EQUAL(pk2.value(), expectedPk2); } else { UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); } @@ -1953,8 +1953,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto&[expectedPk1, expectedPk2] = expectedRanges[n]; UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); - if (pk2.Defined()) { - UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); + if (pk2.has_value()) { + UNIT_ASSERT_VALUES_EQUAL(pk2.value(), expectedPk2); } else { UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); } @@ -2146,7 +2146,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { for (const auto& family : columnFamilies) { if (family.GetName() == "Family1") { UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::None); } else { UNIT_ASSERT(family.GetName() == "default" || family.GetName() == "Family2"); } @@ -2273,11 +2273,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { for (const auto& family : columnFamilies) { if (family.GetName() == "Family1") { UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::None); } else { UNIT_ASSERT(family.GetName() == "default"); UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::LZ4); } } } @@ -2305,14 +2305,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { for (const auto& family : columnFamilies) { if (family.GetName() == "Family1") { UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::LZ4); } else if (family.GetName() == "Family2") { UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::None); } else { UNIT_ASSERT(family.GetName() == "default"); UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression().value(), EColumnFamilyCompression::LZ4); } } const auto& columns = describeResult.GetTableDescription().GetColumns(); @@ -2345,7 +2345,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); - UNIT_ASSERT(describeResult.GetTableDescription().GetStorageSettings().GetStoreExternalBlobs().GetOrElse(false)); + UNIT_ASSERT(describeResult.GetTableDescription().GetStorageSettings().GetStoreExternalBlobs().value_or(false)); } Y_UNIT_TEST(CreateAndAlterTableComplex) { @@ -2504,7 +2504,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL_C(describe.GetStatus(), EStatus::SUCCESS, describe.GetIssues().ToString()); auto tableDesc = describe.GetTableDescription(); - TVector columns = tableDesc.GetTableColumns(); + std::vector columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2); TType valueType = columns[1].Type; TTypeParser parser(valueType); @@ -2549,7 +2549,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL_C(describe.GetStatus(), EStatus::SUCCESS, describe.GetIssues().ToString()); auto tableDesc = describe.GetTableDescription(); - TVector columns = tableDesc.GetTableColumns(); + std::vector columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2); TType valueType = columns[1].Type; TTypeParser parser(valueType); @@ -3254,7 +3254,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL_C(describe.GetStatus(), EStatus::SUCCESS, describe.GetIssues().ToString()); auto tableDesc = describe.GetTableDescription(); - TVector columns = tableDesc.GetTableColumns(); + std::vector columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 7); auto checkColumn = [&] (ui64 columnIdx, ui32 precision, ui32 scale) { @@ -3318,7 +3318,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL_C(describe.GetStatus(), EStatus::SUCCESS, describe.GetIssues().ToString()); auto tableDesc = describe.GetTableDescription(); - TVector columns = tableDesc.GetTableColumns(); + std::vector columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 13); auto checkColumn = [&] (ui64 columnIdx, const TString& typeName) { @@ -4354,7 +4354,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); - TVector columns = tableDesc.GetTableColumns(); + std::vector columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); TTableColumn& familyColumn = columns[2]; UNIT_ASSERT_EQUAL(familyColumn.Name, "FAMILY"); @@ -5011,7 +5011,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& changefeeds = result.GetTableDescription().GetChangefeedDescriptions(); UNIT_ASSERT_VALUES_EQUAL(changefeeds.size(), 1); UNIT_ASSERT_VALUES_EQUAL(changefeeds.at(0), changefeed); - UNIT_ASSERT_VALUES_EQUAL(changefeeds.at(0).GetAttributes(), changefeed.GetAttributes()); + UNIT_ASSERT(changefeeds.at(0).GetAttributes() == changefeed.GetAttributes()); } } @@ -6005,19 +6005,19 @@ Y_UNIT_TEST_SUITE(KqpScheme) { for (size_t i = 0; parser.TryNextRow(); ++i) { { auto& c = parser.ColumnParser("CUint8"); - UNIT_ASSERT_VALUES_EQUAL(i, *c.GetOptionalUint8().Get()); + UNIT_ASSERT_VALUES_EQUAL(i, c.GetOptionalUint8().value()); } { auto& c = parser.ColumnParser("CInt8"); - UNIT_ASSERT_VALUES_EQUAL(i, *c.GetOptionalInt8().Get()); + UNIT_ASSERT_VALUES_EQUAL(i, c.GetOptionalInt8().value()); } { auto& c = parser.ColumnParser("CUint16"); - UNIT_ASSERT_VALUES_EQUAL(i, *c.GetOptionalUint16().Get()); + UNIT_ASSERT_VALUES_EQUAL(i, c.GetOptionalUint16().value()); } { auto& c = parser.ColumnParser("CInt16"); - UNIT_ASSERT_VALUES_EQUAL(i, *c.GetOptionalInt16().Get()); + UNIT_ASSERT_VALUES_EQUAL(i, c.GetOptionalInt16().value()); } } } diff --git a/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp b/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp index db5bfaa6f339..ed03d115035e 100644 --- a/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 0c5c0dd171d1..83ba5ca603dd 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -5,10 +5,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -44,7 +44,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { { auto result = db.GetSession().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetSession().GetId()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); id = result.GetSession().GetId(); } { @@ -68,7 +68,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { { auto result = db.GetSession().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetSession().GetId()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); auto session = result.GetSession(); id = session.GetId(); @@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr->GetEndpoint()); for (ui32 i = 0; i < sessions.size(); ++i) { bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, sessions[i].GetId(), Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{sessions[i].GetId()}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -167,7 +167,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } auto session = sessions[id]; - TMaybe tx; + std::optional tx; while (true) { if (tx) { @@ -217,7 +217,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { { auto result = db.GetSession().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetSession().GetId()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); auto session = result.GetSession(); id = session.GetId(); @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { { auto result = db.GetSession().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetSession().GetId()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); auto session = result.GetSession(); id = session.GetId(); @@ -297,7 +297,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { { auto result = db.GetSession().GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetSession().GetId()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); auto session = result.GetSession(); id = session.GetId(); } @@ -345,7 +345,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString()); sessions.push_back(sessionResult.GetSession()); - sids.push_back(sessions.back().GetId()); + sids.push_back(TString{sessions.back().GetId()}); } if (iterations & 1) { @@ -983,8 +983,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { const TString expected = R"([[["Payload2"]]])"; CompareYson(expected, FormatResultSetYson(selectRes.GetResultSet(0))); UNIT_ASSERT_C(HasIssue(selectRes.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, - [](const NYql::TIssue& issue) { - return issue.GetMessage().Contains("Given predicate is not suitable for used index: Index"); + [](const auto& issue) { + return issue.GetMessage().contains("Given predicate is not suitable for used index: Index"); }), selectRes.GetIssues().ToString()); } @@ -1226,8 +1226,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT(result.GetResultSets().empty()); - UNIT_ASSERT(result.GetStats().Defined()); - UNIT_ASSERT(result.GetStats()->GetPlan().Defined()); + UNIT_ASSERT(result.GetStats().has_value()); + UNIT_ASSERT(result.GetStats()->GetPlan().has_value()); NJson::TJsonValue plan; NJson::ReadJsonTree(*result.GetStats()->GetPlan(), &plan, true); @@ -1252,8 +1252,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSet(0).RowsCount(), 3); - UNIT_ASSERT(result.GetStats().Defined()); - UNIT_ASSERT(!result.GetStats()->GetPlan().Defined()); + UNIT_ASSERT(result.GetStats().has_value()); + UNIT_ASSERT(!result.GetStats()->GetPlan().has_value()); auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); @@ -1277,8 +1277,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSet(0).RowsCount(), 3); - UNIT_ASSERT(result.GetStats().Defined()); - UNIT_ASSERT(result.GetStats()->GetPlan().Defined()); + UNIT_ASSERT(result.GetStats().has_value()); + UNIT_ASSERT(result.GetStats()->GetPlan().has_value()); NJson::TJsonValue plan; NJson::ReadJsonTree(*result.GetStats()->GetPlan(), &plan, true); @@ -1311,8 +1311,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto result = db.ExecuteQuery(sql, TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetStats().Defined()); - UNIT_ASSERT(result.GetStats()->GetAst().Defined()); + UNIT_ASSERT(result.GetStats().has_value()); + UNIT_ASSERT(result.GetStats()->GetAst().has_value()); UNIT_ASSERT_STRING_CONTAINS(*result.GetStats()->GetAst(), "test_ast_column"); } } @@ -1697,10 +1697,10 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto resultCreateRestricted = session.ExecuteQuery(queryCreateRestricted, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT(!resultCreateRestricted.IsSuccess()); - UNIT_ASSERT_C(resultCreateRestricted.GetIssues().ToString().Contains("error: path is temporary"), resultCreateRestricted.GetIssues().ToString()); + UNIT_ASSERT_C(resultCreateRestricted.GetIssues().ToString().contains("error: path is temporary"), resultCreateRestricted.GetIssues().ToString()); bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -1872,7 +1872,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); } @@ -1947,7 +1947,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } bool allDoneOk = true; - NTestHelpers::CheckDelete(clientConfig, id, Ydb::StatusIds::SUCCESS, allDoneOk); + NTestHelpers::CheckDelete(clientConfig, TString{id}, Ydb::StatusIds::SUCCESS, allDoneOk); UNIT_ASSERT(allDoneOk); @@ -2609,7 +2609,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NOperation::TOperationClient client(kikimr.GetDriver()); TMaybe readyOp; @@ -2758,7 +2758,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { ); )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Scheme operations cannot be executed inside transaction")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Scheme operations cannot be executed inside transaction")); } { @@ -2774,7 +2774,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { ); )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Scheme operations cannot be executed inside transaction")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Scheme operations cannot be executed inside transaction")); } { @@ -2799,7 +2799,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { SELECT * FROM TestDdlDml2; )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Queries with mixed data and scheme operations are not supported.")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Queries with mixed data and scheme operations are not supported.")); } } @@ -2934,7 +2934,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { ); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl1', error: path exist")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Check failed: path: '/Root/TestDdl1', error: path exist")); result = db.ExecuteQuery(R"( CREATE TABLE TestDdl2 ( @@ -2944,7 +2944,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { ); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl2', error: path exist")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Check failed: path: '/Root/TestDdl2', error: path exist")); result = db.ExecuteQuery(R"( UPSERT INTO TestDdl2 SELECT * FROM TestDdl1; @@ -2985,7 +2985,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 0); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl2', error: path exist")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Check failed: path: '/Root/TestDdl2', error: path exist")); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl2; @@ -3007,7 +3007,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { SELECT * FROM TestDdl4; )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Cannot find table 'db.[/Root/TestDdl4]'")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Cannot find table 'db.[/Root/TestDdl4]'")); } { @@ -3059,7 +3059,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UPSERT INTO TestDdl4 (Key, Value) VALUES (3, 3); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Unknown name: $a")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Unknown name: $a")); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl4; @@ -3073,7 +3073,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UPSERT INTO TestDdl4 (Key, Value) VALUES (3, "3"); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Error: Failed to convert 'Value': String to Optional")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Error: Failed to convert 'Value': String to Optional")); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl4; @@ -3092,7 +3092,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UPSERT INTO TestDdl5 (Key, Value) VALUES (3, "3"); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Error: Failed to convert 'Value': String to Optional")); + UNIT_ASSERT(result.GetIssues().ToOneLineString().contains("Error: Failed to convert 'Value': String to Optional")); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl5; @@ -3132,7 +3132,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_C( - result.GetIssues().ToString().Contains("Several CTAS statement can't be used without per-statement mode."), + result.GetIssues().ToString().contains("Several CTAS statement can't be used without per-statement mode."), result.GetIssues().ToString()); } @@ -3146,7 +3146,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_C( - result.GetIssues().ToString().Contains("CTAS statement can't be used with other statements without per-statement mode."), + result.GetIssues().ToString().contains("CTAS statement can't be used with other statements without per-statement mode."), result.GetIssues().ToString()); } @@ -3160,7 +3160,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_C( - result.GetIssues().ToString().Contains("CTAS statement can't be used with other statements without per-statement mode."), + result.GetIssues().ToString().contains("CTAS statement can't be used with other statements without per-statement mode."), result.GetIssues().ToString()); } @@ -3972,7 +3972,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto insertResult = client.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT(!insertResult.IsSuccess()); UNIT_ASSERT_C( - insertResult.GetIssues().ToString().Contains("Write transactions between column and row tables are disabled at current time"), + insertResult.GetIssues().ToString().contains("Write transactions between column and row tables are disabled at current time"), insertResult.GetIssues().ToString()); } @@ -3985,7 +3985,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto insertResult = client.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT(!insertResult.IsSuccess()); UNIT_ASSERT_C( - insertResult.GetIssues().ToString().Contains("Write transactions between column and row tables are disabled at current time"), + insertResult.GetIssues().ToString().contains("Write transactions between column and row tables are disabled at current time"), insertResult.GetIssues().ToString()); } @@ -4000,7 +4000,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto insertResult = client.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT(!insertResult.IsSuccess()); UNIT_ASSERT_C( - insertResult.GetIssues().ToString().Contains("Write transactions between column and row tables are disabled at current time"), + insertResult.GetIssues().ToString().contains("Write transactions between column and row tables are disabled at current time"), insertResult.GetIssues().ToString()); } @@ -4014,7 +4014,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto insertResult = client.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT(!insertResult.IsSuccess()); UNIT_ASSERT_C( - insertResult.GetIssues().ToString().Contains("Write transactions between column and row tables are disabled at current time"), + insertResult.GetIssues().ToString().contains("Write transactions between column and row tables are disabled at current time"), insertResult.GetIssues().ToString()); } @@ -4028,7 +4028,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto insertResult = client.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT(!insertResult.IsSuccess()); UNIT_ASSERT_C( - insertResult.GetIssues().ToString().Contains("Write transactions between column and row tables are disabled at current time"), + insertResult.GetIssues().ToString().contains("Write transactions between column and row tables are disabled at current time"), insertResult.GetIssues().ToString()); } @@ -4396,8 +4396,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), TExecuteQuerySettings().ClientTimeout(TDuration::MilliSeconds(1000))).ExtractValueSync(); UNIT_ASSERT_C(!it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT_C( - it.GetIssues().ToString().Contains("Operation is aborting because an duplicate key") - || it.GetIssues().ToString().Contains("Conflict with existing key."), + it.GetIssues().ToString().contains("Operation is aborting because an duplicate key") + || it.GetIssues().ToString().contains("Conflict with existing key."), it.GetIssues().ToString()); } @@ -4663,7 +4663,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT(!prepareResult.IsSuccess()); UNIT_ASSERT_C( - prepareResult.GetIssues().ToString().Contains("Data manipulation queries do not support column shard tables."), + prepareResult.GetIssues().ToString().contains("Data manipulation queries do not support column shard tables."), prepareResult.GetIssues().ToString()); } diff --git a/ydb/core/kqp/ut/service/kqp_qs_scripts_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_scripts_ut.cpp index 8c284cf4981d..5eb51d075c35 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_scripts_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_scripts_ut.cpp @@ -1,9 +1,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42 )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); CheckScriptResults(scriptExecutionOperation, readyOp, db); @@ -70,7 +70,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42; SELECT 101; )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); @@ -133,11 +133,11 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { } } - void ValidatePlan(const TMaybe& plan) { + void ValidatePlan(const std::optional& plan) { UNIT_ASSERT(plan); UNIT_ASSERT(plan != "{}"); NJson::TJsonValue jsonPlan; - NJson::ReadJsonTree(plan.GetRef(), &jsonPlan, true); + NJson::ReadJsonTree(plan.value(), &jsonPlan, true); UNIT_ASSERT(ValidatePlanNodeIds(jsonPlan)); } @@ -223,7 +223,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { )", settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -255,7 +255,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { )", params).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -279,7 +279,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42 )", settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); auto readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); @@ -335,7 +335,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42 )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); ops.emplace(scriptExecutionOperation.Metadata().ExecutionId); } UNIT_ASSERT_VALUES_EQUAL(ops.size(), ScriptExecutionsCount); @@ -376,7 +376,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42 )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); ops.emplace(scriptExecutionOperation.Metadata().ExecutionId); } UNIT_ASSERT_VALUES_EQUAL(ops.size(), ScriptExecutionsCount); @@ -396,7 +396,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { UNIT_ASSERT_C(status.GetStatus() == NYdb::EStatus::SUCCESS || status.GetStatus() == NYdb::EStatus::PRECONDITION_FAILED || status.GetStatus() == NYdb::EStatus::ABORTED, status.GetIssues().ToString()); if (status.GetStatus() == NYdb::EStatus::SUCCESS) { - rememberedOps.erase(op.Metadata().ExecutionId); + rememberedOps.erase(TString{op.Metadata().ExecutionId}); } } forgetNextOperation = !forgetNextOperation; @@ -423,10 +423,10 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT * FROM TwoShard WHERE Key < 10 ORDER BY Key; )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NOperation::TOperationClient opClient(kikimr.GetDriver()); - TStatus forgetStatus = {EStatus::STATUS_UNDEFINED, NYql::TIssues()}; + TStatus forgetStatus = {EStatus::STATUS_UNDEFINED, NYdb::NIssue::TIssues()}; while (forgetStatus.GetStatus() != NYdb::EStatus::SUCCESS) { forgetStatus = opClient.Forget(scriptExecutionOperation.Id()).ExtractValueSync(); UNIT_ASSERT_C(forgetStatus.GetStatus() == NYdb::EStatus::SUCCESS || forgetStatus.GetStatus() == NYdb::EStatus::PRECONDITION_FAILED || @@ -446,7 +446,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT 42 )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NOperation::TOperationClient opClient(kikimr.GetDriver()); WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); @@ -482,7 +482,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { SELECT * FROM TwoShard WHERE Key < 10 ORDER BY Key; )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); - UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty()); NYdb::NOperation::TOperationClient opClient(kikimr.GetDriver()); std::vector cancelFutures(3); @@ -820,7 +820,7 @@ Y_UNIT_TEST_SUITE(KqpQueryServiceScripts) { auto db = kikimr.GetQueryClient(); auto scriptExecutionOperation = CreateScriptExecutionOperation(1, db, kikimr.GetDriver()); - UNIT_ASSERT_STRING_CONTAINS(scriptExecutionOperation.Metadata().ExecStats.GetAst().GetRef(), "\"idx\" (DataType 'Int32)"); + UNIT_ASSERT_STRING_CONTAINS(scriptExecutionOperation.Metadata().ExecStats.GetAst().value(), "\"idx\" (DataType 'Int32)"); } } diff --git a/ydb/core/kqp/ut/service/kqp_service_ut.cpp b/ydb/core/kqp/ut/service/kqp_service_ut.cpp index 4882a4128492..1061603e1bbc 100644 --- a/ydb/core/kqp/ut/service/kqp_service_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_service_ut.cpp @@ -118,7 +118,7 @@ Y_UNIT_TEST_SUITE(KqpService) { } auto session = sessions[id]; - TMaybe tx; + std::optional tx; while (true) { if (tx) { @@ -198,7 +198,7 @@ Y_UNIT_TEST_SUITE(KqpService) { ui32 busyResultCount = 0; auto status = db.RetryOperation([&queriesCount, &busyResultCount](TSession session) { UNIT_ASSERT(queriesCount); - UNIT_ASSERT(session.GetId()); + UNIT_ASSERT(!session.GetId().empty()); auto futures = simulateSessionBusy(queriesCount, session); @@ -213,7 +213,7 @@ Y_UNIT_TEST_SUITE(KqpService) { return NThreading::MakeFuture(result); } } - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); }).GetValueSync(); // Result should be SUCCESS in case of SESSION_BUSY UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); @@ -229,7 +229,7 @@ Y_UNIT_TEST_SUITE(KqpService) { ui32 busyResultCount = 0; auto status = db.RetryOperationSync([&queriesCount, &busyResultCount](TSession session) { UNIT_ASSERT(queriesCount); - UNIT_ASSERT(session.GetId()); + UNIT_ASSERT(!session.GetId().empty()); auto futures = simulateSessionBusy(queriesCount, session); @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(KqpService) { return (TStatus)result; } } - return TStatus(EStatus::SUCCESS, NYql::TIssues()); + return TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()); }); // Result should be SUCCESS in case of SESSION_BUSY UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/service/ya.make b/ydb/core/kqp/ut/service/ya.make index 941c834a7f28..f836d7d44ef5 100644 --- a/ydb/core/kqp/ut/service/ya.make +++ b/ydb/core/kqp/ut/service/ya.make @@ -26,8 +26,8 @@ PEERDIR( yql/essentials/sql/pg yql/essentials/parser/pg_wrapper ydb/public/lib/ut_helpers - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_types/operation + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/types/operation ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/spilling/ya.make b/ydb/core/kqp/ut/spilling/ya.make index fd87d60b7400..d5642cc57567 100644 --- a/ydb/core/kqp/ut/spilling/ya.make +++ b/ydb/core/kqp/ut/spilling/ya.make @@ -9,7 +9,7 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/proto ydb/core/kqp ydb/core/kqp/counters ydb/core/kqp/host diff --git a/ydb/core/kqp/ut/sysview/kqp_sys_view_ut.cpp b/ydb/core/kqp/ut/sysview/kqp_sys_view_ut.cpp index 2ef518150957..34ba3b2642d8 100644 --- a/ydb/core/kqp/ut/sysview/kqp_sys_view_ut.cpp +++ b/ydb/core/kqp/ut/sysview/kqp_sys_view_ut.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/sysview/ya.make b/ydb/core/kqp/ut/sysview/ya.make index a15ec0e53245..92b4422d4e23 100644 --- a/ydb/core/kqp/ut/sysview/ya.make +++ b/ydb/core/kqp/ut/sysview/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/core/kqp) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() SPLIT_FACTOR(50) diff --git a/ydb/core/kqp/ut/tx/kqp_locks_tricky_ut.cpp b/ydb/core/kqp/ut/tx/kqp_locks_tricky_ut.cpp index 6ddeee035301..4a8853f07636 100644 --- a/ydb/core/kqp/ut/tx/kqp_locks_tricky_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_locks_tricky_ut.cpp @@ -7,8 +7,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -191,7 +191,7 @@ Y_UNIT_TEST_SUITE(KqpLocksTricky) { UNIT_ASSERT(txSnaphsot.IsValid()); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT(result.GetTransaction().Defined()); + UNIT_ASSERT(result.GetTransaction().has_value()); tx.emplace(*result.GetTransaction()); // running the query that touches the main table and the index. diff --git a/ydb/core/kqp/ut/tx/kqp_locks_ut.cpp b/ydb/core/kqp/ut/tx/kqp_locks_ut.cpp index 1efde9224557..768e1d5a66e9 100644 --- a/ydb/core/kqp/ut/tx/kqp_locks_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_locks_ut.cpp @@ -38,8 +38,8 @@ Y_UNIT_TEST_SUITE(KqpLocks) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); })); result = session2.ExecuteDataQuery(Q_(R"( @@ -77,8 +77,8 @@ Y_UNIT_TEST_SUITE(KqpLocks) { UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::ABORTED, commitResult.GetIssues().ToString()); commitResult.GetIssues().PrintTo(Cerr); UNIT_ASSERT(HasIssue(commitResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); })); result = session2.ExecuteDataQuery(Q_(R"( @@ -147,8 +147,8 @@ Y_UNIT_TEST_SUITE(KqpLocks) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); })); result = session1.ExecuteDataQuery(Q1_(R"( @@ -194,8 +194,8 @@ Y_UNIT_TEST_SUITE(KqpLocks) { UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED)); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); })); result = session1.ExecuteDataQuery(Q1_(R"( @@ -236,8 +236,8 @@ Y_UNIT_TEST_SUITE(KqpLocks) { UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); commitResult.GetIssues().PrintTo(Cerr); UNIT_ASSERT_C(HasIssue(commitResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); }), commitResult.GetIssues().ToString()); } diff --git a/ydb/core/kqp/ut/tx/kqp_mvcc_ut.cpp b/ydb/core/kqp/ut/tx/kqp_mvcc_ut.cpp index c0b8fa4e89a5..e32f0803dbe2 100644 --- a/ydb/core/kqp/ut/tx/kqp_mvcc_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_mvcc_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { @@ -99,8 +99,8 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { continue; UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, - [](const NYql::TIssue& issue){ - return issue.GetMessage().Contains("has no snapshot at"); + [](const auto& issue){ + return issue.GetMessage().contains("has no snapshot at"); }), result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ABORTED); diff --git a/ydb/core/kqp/ut/tx/kqp_sink_locks_ut.cpp b/ydb/core/kqp/ut/tx/kqp_sink_locks_ut.cpp index 4d2361114b15..041c5a59c9ec 100644 --- a/ydb/core/kqp/ut/tx/kqp_sink_locks_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_sink_locks_ut.cpp @@ -44,8 +44,8 @@ Y_UNIT_TEST_SUITE(KqpSinkLocks) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); }), result.GetIssues().ToString()); result = session2.ExecuteQuery(Q_(R"( @@ -97,8 +97,8 @@ Y_UNIT_TEST_SUITE(KqpSinkLocks) { UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::ABORTED, commitResult.GetIssues().ToString()); commitResult.GetIssues().PrintTo(Cerr); UNIT_ASSERT_C(HasIssue(commitResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); }), commitResult.GetIssues().ToString()); result = session2.ExecuteQuery(Q_(R"( @@ -194,8 +194,8 @@ Y_UNIT_TEST_SUITE(KqpSinkLocks) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); }), result.GetIssues().ToString()); result = session1.ExecuteQuery(Q1_(R"( @@ -253,8 +253,8 @@ Y_UNIT_TEST_SUITE(KqpSinkLocks) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED, - [] (const NYql::TIssue& issue) { - return issue.GetMessage().Contains("/Root/Test"); + [] (const auto& issue) { + return issue.GetMessage().contains("/Root/Test"); }), result.GetIssues().ToString()); result = session1.ExecuteQuery(Q1_(R"( diff --git a/ydb/core/kqp/ut/tx/kqp_sink_mvcc_ut.cpp b/ydb/core/kqp/ut/tx/kqp_sink_mvcc_ut.cpp index 83a62f070faf..6894e6a6db5d 100644 --- a/ydb/core/kqp/ut/tx/kqp_sink_mvcc_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_sink_mvcc_ut.cpp @@ -49,8 +49,8 @@ Y_UNIT_TEST_SUITE(KqpSinkMvcc) { continue; UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, - [](const NYql::TIssue& issue){ - return issue.GetMessage().Contains("has no snapshot at"); + [](const auto& issue){ + return issue.GetMessage().contains("has no snapshot at"); }), result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ABORTED); diff --git a/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp b/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp index d9b77c2cf891..0976bbadcda3 100644 --- a/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/view/view_ut.cpp b/ydb/core/kqp/ut/view/view_ut.cpp index 510d9a9ff376..b72de39bedf3 100644 --- a/ydb/core/kqp/ut/view/view_ut.cpp +++ b/ydb/core/kqp/ut/view/view_ut.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include @@ -117,14 +117,14 @@ TMaybe GetFromCacheStat(const TQueryStats& stats) { return proto.Getcompilation().Getfrom_cache(); } -void AssertFromCache(const TMaybe& stats, bool expectedValue) { - UNIT_ASSERT(stats.Defined()); +void AssertFromCache(const std::optional& stats, bool expectedValue) { + UNIT_ASSERT(stats.has_value()); const auto isFromCache = GetFromCacheStat(*stats); UNIT_ASSERT_C(isFromCache.Defined(), stats->ToString()); UNIT_ASSERT_VALUES_EQUAL_C(*isFromCache, expectedValue, stats->ToString()); } -void CompareResults(const TVector& firstResults, const TVector& secondResults) { +void CompareResults(const std::vector& firstResults, const std::vector& secondResults) { UNIT_ASSERT_VALUES_EQUAL(firstResults.size(), secondResults.size()); for (size_t i = 0; i < firstResults.size(); ++i) { CompareYson(FormatResultSetYson(firstResults[i]), FormatResultSetYson(secondResults[i])); diff --git a/ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp b/ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp index edf5740b7ef2..0b3f2042510b 100644 --- a/ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp +++ b/ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/yql/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/yql/kqp_scripting_ut.cpp index 7c5399a8b402..9916bd8c9194 100644 --- a/ydb/core/kqp/ut/yql/kqp_scripting_ut.cpp +++ b/ydb/core/kqp/ut/yql/kqp_scripting_ut.cpp @@ -1,11 +1,11 @@ #include -#include +#include #include #include -#include +#include #include @@ -611,8 +611,8 @@ Y_UNIT_TEST_SUITE(KqpScripting) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 6); - auto stats = result.GetStats().Get(); - auto planJson = NYdb::TProtoAccessor::GetProto(*stats).query_plan(); + auto stats = result.GetStats().value(); + auto planJson = NYdb::TProtoAccessor::GetProto(stats).query_plan(); NJson::TJsonValue plan; NJson::ReadJsonTree(planJson, &plan, true); @@ -904,7 +904,7 @@ Y_UNIT_TEST_SUITE(KqpScripting) { expected += createExpectedRow(createKey(i)); if (i != maxTimeoutMs) expected += ";"; - CompareYson(TString("[") + expected + "]", yson); + CompareYson(TString("[") + expected + "]", TString{yson}); } } @@ -1112,10 +1112,10 @@ Y_UNIT_TEST_SUITE(KqpScripting) { UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets()[0].RowsCount(), 1); TResultSetParser rs0(result.GetResultSets()[0]); UNIT_ASSERT(rs0.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(*rs0.ColumnParser(0).GetOptionalUint32().Get(), 101u); + UNIT_ASSERT_VALUES_EQUAL(rs0.ColumnParser(0).GetOptionalUint32().value(), 101u); TResultSetParser rs1(result.GetResultSets()[1]); UNIT_ASSERT(rs1.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(*rs1.ColumnParser(0).GetOptionalUint32().Get(), 102u); + UNIT_ASSERT_VALUES_EQUAL(rs1.ColumnParser(0).GetOptionalUint32().value(), 102u); } Y_UNIT_TEST(StreamExecuteYqlScriptEmptyResults) { diff --git a/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp b/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp index 8b1817164013..5e2cd6240305 100644 --- a/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp +++ b/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include @@ -738,7 +738,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); for (size_t i = 0; parser.TryNextRow(); ++i) { UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("key").GetUuid().ToString(), testUuids[i]); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), i); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().value(), i); } } { @@ -752,7 +752,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); UNIT_ASSERT(parser.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), val++); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().value(), val++); UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); } } @@ -822,7 +822,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); for (size_t i = 0; parser.TryNextRow(); ++i) { UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Key").GetUuid().ToString(), testUuids[i]); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Value").GetOptionalInt32().GetRef(), i); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Value").GetOptionalInt32().value(), i); } } } @@ -877,7 +877,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); UNIT_ASSERT(parser.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), val++); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().value(), val++); UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); } } @@ -963,7 +963,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); for (size_t i = 0; parser.TryNextRow(); ++i) { UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("key").GetUuid().ToString(), testUuids[i]); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), i); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().value(), i); } } } @@ -1029,7 +1029,7 @@ Y_UNIT_TEST_SUITE(KqpYql) { TResultSetParser parser(result.GetResultSetParser(0)); UNIT_ASSERT(parser.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), val++); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().value(), val++); UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); } } diff --git a/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp b/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp index 425c8cfbd30e..761a4628f216 100644 --- a/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp +++ b/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp @@ -44,8 +44,8 @@ class TCpuLoadFetcherActor : public NKikimr::TQueryBase { return; } - ThreadsCount = result.ColumnParser("ThreadsCount").GetOptionalUint64().GetOrElse(0); - TotalLoad = result.ColumnParser("TotalLoad").GetOptionalDouble().GetOrElse(0.0); + ThreadsCount = result.ColumnParser("ThreadsCount").GetOptionalUint64().value_or(0); + TotalLoad = result.ColumnParser("TotalLoad").GetOptionalDouble().value_or(0.0); if (!ThreadsCount) { Finish(Ydb::StatusIds::NOT_FOUND, "Cpu info not found"); diff --git a/ydb/core/kqp/workload_service/actors/pool_handlers_acors.cpp b/ydb/core/kqp/workload_service/actors/pool_handlers_acors.cpp index 63fb1adbe068..fc83445d4d7e 100644 --- a/ydb/core/kqp/workload_service/actors/pool_handlers_acors.cpp +++ b/ydb/core/kqp/workload_service/actors/pool_handlers_acors.cpp @@ -409,11 +409,11 @@ class TPoolHandlerActorBase : public TActor { return LocalInFlight; } - TMaybe GetWaitDeadline(TInstant startTime) const { + std::optional GetWaitDeadline(TInstant startTime) const { if (auto cancelAfter = PoolConfig.QueryCancelAfter) { return startTime + cancelAfter; } - return Nothing(); + return std::nullopt; } TMaybe GetLoadCpuThreshold() const { diff --git a/ydb/core/kqp/workload_service/common/cpu_quota_manager.h b/ydb/core/kqp/workload_service/common/cpu_quota_manager.h index 86b5914fd630..6bf87c782251 100644 --- a/ydb/core/kqp/workload_service/common/cpu_quota_manager.h +++ b/ydb/core/kqp/workload_service/common/cpu_quota_manager.h @@ -4,7 +4,7 @@ #include -#include +#include namespace NKikimr::NKqp::NWorkload { diff --git a/ydb/core/kqp/workload_service/common/ya.make b/ydb/core/kqp/workload_service/common/ya.make index 4026b389648b..a8eb736baaaf 100644 --- a/ydb/core/kqp/workload_service/common/ya.make +++ b/ydb/core/kqp/workload_service/common/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/library/actors/core - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/src/client/types library/cpp/retry ) diff --git a/ydb/core/kqp/workload_service/tables/table_queries.cpp b/ydb/core/kqp/workload_service/tables/table_queries.cpp index 2a18cadaaa6e..e18807fb8d60 100644 --- a/ydb/core/kqp/workload_service/tables/table_queries.cpp +++ b/ydb/core/kqp/workload_service/tables/table_queries.cpp @@ -438,7 +438,7 @@ class TRefreshPoolStateQuery : public TQueryBase { class TDelayRequestQuery : public TQueryBase { public: - TDelayRequestQuery(const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, TMaybe waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters) + TDelayRequestQuery(const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, std::optional waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters) : TQueryBase(__func__, poolId, databaseId, sessionId, counters) , DatabaseId(databaseId) , PoolId(poolId) @@ -507,7 +507,7 @@ class TDelayRequestQuery : public TQueryBase { const TString PoolId; const TString SessionId; const TInstant StartTime; - const TMaybe WaitDeadline; + const std::optional WaitDeadline; const TDuration LeaseDuration; }; @@ -564,7 +564,7 @@ class TStartFirstDelayedRequestQuery : public TQueryBase { return; } - TMaybe nodeId = result.ColumnParser("node_id").GetOptionalUint32(); + std::optional nodeId = result.ColumnParser("node_id").GetOptionalUint32(); if (!nodeId) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Node id is not specified for delayed request"); return; @@ -576,7 +576,7 @@ class TStartFirstDelayedRequestQuery : public TQueryBase { return; } - TMaybe sessionId = result.ColumnParser("session_id").GetOptionalUtf8(); + std::optional sessionId = result.ColumnParser("session_id").GetOptionalUtf8(); if (!sessionId) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Session id is not specified for delayed request"); return; @@ -585,7 +585,7 @@ class TStartFirstDelayedRequestQuery : public TQueryBase { RequestSessionId = *sessionId; UpdateLogInfo(PoolId, DatabaseId, RequestSessionId); - TMaybe startTime = result.ColumnParser("start_time").GetOptionalTimestamp(); + std::optional startTime = result.ColumnParser("start_time").GetOptionalTimestamp(); if (!startTime) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Start time is not specified for delayed request"); return; @@ -855,8 +855,8 @@ IActor* CreateRefreshPoolStateActor(const TActorId& replyActorId, const TString& return new TQueryRetryActor(replyActorId, databaseId, poolId, leaseDuration, counters); } -IActor* CreateDelayRequestActor(const TActorId& replyActorId, const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, TMaybe waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters) { - return new TQueryRetryActor, TDuration, NMonitoring::TDynamicCounterPtr>(replyActorId, databaseId, poolId, sessionId, startTime, waitDeadline, leaseDuration, counters); +IActor* CreateDelayRequestActor(const TActorId& replyActorId, const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, std::optional waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters) { + return new TQueryRetryActor, TDuration, NMonitoring::TDynamicCounterPtr>(replyActorId, databaseId, poolId, sessionId, startTime, waitDeadline, leaseDuration, counters); } IActor* CreateStartRequestActor(const TActorId& replyActorId, const TString& databaseId, const TString& poolId, const std::optional& sessionId, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters) { diff --git a/ydb/core/kqp/workload_service/tables/table_queries.h b/ydb/core/kqp/workload_service/tables/table_queries.h index b7727bbff28a..599b6ba08a96 100644 --- a/ydb/core/kqp/workload_service/tables/table_queries.h +++ b/ydb/core/kqp/workload_service/tables/table_queries.h @@ -15,7 +15,7 @@ NActors::IActor* CreateCleanupTablesActor(); NActors::IActor* CreateRefreshPoolStateActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters); // Push / Start / Finish requests in pool -NActors::IActor* CreateDelayRequestActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, TMaybe waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters); +NActors::IActor* CreateDelayRequestActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const TString& sessionId, TInstant startTime, std::optional waitDeadline, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters); NActors::IActor* CreateStartRequestActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const std::optional& sessionId, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters); NActors::IActor* CreateCleanupRequestsActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const std::vector& sessionIds, NMonitoring::TDynamicCounterPtr counters); diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h index 447e345cea56..9b520fcc49e3 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h +++ b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h @@ -8,8 +8,8 @@ #include -#include -#include +#include +#include namespace NKikimr::NKqp::NWorkload { diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp index f5972e873f4e..495140866893 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp @@ -27,7 +27,7 @@ void DelayRequest(TIntrusivePtr ydb, const TString& sessionId, TDurat auto runtime = ydb->GetRuntime(); const auto& edgeActor = runtime->AllocateEdgeActor(); - runtime->Register(CreateDelayRequestActor(edgeActor, CanonizePath(settings.DomainName_), settings.PoolId_, sessionId, TInstant::Now(), Nothing(), leaseDuration, runtime->GetAppData().Counters)); + runtime->Register(CreateDelayRequestActor(edgeActor, CanonizePath(settings.DomainName_), settings.PoolId_, sessionId, TInstant::Now(), std::nullopt, leaseDuration, runtime->GetAppData().Counters)); auto response = runtime->GrabEdgeEvent(edgeActor, FUTURE_WAIT_TIMEOUT); UNIT_ASSERT_VALUES_EQUAL_C(response->Get()->Status, Ydb::StatusIds::SUCCESS, response->Get()->Issues.ToOneLineString()); UNIT_ASSERT_VALUES_EQUAL(response->Get()->SessionId, sessionId); diff --git a/ydb/core/kqp/ya.make b/ydb/core/kqp/ya.make index 484bd02f063a..df5b517a945f 100644 --- a/ydb/core/kqp/ya.make +++ b/ydb/core/kqp/ya.make @@ -43,7 +43,7 @@ PEERDIR( yql/essentials/utils/log ydb/public/api/protos ydb/public/lib/base - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/load_test/aggregated_result.cpp b/ydb/core/load_test/aggregated_result.cpp index 548a5628b442..1acbf2309c99 100644 --- a/ydb/core/load_test/aggregated_result.cpp +++ b/ydb/core/load_test/aggregated_result.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include namespace NKikimr { @@ -86,27 +86,27 @@ T ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { template<> ui32 ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { - return parser.ColumnParser(column).GetOptionalUint32().GetOrElse(0); + return parser.ColumnParser(column).GetOptionalUint32().value_or(0); } template<> ui64 ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { - return parser.ColumnParser(column).GetOptionalUint64().GetOrElse(0); + return parser.ColumnParser(column).GetOptionalUint64().value_or(0); } template<> double ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { - return parser.ColumnParser(column).GetOptionalDouble().GetOrElse(static_cast(0)); + return parser.ColumnParser(column).GetOptionalDouble().value_or(static_cast(0)); } template<> TString ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { - return parser.ColumnParser(column).GetOptionalString().GetOrElse(""); + return parser.ColumnParser(column).GetOptionalString().value_or(""); } template<> TInstant ExtractValue(NYdb::TResultSetParser& parser, const TString& column) { - return TInstant::Seconds(parser.ColumnParser(column).GetOptionalUint32().GetOrElse(0)); + return TInstant::Seconds(parser.ColumnParser(column).GetOptionalUint32().value_or(0)); } bool GetStatName(TStringBuf columnName, TStringBuf& statName, TStringBuf& suffix) { @@ -148,7 +148,7 @@ TAggregatedResult GetResultFromValueListItem(NYdb::TResultSetParser& parser, con TStringBuf suffix; TStringBuf levelSb; for (const auto& columnMeta : rs.GetColumnsMeta()) { - TString column = columnMeta.Name; + auto column = TString{columnMeta.Name}; if (column == "id") { result.Uuid = ExtractValue(parser, column); diff --git a/ydb/core/load_test/kqp.cpp b/ydb/core/load_test/kqp.cpp index 32751743434b..0fed335c4fed 100644 --- a/ydb/core/load_test/kqp.cpp +++ b/ydb/core/load_test/kqp.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/load_test/ut_ycsb/ya.make b/ydb/core/load_test/ut_ycsb/ya.make index 1b6ba35c7e6f..15663d923ecb 100644 --- a/ydb/core/load_test/ut_ycsb/ya.make +++ b/ydb/core/load_test/ut_ycsb/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/load_test/ya.make b/ydb/core/load_test/ya.make index 354d1b11a8d5..0b69b9497fb1 100644 --- a/ydb/core/load_test/ya.make +++ b/ydb/core/load_test/ya.make @@ -20,8 +20,8 @@ PEERDIR( ydb/library/workload/kv ydb/library/workload/stock ydb/public/lib/base - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/proto ydb/services/kesus ydb/services/metadata ydb/services/persqueue_cluster_discovery diff --git a/ydb/core/load_test/ycsb/kqp_select.cpp b/ydb/core/load_test/ycsb/kqp_select.cpp index 885ede6dcf13..6004cf3f209f 100644 --- a/ydb/core/load_test/ycsb/kqp_select.cpp +++ b/ydb/core/load_test/ycsb/kqp_select.cpp @@ -8,10 +8,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/ydb/core/load_test/ycsb/kqp_upsert.cpp b/ydb/core/load_test/ycsb/kqp_upsert.cpp index 22ab9c14ec89..0c7b844c91be 100644 --- a/ydb/core/load_test/ycsb/kqp_upsert.cpp +++ b/ydb/core/load_test/ycsb/kqp_upsert.cpp @@ -5,10 +5,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/ydb/core/local_pgwire/local_pgwire_connection.cpp b/ydb/core/local_pgwire/local_pgwire_connection.cpp index ef7469ced315..ef54e6b409e5 100644 --- a/ydb/core/local_pgwire/local_pgwire_connection.cpp +++ b/ydb/core/local_pgwire/local_pgwire_connection.cpp @@ -9,10 +9,10 @@ #include #include #define INCLUDE_YDB_INTERNAL_H -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include diff --git a/ydb/core/local_pgwire/local_pgwire_util.cpp b/ydb/core/local_pgwire/local_pgwire_util.cpp index 3d2c280001b7..c63d88f0ffdc 100644 --- a/ydb/core/local_pgwire/local_pgwire_util.cpp +++ b/ydb/core/local_pgwire/local_pgwire_util.cpp @@ -47,23 +47,23 @@ TString ColumnPrimitiveValueToString(NYdb::TValueParser& valueParser) { case NYdb::EPrimitiveType::Interval64: return TStringBuilder() << valueParser.GetInterval64(); case NYdb::EPrimitiveType::TzDate: - return valueParser.GetTzDate(); + return TString(valueParser.GetTzDate()); case NYdb::EPrimitiveType::TzDatetime: - return valueParser.GetTzDatetime(); + return TString(valueParser.GetTzDatetime()); case NYdb::EPrimitiveType::TzTimestamp: - return valueParser.GetTzTimestamp(); + return TString(valueParser.GetTzTimestamp()); case NYdb::EPrimitiveType::String: return Base64Encode(valueParser.GetString()); case NYdb::EPrimitiveType::Yson: - return valueParser.GetYson(); + return TString(valueParser.GetYson()); case NYdb::EPrimitiveType::Json: - return valueParser.GetJson(); + return TString(valueParser.GetJson()); case NYdb::EPrimitiveType::JsonDocument: - return valueParser.GetJsonDocument(); + return TString(valueParser.GetJsonDocument()); case NYdb::EPrimitiveType::DyNumber: - return valueParser.GetDyNumber(); + return TString(valueParser.GetDyNumber()); case NYdb::EPrimitiveType::Uuid: - return valueParser.GetUuid().ToString(); + return TString(valueParser.GetUuid().ToString()); } return {}; } @@ -133,10 +133,10 @@ NPG::TEvPGEvents::TRowValueField ColumnValueToRowValueField(NYdb::TValueParser& if (!pg.IsNull()) { switch (format) { case EFormatText: - return {.Value = pg.Content_}; + return {.Value = TString(pg.Content_)}; case EFormatBinary: { // HACK(xenoxeno) - NKikimr::NPg::TConvertResult result = NKikimr::NPg::PgNativeBinaryFromNativeText(pg.Content_, pg.PgType_.Oid); + NKikimr::NPg::TConvertResult result = NKikimr::NPg::PgNativeBinaryFromNativeText(TString{pg.Content_}, pg.PgType_.Oid); if (result.Error.Empty()) { auto begin(reinterpret_cast(result.Str.data())); auto end(begin + result.Str.size()); @@ -200,7 +200,7 @@ Ydb::TypedValue GetTypedValueFromParam(int16_t format, const std::vector res; - TMaybe queryStats; + std::optional queryStats; { Ydb::Scripting::ExecuteYqlResult result; proto.mutable_operation()->mutable_result()->UnpackTo(&result); diff --git a/ydb/core/local_pgwire/local_pgwire_util.h b/ydb/core/local_pgwire/local_pgwire_util.h index 2c1225e4c0cd..3419c7a63824 100644 --- a/ydb/core/local_pgwire/local_pgwire_util.h +++ b/ydb/core/local_pgwire/local_pgwire_util.h @@ -6,12 +6,12 @@ #include -#include +#include #define INCLUDE_YDB_INTERNAL_H -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/core/local_pgwire/pgwire_kqp_proxy.cpp b/ydb/core/local_pgwire/pgwire_kqp_proxy.cpp index 4db110b869d1..8848f8ab9ed8 100644 --- a/ydb/core/local_pgwire/pgwire_kqp_proxy.cpp +++ b/ydb/core/local_pgwire/pgwire_kqp_proxy.cpp @@ -8,7 +8,8 @@ #include #define INCLUDE_YDB_INTERNAL_H #include -#include +#include +#include #include #include @@ -209,7 +210,7 @@ class TPgwireKqpProxy : public TActorBootstrapped { static void FillError(const NKikimrKqp::TEvQueryResponse& record, typename TResponseEventPtr::element_type& response) { NYql::TIssues issues; NYql::IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues); - NYdb::TStatus status(NYdb::EStatus(record.GetYdbStatus()), std::move(issues)); + NYdb::TStatus status(NYdb::EStatus(record.GetYdbStatus()), NYdb::NAdapters::ToSdkIssues(std::move(issues))); TString message(TStringBuilder() << status); response.ErrorFields.push_back({'E', "ERROR"}); response.ErrorFields.push_back({'M', message}); @@ -231,14 +232,14 @@ class TPgwireKqpProxy : public TActorBootstrapped { std::optional pgType = GetPgTypeFromYdbType(column.Type); if (pgType.has_value()) { response->DataFields.push_back({ - .Name = column.Name, + .Name = TString{column.Name}, .DataType = pgType->Oid, .DataTypeSize = pgType->Typlen, .DataTypeModifier = pgType->Typmod, }); } else { response->DataFields.push_back({ - .Name = column.Name + .Name = TString{column.Name} }); } } diff --git a/ydb/core/local_pgwire/ya.make b/ydb/core/local_pgwire/ya.make index fac1788acc76..0e8fe5676a6b 100644 --- a/ydb/core/local_pgwire/ya.make +++ b/ydb/core/local_pgwire/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( local_pgwire_auth_actor.cpp local_pgwire_connection.cpp @@ -26,7 +30,8 @@ PEERDIR( ydb/core/pgproxy ydb/core/ydb_convert ydb/public/api/grpc - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/library/operation_id ydb/services/persqueue_v1/actors ) diff --git a/ydb/core/log_backend/ya.make b/ydb/core/log_backend/ya.make index 722619c50b40..4a61e621e966 100644 --- a/ydb/core/log_backend/ya.make +++ b/ydb/core/log_backend/ya.make @@ -12,6 +12,7 @@ PEERDIR( library/cpp/unified_agent_client library/cpp/messagebus/monitoring ydb/core/base + ydb/public/sdk/cpp/src/library/grpc/client ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index b53d1259e419..1bbe9d0b4e44 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include #include #include @@ -152,7 +154,7 @@ NActors::IEventHandle* GetAuthorizeTicketResult(const NActors::TActorId& owner) } void MakeJsonErrorReply(NJson::TJsonValue& jsonResponse, TString& message, const NYdb::TStatus& status) { - MakeJsonErrorReply(jsonResponse, message, status.GetIssues(), status.GetStatus()); + MakeJsonErrorReply(jsonResponse, message, NYdb::NAdapters::ToYqlIssues(status.GetIssues()), status.GetStatus()); } void MakeJsonErrorReply(NJson::TJsonValue& jsonResponse, TString& message, const NYql::TIssues& issues, NYdb::EStatus status) { diff --git a/ydb/core/mon/mon.h b/ydb/core/mon/mon.h index 0fca27dd820d..86c633681f2e 100644 --- a/ydb/core/mon/mon.h +++ b/ydb/core/mon/mon.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include "mon.h" diff --git a/ydb/core/mon/ya.make b/ydb/core/mon/ya.make index 9edde6bd8455..f3e1bf80f43d 100644 --- a/ydb/core/mon/ya.make +++ b/ydb/core/mon/ya.make @@ -19,7 +19,8 @@ PEERDIR( ydb/library/actors/core ydb/library/actors/http yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_types/status + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/types/status ) END() diff --git a/ydb/core/persqueue/account_read_quoter.cpp b/ydb/core/persqueue/account_read_quoter.cpp index 6c8537a242f5..a2cc401a26d1 100644 --- a/ydb/core/persqueue/account_read_quoter.cpp +++ b/ydb/core/persqueue/account_read_quoter.cpp @@ -157,7 +157,7 @@ void TBasicAccountQuoter::ApproveQuota(NAccountQuoterEvents::TEvRequest::TPtr& e } TQuoterParams TAccountReadQuoter::GetQuoterParams(const TString& user) { - auto consumerPath = NPersQueue::ConvertOldConsumerName(user); + TString consumerPath = NPersQueue::ConvertOldConsumerName(user); TQuoterParams ret; auto userParts = SplitString(consumerPath, "/"); // account/folder/topic // account is first element diff --git a/ydb/core/persqueue/actor_persqueue_client_iface.h b/ydb/core/persqueue/actor_persqueue_client_iface.h index 7e7cad3fda63..9832081c9441 100644 --- a/ydb/core/persqueue/actor_persqueue_client_iface.h +++ b/ydb/core/persqueue/actor_persqueue_client_iface.h @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp index 4d1a968c7d91..129a31ccec4f 100644 --- a/ydb/core/persqueue/cluster_tracker.cpp +++ b/ydb/core/persqueue/cluster_tracker.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include namespace NKikimr::NPQ::NClusterTracker { diff --git a/ydb/core/persqueue/codecs/pqv1.h b/ydb/core/persqueue/codecs/pqv1.h index 29fe4e8cfdaf..ba8af638b928 100644 --- a/ydb/core/persqueue/codecs/pqv1.h +++ b/ydb/core/persqueue/codecs/pqv1.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace NKikimr::NPQ { diff --git a/ydb/core/persqueue/codecs/ya.make b/ydb/core/persqueue/codecs/ya.make index fffacc38a7c5..abc623d3cf0a 100644 --- a/ydb/core/persqueue/codecs/ya.make +++ b/ydb/core/persqueue/codecs/ya.make @@ -6,7 +6,7 @@ SRCS( PEERDIR( ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_persqueue_core + ydb/public/sdk/cpp/src/client/persqueue_public ) END() diff --git a/ydb/core/persqueue/dread_cache_service/ut/ya.make b/ydb/core/persqueue/dread_cache_service/ut/ya.make index 65f623a4498e..c7acaff81f7a 100644 --- a/ydb/core/persqueue/dread_cache_service/ut/ya.make +++ b/ydb/core/persqueue/dread_cache_service/ut/ya.make @@ -7,7 +7,7 @@ SIZE(MEDIUM) PEERDIR( ydb/core/persqueue/ut/common ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index bc7e3df210db..61172dd2d420 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -21,7 +21,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V3 { class ICredentialsProviderFactory; } diff --git a/ydb/core/persqueue/mirrorer.cpp b/ydb/core/persqueue/mirrorer.cpp index 16df6fa133f8..fb61fb7d7ab1 100644 --- a/ydb/core/persqueue/mirrorer.cpp +++ b/ydb/core/persqueue/mirrorer.cpp @@ -116,7 +116,7 @@ bool TMirrorer::AddToWriteRequest( auto write = request.AddCmdWrite(); write->SetData(GetSerializedData(message)); - TString producerId = message.GetProducerId(); + TString producerId{message.GetProducerId()}; for (const auto& item : message.GetMeta()->Fields) { if (item.first == "_encoded_producer_id") { producerId = Base64Decode(item.second); @@ -516,7 +516,7 @@ void TMirrorer::TryUpdateWriteTimetsamp(const TActorContext &ctx) { } } -void TMirrorer::AddMessagesToQueue(TVector&& messages) { +void TMirrorer::AddMessagesToQueue(std::vector&& messages) { for (auto& msg : messages) { ui64 offset = msg.GetOffset(); Y_ABORT_UNLESS(OffsetToRead <= offset); @@ -597,7 +597,7 @@ void TMirrorer::DoProcessNextReaderEvent(const TActorContext& ctx, bool wakeup) if (!WaitNextReaderEventInFlight) { return; } - TMaybe event = ReadSession->GetEvent(false); + std::optional event = ReadSession->GetEvent(false); LOG_DEBUG_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got next reader event: " << bool(event)); if (wakeup && !event) { @@ -611,9 +611,9 @@ void TMirrorer::DoProcessNextReaderEvent(const TActorContext& ctx, bool wakeup) } LastReadEventTime = ctx.Now(); - if (auto* dataEvent = std::get_if(&event.GetRef())) { + if (auto* dataEvent = std::get_if(&event.value())) { AddMessagesToQueue(std::move(dataEvent->GetCompressedMessages())); - } else if (auto* createStream = std::get_if(&event.GetRef())) { + } else if (auto* createStream = std::get_if(&event.value())) { LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got create stream event for '" << createStream->DebugString() << " and will set offset=" << OffsetToRead); @@ -641,14 +641,14 @@ void TMirrorer::DoProcessNextReaderEvent(const TActorContext& ctx, bool wakeup) createStream->Confirm(OffsetToRead, createStream->GetCommittedOffset()); RequestSourcePartitionStatus(); - } else if (auto* destroyStream = std::get_if(&event.GetRef())) { + } else if (auto* destroyStream = std::get_if(&event.value())) { destroyStream->Confirm(); PartitionStream.Reset(); LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got destroy stream event: " << destroyStream->DebugString()); - } else if (auto* streamClosed = std::get_if(&event.GetRef())) { + } else if (auto* streamClosed = std::get_if(&event.value())) { PartitionStream.Reset(); LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() @@ -660,7 +660,7 @@ void TMirrorer::DoProcessNextReaderEvent(const TActorContext& ctx, bool wakeup) ScheduleConsumerCreation(ctx); return; - } else if (auto* streamStatus = std::get_if(&event.GetRef())) { + } else if (auto* streamStatus = std::get_if(&event.value())) { if (PartitionStream && PartitionStream->GetPartitionSessionId() == streamStatus->GetPartitionSession()->GetPartitionSessionId() ) { @@ -669,15 +669,15 @@ void TMirrorer::DoProcessNextReaderEvent(const TActorContext& ctx, bool wakeup) ctx.Schedule(TDuration::Seconds(1), new TEvPQ::TEvRequestPartitionStatus); TryUpdateWriteTimetsamp(ctx); } - } else if (auto* commitAck = std::get_if(&event.GetRef())) { + } else if (auto* commitAck = std::get_if(&event.value())) { LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got commit responce, commited offset: " << commitAck->GetCommittedOffset()); - } else if (auto* closeSessionEvent = std::get_if(&event.GetRef())) { + } else if (auto* closeSessionEvent = std::get_if(&event.value())) { ProcessError(ctx, TStringBuilder() << " read session closed: " << closeSessionEvent->DebugString()); ScheduleConsumerCreation(ctx); return; } else { - ProcessError(ctx, TStringBuilder() << " got unmatched event: " << event.GetRef().index()); + ProcessError(ctx, TStringBuilder() << " got unmatched event: " << event.value().index()); ScheduleConsumerCreation(ctx); return; } diff --git a/ydb/core/persqueue/mirrorer.h b/ydb/core/persqueue/mirrorer.h index db84659f6298..d6517ffecd1e 100644 --- a/ydb/core/persqueue/mirrorer.h +++ b/ydb/core/persqueue/mirrorer.h @@ -11,8 +11,8 @@ #include #include #include -#include -#include +#include +#include namespace NKikimr { @@ -146,7 +146,7 @@ class TMirrorer : public TActorBootstrapped { void RequestSourcePartitionStatus(); void TryUpdateWriteTimetsamp(const TActorContext &ctx); void AddMessagesToQueue( - TVector&& messages + std::vector&& messages ); void StartWaitNextReaderEvent(const TActorContext& ctx); diff --git a/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp b/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp index 0f7cb25e20a9..dc877d041c9e 100644 --- a/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp +++ b/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp @@ -63,7 +63,7 @@ ui64 SplitPartition(TTopicSdkTestSetup& setup, ui64& txId, const ui32 partition, } ui64 SplitPartition(NActors::TTestActorRuntime& runtime, ui64& txId, const ui32 partition, TString boundary) { - return SplitPartition(runtime, txId, TEST_TOPIC, partition, boundary); + return SplitPartition(runtime, txId, TString{TEST_TOPIC}, partition, boundary); } ui64 SplitPartition(NActors::TTestActorRuntime& runtime, ui64& txId, const TString& topic, const ui32 partition, TString boundary) { @@ -78,7 +78,7 @@ ui64 SplitPartition(NActors::TTestActorRuntime& runtime, ui64& txId, const TStri void MergePartition(TTopicSdkTestSetup& setup, ui64& txId, const ui32 partitionLeft, const ui32 partitionRight) { ::NKikimrSchemeOp::TPersQueueGroupDescription scheme; - scheme.SetName(TEST_TOPIC); + scheme.SetName(TString{TEST_TOPIC}); auto* merge = scheme.AddMerge(); merge->SetPartition(partitionLeft); merge->SetAdjacentPartition(partitionRight); @@ -258,7 +258,7 @@ std::shared_ptr::TSdkReadSession> TTestReadS auto msg = MsgInfo(message.GetPartitionSession()->GetPartitionId(), message.GetSeqNo(), message.GetOffset(), - message.GetData(), + TString{message.GetData()}, impl->AutoCommit) .WithMsg(new MsgWrapper(message)); @@ -281,10 +281,10 @@ std::shared_ptr::TSdkReadSession> TTestReadS auto partitionId = ev.GetPartitionSession()->GetPartitionId(); auto topic = ev.GetPartitionSession()->GetTopicPath(); auto offset = impl->GetOffset(partitionId); - impl->Modify([&](std::unordered_map>& s) { s[topic].insert(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].insert(partitionId); }); if (offset) { Cerr << ">>>>> " << impl->Name << " Start reading partition " << partitionId << " from offset " << offset.value() << Endl << Flush; - ev.Confirm(offset.value(), TMaybe()); + ev.Confirm(offset.value(), std::optional()); } else { Cerr << ">>>>> " << impl->Name << " Start reading partition " << partitionId << " without offset" << Endl << Flush; ev.Confirm(); @@ -297,7 +297,7 @@ std::shared_ptr::TSdkReadSession> TTestReadS Cerr << ">>>>> " << impl->Name << " Received TStopPartitionSessionEvent message " << ev.DebugString() << Endl << Flush; auto partitionId = ev.GetPartitionSession()->GetPartitionId(); auto topic = ev.GetPartitionSession()->GetTopicPath(); - impl->Modify([&](std::unordered_map>& s) { s[topic].erase(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].erase(partitionId); }); Cerr << ">>>>> " << impl->Name << " Stop reading partition " << partitionId << Endl << Flush; ev.Confirm(); }); @@ -308,7 +308,7 @@ std::shared_ptr::TSdkReadSession> TTestReadS Cerr << ">>>>> " << impl->Name << " Received TPartitionSessionClosedEvent message " << ev.DebugString() << Endl << Flush; auto partitionId = ev.GetPartitionSession()->GetPartitionId(); auto topic = ev.GetPartitionSession()->GetTopicPath(); - impl->Modify([&](std::unordered_map>& s) { s[topic].erase(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].erase(partitionId); }); Cerr << ">>>>> " << impl->Name << " Stop (closed) reading partition " << partitionId << Endl << Flush; }); @@ -377,7 +377,7 @@ std::shared_ptr::TSdkReadSession> TTestReadSe auto msg = MsgInfo(message.GetPartitionStream()->GetPartitionId(), message.GetSeqNo(), message.GetOffset(), - message.GetData(), + TString{message.GetData()}, impl->AutoCommit) .WithMsg(new MsgWrapper(message)); @@ -400,10 +400,10 @@ std::shared_ptr::TSdkReadSession> TTestReadSe auto partitionId = ev.GetPartitionStream()->GetPartitionId(); auto topic = ev.GetPartitionStream()->GetTopicPath(); auto offset = impl->GetOffset(partitionId); - impl->Modify([&](std::unordered_map>& s) { s[topic].insert(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].insert(partitionId); }); if (offset) { Cerr << ">>>>> " << impl->Name << " Start reading partition " << partitionId << " from offset " << offset.value() << Endl << Flush; - ev.Confirm(offset.value(), TMaybe()); + ev.Confirm(offset.value(), std::optional()); } else { Cerr << ">>>>> " << impl->Name << " Start reading partition " << partitionId << " without offset" << Endl << Flush; ev.Confirm(); @@ -416,7 +416,7 @@ std::shared_ptr::TSdkReadSession> TTestReadSe Cerr << ">>>>> " << impl->Name << " Received TDestroyPartitionStreamEvent message " << ev.DebugString() << Endl << Flush; auto partitionId = ev.GetPartitionStream()->GetPartitionId(); auto topic = ev.GetPartitionStream()->GetTopicPath(); - impl->Modify([&](std::unordered_map>& s) { s[topic].erase(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].erase(partitionId); }); Cerr << ">>>>> " << impl->Name << " Stop reading partition " << partitionId << Endl << Flush; ev.Confirm(); }); @@ -427,7 +427,7 @@ std::shared_ptr::TSdkReadSession> TTestReadSe Cerr << ">>>>> " << impl->Name << " Received TPartitionSessionClosedEvent message " << ev.DebugString() << Endl << Flush; auto partitionId = ev.GetPartitionStream()->GetPartitionId(); auto topic = ev.GetPartitionStream()->GetTopicPath(); - impl->Modify([&](std::unordered_map>& s) { s[topic].erase(partitionId); }); + impl->Modify([&](std::unordered_map>& s) { s[TString{topic}].erase(partitionId); }); Cerr << ">>>>> " << impl->Name << " Stop (closed) reading partition " << partitionId << Endl << Flush; }); @@ -490,7 +490,7 @@ NThreading::TFuture>> TTestReadSess template void TTestReadSession::Assert(const std::set& expected, NThreading::TFuture>> f, const TString& message) { - Assert(std::unordered_map>{{TEST_TOPIC, expected}}, f, message); + Assert(std::unordered_map>{{TString{TEST_TOPIC}, expected}}, f, message); } template @@ -503,7 +503,7 @@ void TTestReadSession::Assert(const std::unordered_map void TTestReadSession::WaitAndAssertPartitions(std::set partitions, const TString& message) { - WaitAndAssert(std::unordered_map>{{TEST_TOPIC, partitions}}, message); + WaitAndAssert(std::unordered_map>{{TString{TEST_TOPIC}, partitions}}, message); } template @@ -530,7 +530,7 @@ void TTestReadSession::Close() { template std::set TTestReadSession::GetPartitions() { - return GetPartitionsA()[TEST_TOPIC]; + return GetPartitionsA()[TString{TEST_TOPIC}]; } template diff --git a/ydb/core/persqueue/ut/common/autoscaling_ut_common.h b/ydb/core/persqueue/ut/common/autoscaling_ut_common.h index dfa557e4a723..f841011dd669 100644 --- a/ydb/core/persqueue/ut/common/autoscaling_ut_common.h +++ b/ydb/core/persqueue/ut/common/autoscaling_ut_common.h @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -46,7 +46,7 @@ TWriteMessage Msg(const TString& data, ui64 seqNo); TTopicSdkTestSetup CreateSetup(); -std::shared_ptr CreateWriteSession(TTopicClient& client, const TString& producer, std::optional partition = std::nullopt, TString topic = TEST_TOPIC, bool useCodec = true); +std::shared_ptr CreateWriteSession(TTopicClient& client, const TString& producer, std::optional partition = std::nullopt, TString topic = TString{TEST_TOPIC}, bool useCodec = true); enum class SdkVersion { Topic, @@ -98,8 +98,8 @@ struct TestReadSessionSettings { bool AutoCommit = true; std::set Partitions = {}; bool AutoPartitioningSupport = true; - std::vector Topics = {TEST_TOPIC}; - TMaybe ReadLag; + std::vector Topics = {TEST_TOPIC}; + std::optional ReadLag; }; struct ITestReadSession { diff --git a/ydb/core/persqueue/ut/common/ya.make b/ydb/core/persqueue/ut/common/ya.make index 6bb59b970ebe..8fe63c02377e 100644 --- a/ydb/core/persqueue/ut/common/ya.make +++ b/ydb/core/persqueue/ut/common/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( pq_ut_common.cpp pq_ut_common.h diff --git a/ydb/core/persqueue/ut/fetch_request_ut.cpp b/ydb/core/persqueue/ut/fetch_request_ut.cpp index 3666b78c9035..ff64290908a7 100644 --- a/ydb/core/persqueue/ut/fetch_request_ut.cpp +++ b/ydb/core/persqueue/ut/fetch_request_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/ydb/core/persqueue/ut/partition_chooser_ut.cpp b/ydb/core/persqueue/ut/partition_chooser_ut.cpp index 12ae4665739d..45bd3544f9cb 100644 --- a/ydb/core/persqueue/ut/partition_chooser_ut.cpp +++ b/ydb/core/persqueue/ut/partition_chooser_ut.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include @@ -342,8 +342,8 @@ void AssertTable(NPersQueue::TTestServer& server, const TString& sourceId, ui32 UNIT_ASSERT(parser.TryNextRow()); NYdb::TValueParser p(parser.GetValue(0)); NYdb::TValueParser s(parser.GetValue(1)); - UNIT_ASSERT_VALUES_EQUAL(*p.GetOptionalUint32().Get(), partitionId); - UNIT_ASSERT_VALUES_EQUAL(*s.GetOptionalUint64().Get(), seqNo); + UNIT_ASSERT_VALUES_EQUAL(p.GetOptionalUint32().value(), partitionId); + UNIT_ASSERT_VALUES_EQUAL(s.GetOptionalUint64().value(), seqNo); } class TPQTabletMock: public TActor { diff --git a/ydb/core/persqueue/ut/pqrb_describes_ut.cpp b/ydb/core/persqueue/ut/pqrb_describes_ut.cpp index 956b00083dd5..46525d8635e9 100644 --- a/ydb/core/persqueue/ut/pqrb_describes_ut.cpp +++ b/ydb/core/persqueue/ut/pqrb_describes_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include namespace NKikimr::NPQ { diff --git a/ydb/core/persqueue/ut/slow/autopartitioning_ut.cpp b/ydb/core/persqueue/ut/slow/autopartitioning_ut.cpp index 9d6ee9de31c8..df26358e7356 100644 --- a/ydb/core/persqueue/ut/slow/autopartitioning_ut.cpp +++ b/ydb/core/persqueue/ut/slow/autopartitioning_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/ydb/core/persqueue/ut/slow/ya.make b/ydb/core/persqueue/ut/slow/ya.make index 6225dc9a1c27..9553b8d825b3 100644 --- a/ydb/core/persqueue/ut/slow/ya.make +++ b/ydb/core/persqueue/ut/slow/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/core/persqueue) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() SPLIT_FACTOR(5) @@ -11,9 +15,8 @@ PEERDIR( library/cpp/svnversion ydb/core/persqueue/ut/common ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/topic/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers ) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index f16c2f9bd428..b39ad0c0c7fb 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -175,7 +175,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionSplit_oldSDK(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -228,7 +228,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(PartitionSplit_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -273,7 +273,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionSplit_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -354,7 +354,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionMerge_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 2, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 2, 100); TTopicClient client = setup.MakeClient(); @@ -426,7 +426,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionSplit_ReadEmptyPartitions(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoPartitioningSupport = autoscaleAwareSDK }); @@ -455,7 +455,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionSplit_ReadNotEmptyPartitions(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .AutoPartitioningSupport = false }); @@ -490,7 +490,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); @@ -511,7 +511,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { void PartitionSplit_ManySession(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -551,7 +551,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(PartitionSplit_ManySession_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -596,7 +596,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(PartitionSplit_ManySession_existed_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(CommitTopPast_BeforeAutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); TTopicClient client = setup.MakeClient(); @@ -889,8 +889,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto msg = TString(1_MB, 'a'); - auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); - auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); @@ -910,8 +910,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } - auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); - auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, std::string{TEST_TOPIC}, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); @@ -1157,7 +1157,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(BalancingAfterSplit_sessionsWithPartition) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopicWithAutoscale(TString{TEST_TOPIC}, TEST_CONSUMER, 1, 100); TTopicClient client = setup.MakeClient(); @@ -1197,7 +1197,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { Y_UNIT_TEST(ReBalancingAfterSplit_sessionsWithPartition) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(TEST_TOPIC, TEST_CONSUMER, 2, 100); + setup.CreateTopicWithAutoscale(TString{TEST_TOPIC}, TEST_CONSUMER, 2, 100); TTopicClient client = setup.MakeClient(); diff --git a/ydb/core/persqueue/ut/ut_with_sdk/balancing_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/balancing_ut.cpp index 21ac5ef12568..4533561e07ad 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/balancing_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/balancing_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -20,7 +20,7 @@ Y_UNIT_TEST_SUITE(Balancing) { void Simple(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopic(TEST_TOPIC, TEST_CONSUMER, 10); + setup.CreateTopic(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 10); auto readSession0 = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk }); { @@ -116,8 +116,8 @@ Y_UNIT_TEST_SUITE(Balancing) { void ManyTopics(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopic(TEST_TOPIC, TEST_CONSUMER, 10); - setup.CreateTopic("other-test-topic", TEST_CONSUMER, 10); + setup.CreateTopic(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 10); + setup.CreateTopic("other-test-topic", std::string{TEST_CONSUMER}, 10); TTopicClient client = setup.MakeClient(); @@ -126,7 +126,7 @@ Y_UNIT_TEST_SUITE(Balancing) { { auto p = readSession0->GetPartitionsA(); - UNIT_ASSERT_VALUES_EQUAL(10, p[TEST_TOPIC].size()); + UNIT_ASSERT_VALUES_EQUAL(10, p[std::string{TEST_TOPIC}].size()); UNIT_ASSERT_VALUES_EQUAL(10, p["other-test-topic"].size()); } @@ -135,12 +135,12 @@ Y_UNIT_TEST_SUITE(Balancing) { { auto p = readSession0->GetPartitionsA(); - UNIT_ASSERT_VALUES_EQUAL(5, p[TEST_TOPIC].size()); + UNIT_ASSERT_VALUES_EQUAL(5, p[std::string{TEST_TOPIC}].size()); UNIT_ASSERT_VALUES_EQUAL(5, p["other-test-topic"].size()); } { auto p = readSession1->GetPartitionsA(); - UNIT_ASSERT_VALUES_EQUAL(5, p[TEST_TOPIC].size()); + UNIT_ASSERT_VALUES_EQUAL(5, p[std::string{TEST_TOPIC}].size()); UNIT_ASSERT_VALUES_EQUAL(5, p["other-test-topic"].size()); } diff --git a/ydb/core/persqueue/ut/ut_with_sdk/mirrorer_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/mirrorer_ut.cpp index 4fc68015e6b4..8d728e1b3e44 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/mirrorer_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/mirrorer_ut.cpp @@ -1,8 +1,9 @@ #include "actor_persqueue_client_iface.h" -#include -#include +#include +#include +#include #include @@ -161,17 +162,17 @@ Y_UNIT_TEST_SUITE(TPersQueueMirrorer) { TVector messagesPerPartition(partitionsCount, 0); for (ui32 partition = 0; partition < partitionsCount; ++partition) { TString sourceId = "some_sourceid_" + ToString(partition); - THashMap sessionMeta = { + std::unordered_map sessionMeta = { {"some_extra_field", "some_value"}, - {"some_extra_field2", "another_value" + ToString(partition)}, - {"file", "/home/user/log" + ToString(partition)} + {"some_extra_field2", "another_value" + std::to_string(partition)}, + {"file", "/home/user/log" + std::to_string(partition)} }; auto writer = CreateSimpleWriter(*driver, srcTopic, sourceId, partition + 1, std::nullopt, std::nullopt, sessionMeta); ui64 seqNo = writer->GetInitSeqNo(); for (ui32 i = 1; i <= 11; ++i) { - auto res = writer->Write(TString(i, 'a'), ++seqNo); + auto res = writer->Write(std::string(i, 'a'), ++seqNo); UNIT_ASSERT(res); ++messagesPerPartition[partition]; } diff --git a/ydb/core/persqueue/ut/ut_with_sdk/ya.make b/ydb/core/persqueue/ut/ut_with_sdk/ya.make index abc7c629d914..fdb6c2ff4054 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/ya.make +++ b/ydb/core/persqueue/ut/ut_with_sdk/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/core/persqueue) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) @@ -15,9 +19,9 @@ PEERDIR( library/cpp/svnversion ydb/core/persqueue/ut/common ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/topic/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers ) diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index 44ae44688b15..239480203ae2 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/core/persqueue) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() SPLIT_FACTOR(40) @@ -17,8 +21,8 @@ PEERDIR( library/cpp/svnversion ydb/core/persqueue/ut/common ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers ) diff --git a/ydb/core/persqueue/write_meta.cpp b/ydb/core/persqueue/write_meta.cpp index 73c5d13fd3e7..53ab62ab5a20 100644 --- a/ydb/core/persqueue/write_meta.cpp +++ b/ydb/core/persqueue/write_meta.cpp @@ -25,15 +25,15 @@ void SetMetaField(NKikimrPQClient::TDataChunk& proto, const TString& key, const TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message) { NKikimrPQClient::TDataChunk proto; for (const auto& item : message.GetMeta(0)->Fields) { - SetMetaField(proto, item.first, item.second); + SetMetaField(proto, TString{item.first}, TString{item.second}); } - proto.SetIp(message.GetIp(0)); + proto.SetIp(TString{message.GetIp(0)}); proto.SetSeqNo(message.GetSeqNo(0)); proto.SetCreateTime(message.GetCreateTime(0).MilliSeconds()); auto codec = NPQ::FromV1Codec(message.GetCodec()); Y_ABORT_UNLESS(codec); proto.SetCodec(codec.value()); - proto.SetData(message.GetData()); + proto.SetData(TString{message.GetData()}); TString str; bool res = proto.SerializeToString(&str); @@ -45,26 +45,26 @@ TString GetSerializedData(const NYdb::NTopic::TReadSessionEvent::TDataReceivedEv NKikimrPQClient::TDataChunk proto; for (const auto& item : message.GetMeta()->Fields) { if (item.first == "_ip") { - proto.SetIp(item.second); + proto.SetIp(TString{item.second}); } else if (item.first == "_encoded_producer_id") { // Skip. } else { - SetMetaField(proto, item.first, item.second); + SetMetaField(proto, TString{item.first}, TString{item.second}); } } auto& fields = message.GetMessageMeta()->Fields; if (!fields.empty()) { for (const auto& item : fields) { auto& metaItem = *proto.AddMessageMeta(); - metaItem.set_key(item.first); - metaItem.set_value(item.second); + metaItem.set_key(TString{item.first}); + metaItem.set_value(TString{item.second}); } } proto.SetSeqNo(message.GetSeqNo()); proto.SetCreateTime(message.GetCreateTime().MilliSeconds()); auto codec = NPQ::FromTopicCodec(message.GetCodec()); proto.SetCodec(codec); - proto.SetData(message.GetData()); + proto.SetData(TString{message.GetData()}); TString str; bool res = proto.SerializeToString(&str); diff --git a/ydb/core/persqueue/write_meta.h b/ydb/core/persqueue/write_meta.h index 2c3c92f5a3b5..5c379efbeada 100644 --- a/ydb/core/persqueue/write_meta.h +++ b/ydb/core/persqueue/write_meta.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include diff --git a/ydb/core/persqueue/writer/partition_chooser_impl__partition_helper.h b/ydb/core/persqueue/writer/partition_chooser_impl__partition_helper.h index d790c3024413..379b4985f260 100644 --- a/ydb/core/persqueue/writer/partition_chooser_impl__partition_helper.h +++ b/ydb/core/persqueue/writer/partition_chooser_impl__partition_helper.h @@ -6,7 +6,7 @@ #include #include -#include +#include namespace NKikimr::NPQ::NPartitionChooser { diff --git a/ydb/core/persqueue/writer/partition_chooser_impl__table_helper.h b/ydb/core/persqueue/writer/partition_chooser_impl__table_helper.h index a06dcfcca605..ef8144907a61 100644 --- a/ydb/core/persqueue/writer/partition_chooser_impl__table_helper.h +++ b/ydb/core/persqueue/writer/partition_chooser_impl__table_helper.h @@ -7,10 +7,10 @@ #include #include #include -#include +#include #include -#include +#include namespace NKikimr::NPQ::NPartitionChooser { @@ -176,13 +176,13 @@ class TTableHelper { while(parser.TryNextRow()) { auto tt = parser.ColumnParser(0).GetOptionalUint32(); - if (tt.Defined()) { //already got partition - auto accessTime = parser.ColumnParser(2).GetOptionalUint64().GetOrElse(0); + if (tt.has_value()) { //already got partition + auto accessTime = parser.ColumnParser(2).GetOptionalUint64().value_or(0); if (accessTime > AccessTime) { // AccessTime PartitionId_ = *tt; - CreateTime = parser.ColumnParser(1).GetOptionalUint64().GetOrElse(0); + CreateTime = parser.ColumnParser(1).GetOptionalUint64().value_or(0); AccessTime = accessTime; - SeqNo_ = parser.ColumnParser(3).GetOptionalUint64().GetOrElse(0); + SeqNo_ = parser.ColumnParser(3).GetOptionalUint64().value_or(0); } } } diff --git a/ydb/core/persqueue/writer/source_id_encoding.h b/ydb/core/persqueue/writer/source_id_encoding.h index 637107505a4c..d2e699b7eb3d 100644 --- a/ydb/core/persqueue/writer/source_id_encoding.h +++ b/ydb/core/persqueue/writer/source_id_encoding.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/ydb/core/persqueue/writer/ya.make b/ydb/core/persqueue/writer/ya.make index c62a1415c932..6f1aa8ce9d87 100644 --- a/ydb/core/persqueue/writer/ya.make +++ b/ydb/core/persqueue/writer/ya.make @@ -19,7 +19,7 @@ PEERDIR( ydb/library/wilson_ids ydb/public/lib/base ydb/public/lib/deprecated/kicli - ydb/public/sdk/cpp/client/ydb_params + ydb/public/sdk/cpp/src/client/params ) END() diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 90ec81e4e109..3d33151c7557 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -74,7 +74,7 @@ PEERDIR( ydb/library/persqueue/topic_parser ydb/library/protobuf_printer ydb/public/lib/base - ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public ) END() diff --git a/ydb/core/pgproxy/pg_proxy_events.h b/ydb/core/pgproxy/pg_proxy_events.h index e0c174f95f80..55296f338c16 100644 --- a/ydb/core/pgproxy/pg_proxy_events.h +++ b/ydb/core/pgproxy/pg_proxy_events.h @@ -36,7 +36,7 @@ struct TEvPGEvents { static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PGWIRE), "ES_PGWIRE event space is too small."); struct TRowDescriptionField { - TString Name; + std::string Name; uint32_t TableId = 0; uint16_t ColumnId = 0; uint32_t DataType; diff --git a/ydb/core/public_http/http_req.cpp b/ydb/core/public_http/http_req.cpp index a2a07139e9ec..9378e8a2391c 100644 --- a/ydb/core/public_http/http_req.cpp +++ b/ydb/core/public_http/http_req.cpp @@ -1,7 +1,7 @@ #include "http_req.h" #include -#include +#include #include #include diff --git a/ydb/core/public_http/ya.make b/ydb/core/public_http/ya.make index ea9593430746..5fb96ad26cf7 100644 --- a/ydb/core/public_http/ya.make +++ b/ydb/core/public_http/ya.make @@ -24,8 +24,7 @@ PEERDIR( ydb/core/public_http/protos ydb/core/viewer/json yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_types - + ydb/public/sdk/cpp/src/client/types ) RESOURCE( diff --git a/ydb/core/security/ticket_parser_impl.h b/ydb/core/security/ticket_parser_impl.h index 330e091e84fe..91c7fd55d7dd 100644 --- a/ydb/core/security/ticket_parser_impl.h +++ b/ydb/core/security/ticket_parser_impl.h @@ -1013,7 +1013,7 @@ class TTicketParserImpl : public TActorBootstrapped { } else { if (record.ResponsesLeft == 0 && (record.TokenType == TDerived::ETokenType::Unknown || record.TokenType == TDerived::ETokenType::AccessService || record.TokenType == TDerived::ETokenType::NebiusAccessService || record.TokenType == TDerived::ETokenType::ApiKey)) { bool retryable = IsRetryableGrpcError(response->Status); - SetError(key, record, {.Message = response->Status.Msg, .Retryable = retryable}); + SetError(key, record, {.Message = TString{response->Status.Msg}, .Retryable = retryable}); } } if (record.ResponsesLeft == 0) { @@ -1042,7 +1042,7 @@ class TTicketParserImpl : public TActorBootstrapped { auto& record = it->second; record.ResponsesLeft--; if (!ev->Get()->Status.Ok()) { - SetError(key, record, {.Message = ev->Get()->Status.Msg}); + SetError(key, record, {.Message = TString{ev->Get()->Status.Msg}}); } else { GetDerived()->SetToken(key, record, ev); } @@ -1064,7 +1064,7 @@ class TTicketParserImpl : public TActorBootstrapped { auto& record = it->second; record.ResponsesLeft--; if (!ev->Get()->Status.Ok()) { - SetError(key, record, {.Message = ev->Get()->Status.Msg}); + SetError(key, record, {.Message = TString{ev->Get()->Status.Msg}}); } else { SetToken(key, record, new NACLib::TUserToken(record.Ticket, ev->Get()->Response.name() + "@" + ServiceDomain, {})); } @@ -1232,7 +1232,7 @@ class TTicketParserImpl : public TActorBootstrapped { } } } else { - SetAccessServiceBulkAuthorizeError(key, record, response->Status.Msg, IsRetryableGrpcError(response->Status)); + SetAccessServiceBulkAuthorizeError(key, record, TString{response->Status.Msg}, IsRetryableGrpcError(response->Status)); } if (record.ResponsesLeft == 0) { Respond(record); @@ -1313,7 +1313,7 @@ class TTicketParserImpl : public TActorBootstrapped { } } } else { - SetAccessServiceBulkAuthorizeError(key, record, response->Status.Msg, IsRetryableGrpcError(response->Status)); + SetAccessServiceBulkAuthorizeError(key, record, TString{response->Status.Msg}, IsRetryableGrpcError(response->Status)); } Respond(record); } @@ -1353,7 +1353,7 @@ class TTicketParserImpl : public TActorBootstrapped { } } else { bool retryable = IsRetryableGrpcError(response->Status); - itPermission->second.Error = {.Message = response->Status.Msg, .Retryable = retryable}; + itPermission->second.Error = {.Message = TString{response->Status.Msg}, .Retryable = retryable}; if (itPermission->second.Subject.empty() || !retryable) { itPermission->second.Subject.clear(); BLOG_TRACE("Ticket " diff --git a/ydb/core/security/ticket_parser_ut.cpp b/ydb/core/security/ticket_parser_ut.cpp index eef1d6d0f43d..442c8628e21b 100644 --- a/ydb/core/security/ticket_parser_ut.cpp +++ b/ydb/core/security/ticket_parser_ut.cpp @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include #include #include #include diff --git a/ydb/core/statistics/database/ut/ut_database.cpp b/ydb/core/statistics/database/ut/ut_database.cpp index 0a676ba3c10c..46e7f1ad0aaa 100644 --- a/ydb/core/statistics/database/ut/ut_database.cpp +++ b/ydb/core/statistics/database/ut/ut_database.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/statistics/service/service_impl.cpp b/ydb/core/statistics/service/service_impl.cpp index 2fc0ecdac17e..041aeba937ad 100644 --- a/ydb/core/statistics/service/service_impl.cpp +++ b/ydb/core/statistics/service/service_impl.cpp @@ -27,7 +27,7 @@ #include #include -#include +#include #include namespace NKikimr { diff --git a/ydb/core/sys_view/ut/ya.make b/ydb/core/sys_view/ut/ya.make index c25815116d83..bcab5546ca83 100644 --- a/ydb/core/sys_view/ut/ya.make +++ b/ydb/core/sys_view/ut/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/core/kqp/ut/common ydb/core/persqueue/ut/common ydb/core/testlib/pg - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/draft ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/sys_view/ut_common.h b/ydb/core/sys_view/ut_common.h index 3587eaca66b8..b0930f30f8cb 100644 --- a/ydb/core/sys_view/ut_common.h +++ b/ydb/core/sys_view/ut_common.h @@ -1,9 +1,9 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #include diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index 251d0255e34e..21e64635ee8b 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include @@ -1734,7 +1734,7 @@ Y_UNIT_TEST_SUITE(SystemView) { THashSet names; for (const auto& child : children) { - names.insert(child.Name); + names.insert(TString{child.Name}); UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); } UNIT_ASSERT(names.contains("partition_stats")); @@ -1752,7 +1752,7 @@ Y_UNIT_TEST_SUITE(SystemView) { THashSet names; for (const auto& child : children) { - names.insert(child.Name); + names.insert(TString{child.Name}); UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table); } UNIT_ASSERT(names.contains("partition_stats")); diff --git a/ydb/core/sys_view/ut_large.cpp b/ydb/core/sys_view/ut_large.cpp index f6391dafdafa..dcf92c800fe8 100644 --- a/ydb/core/sys_view/ut_large.cpp +++ b/ydb/core/sys_view/ut_large.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index b0b268e49130..2cd4bd914129 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -1474,7 +1474,7 @@ TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletLabeledCo groupNames[j] = TString(1, toupper(groupNames[j][0])) + groupNames[j].substr(1); if (groupNames[j] == "Topic") { if (NPersQueue::CorrectName(groups[j])) { - TString dc = to_title(NPersQueue::GetDC(groups[j])); + TString dc = to_title(TString{NPersQueue::GetDC(groups[j])}); TString producer = NPersQueue::GetProducer(groups[j]); TString topic = NPersQueue::GetRealTopic(groups[j]); group = group->GetSubgroup("OriginDC", dc); diff --git a/ydb/core/testlib/common_helper.cpp b/ydb/core/testlib/common_helper.cpp index 0f329d951717..d713f3ad8151 100644 --- a/ydb/core/testlib/common_helper.cpp +++ b/ydb/core/testlib/common_helper.cpp @@ -5,8 +5,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/core/testlib/common_helper.h b/ydb/core/testlib/common_helper.h index cf1c1d1351d8..805634b1ff8a 100644 --- a/ydb/core/testlib/common_helper.h +++ b/ydb/core/testlib/common_helper.h @@ -1,6 +1,6 @@ #pragma once #include "test_client.h" -#include +#include #include namespace NKikimr::NKqp { diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 9626ac5f591b..142bfc3d52bb 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h index 6ced18c41346..48608698f42c 100644 --- a/ydb/core/testlib/test_pq_client.h +++ b/ydb/core/testlib/test_pq_client.h @@ -7,9 +7,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include @@ -501,7 +501,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { void RunYqlSchemeQuery(TString query, bool expectSuccess = true) { auto tableClient = NYdb::NTable::TTableClient(*Driver); - NYdb::TStatus result(NYdb::EStatus::SUCCESS, NYql::TIssues()); + NYdb::TStatus result(NYdb::EStatus::SUCCESS, NYdb::NIssue::TIssues()); for (size_t i = 0; i < 10; ++i) { result = tableClient.RetryOperationSync([&](NYdb::NTable::TSession session) { return session.ExecuteSchemeQuery(query).GetValueSync(); @@ -552,7 +552,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { TString endpoint = TStringBuilder() << "localhost:" << GRpcPort; auto driverConfig = NYdb::TDriverConfig() .SetEndpoint(endpoint) - .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); if (databaseName) driverConfig.SetDatabase(*databaseName); Driver.Reset(MakeHolder(driverConfig)); @@ -895,12 +895,12 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { auto pos = name.rfind("/"); Y_ABORT_UNLESS(pos != TString::npos); auto pref = "/Root/PQ/" + name.substr(0, pos); - ModifyACL(pref, name.substr(pos + 1), acl.SerializeAsString()); + ModifyACL(TString{pref}, name.substr(pos + 1), acl.SerializeAsString()); } void CreateTopicNoLegacy(const TString& name, ui32 partsCount, bool doWait = true, bool canWrite = true, - const TMaybe& dc = Nothing(), TVector rr = {"user"}, - const TMaybe& account = Nothing(), bool expectFail = false) + const std::optional& dc = std::nullopt, std::vector rr = {"user"}, + const std::optional& account = std::nullopt, bool expectFail = false) { CreateTopicNoLegacy({ .Name = name, @@ -1371,7 +1371,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { } public: - void AddTopic(const TString& topic, const TMaybe& dc = Nothing()) { + void AddTopic(const TString& topic, const std::optional& dc = std::nullopt) { Cerr << "AddTopic: " << topic << Endl; return AddOrRemoveTopic(topic, true, dc); } @@ -1381,7 +1381,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { return AddOrRemoveTopic(topic, false); } - void AddOrRemoveTopic(const TString& topic, bool add, const TMaybe& dc = Nothing()) { + void AddOrRemoveTopic(const TString& topic, bool add, const std::optional& dc = std::nullopt) { TStringBuilder query; query << "DECLARE $version as Int64; DECLARE $path AS Utf8; DECLARE $cluster as Utf8; "; if (add) { @@ -1389,7 +1389,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { } else { query << "DELETE FROM `/Root/PQ/Config/V2/Topics` WHERE path = $path AND dc = $cluster; "; } - TString cluster = dc.GetOrElse(NPersQueue::GetDC(topic)); + TString cluster = dc.value_or(NPersQueue::GetDC(topic)); query << GetAlterTopicsVersionQuery(); NYdb::TParamsBuilder builder; auto params = builder @@ -1418,11 +1418,11 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { ui32 PartsCount; bool DoWait = true; bool CanWrite = true; - TMaybe Dc = Nothing(); - TVector ReadRules = {"user"}; - TMaybe Account = Nothing(); + std::optional Dc = std::nullopt; + std::vector ReadRules = {"user"}; + std::optional Account = std::nullopt; bool ExpectFail = false; - TVector Codecs = NYdb::NPersQueue::GetDefaultCodecs(); + std::vector Codecs = NYdb::NPersQueue::GetDefaultCodecs(); }; void CreateTopicNoLegacy(const CreateTopicNoLegacyParams& params) @@ -1430,7 +1430,7 @@ class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { Cerr << "CreateTopicNoLegacy: " << params.Name << Endl; TString path = params.Name; - if (UseConfigTables && !path.StartsWith("/Root") && !params.Account.Defined()) { + if (UseConfigTables && !path.StartsWith("/Root") && !params.Account) { path = TStringBuilder() << "/Root/PQ/" << params.Name; } diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 8876c93bad2b..668310c6cc91 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -25,7 +25,7 @@ PEERDIR( ydb/apps/version ydb/library/actors/core ydb/library/actors/interconnect - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/grpc/server ydb/library/grpc/server/actors library/cpp/regex/pcre @@ -92,10 +92,10 @@ PEERDIR( ydb/public/api/protos ydb/public/lib/base ydb/public/lib/deprecated/kicli - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_topic/codecs - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/table ydb/services/auth ydb/services/cms ydb/services/datastreams diff --git a/ydb/core/tracing/ya.make b/ydb/core/tracing/ya.make index a0be10a3e3a5..9aea8e2c340c 100644 --- a/ydb/core/tracing/ya.make +++ b/ydb/core/tracing/ya.make @@ -10,6 +10,7 @@ SRCS( PEERDIR( ydb/core/base ydb/core/protos + ydb/core/mon ) END() diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp index 2722d8001715..c06dc271aef8 100644 --- a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp +++ b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp @@ -1,13 +1,13 @@ #include "session.h" #include "adapter.h" #include -#include +#include namespace NKikimr::NOlap::NBackground { Ydb::Operations::Operation TSessionInfoReport::SerializeToProto() const { Ydb::Operations::Operation result; - result.set_id("/" + ::ToString((int)Ydb::TOperationId::SS_BG_TASKS) + "?type=" + ClassName + "&id=" + Identifier); + result.set_id("/" + ::ToString((int)NKikimr::NOperationId::TOperationId::SS_BG_TASKS) + "?type=" + ClassName + "&id=" + Identifier); result.set_ready(IsFinished); return result; } diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make index 4cdec626486b..0a51a2516b24 100644 --- a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make +++ b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/library/accessor ydb/library/services ydb/core/tx/columnshard/bg_tasks/protos - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id ydb/public/api/protos ) diff --git a/ydb/core/tx/columnshard/hooks/abstract/ya.make b/ydb/core/tx/columnshard/hooks/abstract/ya.make index 33386775bd35..677bdd7e042b 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/ya.make +++ b/ydb/core/tx/columnshard/hooks/abstract/ya.make @@ -8,6 +8,7 @@ PEERDIR( ydb/core/tx/tiering/tier ydb/core/tx/columnshard/blobs_action/protos ydb/core/tx/columnshard/data_sharing/protos + ydb/public/sdk/cpp/src/client/resources yql/essentials/core/expr_nodes ) diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 611fc1e2a28d..a82954fad6ff 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 205286404ee6..df915af40ece 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/tx/datashard/datashard_ut_background_compaction.cpp b/ydb/core/tx/datashard/datashard_ut_background_compaction.cpp index 7f019efec697..723be3cf7169 100644 --- a/ydb/core/tx/datashard/datashard_ut_background_compaction.cpp +++ b/ydb/core/tx/datashard/datashard_ut_background_compaction.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include namespace NKikimr { diff --git a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp index e3e61412efca..9a0cf23206e4 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include @@ -1149,7 +1149,7 @@ Y_UNIT_TEST_SUITE(Cdc) { // get records auto reader = client.CreateReadSession(TReadSessionSettings() - .AppendTopics(TString("/Root/Table/Stream")) + .AppendTopics(std::string{"/Root/Table/Stream"}) .ConsumerName("user") .DisableClusterDiscovery(true) ); @@ -1164,7 +1164,7 @@ Y_UNIT_TEST_SUITE(Cdc) { pStream = data->GetPartitionStream(); for (const auto& item : data->GetMessages()) { const auto& record = records.at(reads++); - AssertJsonsEqual(item.GetData(), record); + AssertJsonsEqual(TString{item.GetData()}, record); if (checkKey) { UNIT_ASSERT_VALUES_EQUAL(item.GetPartitionKey(), CalcPartitionKey(record)); } @@ -1314,7 +1314,7 @@ Y_UNIT_TEST_SUITE(Cdc) { struct TopicRunner { private: - using TMessageMeta = TVector>; + using TMessageMeta = std::vector>; static TString DumpMessageMeta(TMessageMeta messageMeta) { std::stable_sort(messageMeta.begin(), messageMeta.end()); @@ -1324,7 +1324,7 @@ Y_UNIT_TEST_SUITE(Cdc) { static void AssertMessageMetaContains(const TMessageMeta& actual, const TMessageMeta& expected) { for (const auto& e : expected) { auto it = std::find_if(actual.begin(), actual.end(), [&e](const auto& a) { - return a.first == e.first && CheckJsonsEqual(a.second, e.second); + return a.first == e.first && CheckJsonsEqual(TString{a.second}, TString{e.second}); }); UNIT_ASSERT_C(it != actual.end(), TStringBuilder() << "Message meta '" << e << "' was expected" << ": actual# " << DumpMessageMeta(actual) @@ -1344,7 +1344,7 @@ Y_UNIT_TEST_SUITE(Cdc) { pStream = data->GetPartitionSession(); for (const auto& item : data->GetMessages()) { const auto& [body, meta] = records.at(reads++); - AssertJsonsEqual(item.GetData(), body); + AssertJsonsEqual(TString{item.GetData()}, body); AssertMessageMetaContains(item.GetMessageMeta()->Fields, meta); } } else if (auto* create = std::get_if(&*ev)) { @@ -1383,7 +1383,7 @@ Y_UNIT_TEST_SUITE(Cdc) { // create reader auto reader = client.CreateReadSession(NYdb::NTopic::TReadSessionSettings() - .AppendTopics(TString("/Root/Table/Stream")) + .AppendTopics(std::string{"/Root/Table/Stream"}) .ConsumerName("user") ); @@ -2936,7 +2936,7 @@ Y_UNIT_TEST_SUITE(Cdc) { // create reader auto reader = client.CreateReadSession(NYdb::NTopic::TReadSessionSettings() - .AppendTopics(TString("/Root/Table/Stream")) + .AppendTopics(std::string{"/Root/Table/Stream"}) .ConsumerName("user") ); @@ -4138,6 +4138,11 @@ void Out>(IOutputStream& output, const std::pair +void Out>(IOutputStream& output, const std::pair& x) { + output << x.first << ":" << x.second; +} + void AppendToString(TString& dst, const std::pair& x) { TStringOutput output(dst); output << x; diff --git a/ydb/core/tx/datashard/datashard_ut_common_kqp.h b/ydb/core/tx/datashard/datashard_ut_common_kqp.h index bb4c43d502c8..2173a6ad86da 100644 --- a/ydb/core/tx/datashard/datashard_ut_common_kqp.h +++ b/ydb/core/tx/datashard/datashard_ut_common_kqp.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/datashard_ut_common_pq.cpp b/ydb/core/tx/datashard/datashard_ut_common_pq.cpp index e9f5bcdf998a..87530079e034 100644 --- a/ydb/core/tx/datashard/datashard_ut_common_pq.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common_pq.cpp @@ -2,9 +2,9 @@ #include #include -#include -#include -#include +#include +#include +#include namespace NKikimr { diff --git a/ydb/core/tx/datashard/datashard_ut_ext_blobs_multiple_channels.cpp b/ydb/core/tx/datashard/datashard_ut_ext_blobs_multiple_channels.cpp index 4e37650262ea..35ddae4f5472 100644 --- a/ydb/core/tx/datashard/datashard_ut_ext_blobs_multiple_channels.cpp +++ b/ydb/core/tx/datashard/datashard_ut_ext_blobs_multiple_channels.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include namespace NKikimr { diff --git a/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp b/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp index 8ba963725cc2..ba0195361b0c 100644 --- a/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp +++ b/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp @@ -8,9 +8,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_ut_kqp.cpp b/ydb/core/tx/datashard/datashard_ut_kqp.cpp index dfc8e42f7015..8592702c8ea0 100644 --- a/ydb/core/tx/datashard/datashard_ut_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_ut_kqp.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index de1609bb9e11..252e414d895d 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator_ext_blobs.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator_ext_blobs.cpp index ff081be3e711..4d1b2b91d0a1 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator_ext_blobs.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator_ext_blobs.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include namespace NKikimr { diff --git a/ydb/core/tx/datashard/datashard_ut_read_table.h b/ydb/core/tx/datashard/datashard_ut_read_table.h index 73823f683adf..28cede0d4cfc 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_table.h +++ b/ydb/core/tx/datashard/datashard_ut_read_table.h @@ -4,7 +4,7 @@ #include #include -#include +#include namespace NKikimr { namespace NDataShardReadTableTest { diff --git a/ydb/core/tx/datashard/type_serialization.cpp b/ydb/core/tx/datashard/type_serialization.cpp index d871edbdf55a..e2c8f07ee621 100644 --- a/ydb/core/tx/datashard/type_serialization.cpp +++ b/ydb/core/tx/datashard/type_serialization.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/tx/datashard/ut_background_compaction/ya.make b/ydb/core/tx/datashard/ut_background_compaction/ya.make index da43c9e2f0da..c0d519af2861 100644 --- a/ydb/core/tx/datashard/ut_background_compaction/ya.make +++ b/ydb/core/tx/datashard/ut_background_compaction/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_build_index/ya.make b/ydb/core/tx/datashard/ut_build_index/ya.make index e22c9a7af3f2..103577bd90c8 100644 --- a/ydb/core/tx/datashard/ut_build_index/ya.make +++ b/ydb/core/tx/datashard/ut_build_index/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_change_collector/ya.make b/ydb/core/tx/datashard/ut_change_collector/ya.make index 84e19b6d0b12..a15f8be30886 100644 --- a/ydb/core/tx/datashard/ut_change_collector/ya.make +++ b/ydb/core/tx/datashard/ut_change_collector/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_change_exchange/ya.make b/ydb/core/tx/datashard/ut_change_exchange/ya.make index df652d5bcdbb..f9b037987233 100644 --- a/ydb/core/tx/datashard/ut_change_exchange/ya.make +++ b/ydb/core/tx/datashard/ut_change_exchange/ya.make @@ -22,10 +22,10 @@ PEERDIR( ydb/core/tx/schemeshard/ut_helpers yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp index 2ce5cac7ae22..495516f7f693 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include @@ -2636,7 +2636,7 @@ namespace { if (pg.IsNull()) { out << "(pg null)"; } else { - out << pg.Content_.Quote(); + out << TString{pg.Content_}.Quote(); } } diff --git a/ydb/core/tx/datashard/ut_compaction/ya.make b/ydb/core/tx/datashard/ut_compaction/ya.make index c28c2153f799..5397423fa981 100644 --- a/ydb/core/tx/datashard/ut_compaction/ya.make +++ b/ydb/core/tx/datashard/ut_compaction/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_erase_rows/ya.make b/ydb/core/tx/datashard/ut_erase_rows/ya.make index 63243ca8b5dc..041b11a64efe 100644 --- a/ydb/core/tx/datashard/ut_erase_rows/ya.make +++ b/ydb/core/tx/datashard/ut_erase_rows/ya.make @@ -23,7 +23,7 @@ PEERDIR( yql/essentials/public/udf/service/exception_policy yql/essentials/sql/pg ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_followers/ya.make b/ydb/core/tx/datashard/ut_followers/ya.make index 12a71c460efd..45c2f9c8420a 100644 --- a/ydb/core/tx/datashard/ut_followers/ya.make +++ b/ydb/core/tx/datashard/ut_followers/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_incremental_backup/ya.make b/ydb/core/tx/datashard/ut_incremental_backup/ya.make index 2c1d23bb50d8..7117370e9d73 100644 --- a/ydb/core/tx/datashard/ut_incremental_backup/ya.make +++ b/ydb/core/tx/datashard/ut_incremental_backup/ya.make @@ -21,10 +21,10 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_incremental_restore_scan/ya.make b/ydb/core/tx/datashard/ut_incremental_restore_scan/ya.make index 4b7c71e9a5a1..bb71bee18cd2 100644 --- a/ydb/core/tx/datashard/ut_incremental_restore_scan/ya.make +++ b/ydb/core/tx/datashard/ut_incremental_restore_scan/ya.make @@ -21,10 +21,10 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_init/ya.make b/ydb/core/tx/datashard/ut_init/ya.make index 39ea71c09a71..e41ffd34c79d 100644 --- a/ydb/core/tx/datashard/ut_init/ya.make +++ b/ydb/core/tx/datashard/ut_init/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_keys/ya.make b/ydb/core/tx/datashard/ut_keys/ya.make index f29b2fe48ea5..d350b6874a1e 100644 --- a/ydb/core/tx/datashard/ut_keys/ya.make +++ b/ydb/core/tx/datashard/ut_keys/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_kqp/ya.make b/ydb/core/tx/datashard/ut_kqp/ya.make index 311700901416..911a01facfd5 100644 --- a/ydb/core/tx/datashard/ut_kqp/ya.make +++ b/ydb/core/tx/datashard/ut_kqp/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_kqp_errors/ya.make b/ydb/core/tx/datashard/ut_kqp_errors/ya.make index d123e4eaeee1..ece6f3593d34 100644 --- a/ydb/core/tx/datashard/ut_kqp_errors/ya.make +++ b/ydb/core/tx/datashard/ut_kqp_errors/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_kqp_scan/ya.make b/ydb/core/tx/datashard/ut_kqp_scan/ya.make index 1594a6fbfe60..c923006845d0 100644 --- a/ydb/core/tx/datashard/ut_kqp_scan/ya.make +++ b/ydb/core/tx/datashard/ut_kqp_scan/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_local_kmeans/ya.make b/ydb/core/tx/datashard/ut_local_kmeans/ya.make index a7232750d9ca..313a310175c9 100644 --- a/ydb/core/tx/datashard/ut_local_kmeans/ya.make +++ b/ydb/core/tx/datashard/ut_local_kmeans/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_locks/ya.make b/ydb/core/tx/datashard/ut_locks/ya.make index d4ff5927c0e4..df4b0a8a936d 100644 --- a/ydb/core/tx/datashard/ut_locks/ya.make +++ b/ydb/core/tx/datashard/ut_locks/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_minikql/ya.make b/ydb/core/tx/datashard/ut_minikql/ya.make index e2fb28f28493..3290903451c1 100644 --- a/ydb/core/tx/datashard/ut_minikql/ya.make +++ b/ydb/core/tx/datashard/ut_minikql/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_minstep/ya.make b/ydb/core/tx/datashard/ut_minstep/ya.make index 8c718ee87827..08528be97abe 100644 --- a/ydb/core/tx/datashard/ut_minstep/ya.make +++ b/ydb/core/tx/datashard/ut_minstep/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_object_storage_listing/ya.make b/ydb/core/tx/datashard/ut_object_storage_listing/ya.make index ad93592d1bce..a887d22cd4d2 100644 --- a/ydb/core/tx/datashard/ut_object_storage_listing/ya.make +++ b/ydb/core/tx/datashard/ut_object_storage_listing/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_order/ya.make b/ydb/core/tx/datashard/ut_order/ya.make index 3f1f9a49e3ea..6afce40c08cc 100644 --- a/ydb/core/tx/datashard/ut_order/ya.make +++ b/ydb/core/tx/datashard/ut_order/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_range_ops/ya.make b/ydb/core/tx/datashard/ut_range_ops/ya.make index 9dbe1b26ae14..179187d28aab 100644 --- a/ydb/core/tx/datashard/ut_range_ops/ya.make +++ b/ydb/core/tx/datashard/ut_range_ops/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_read_iterator/ya.make b/ydb/core/tx/datashard/ut_read_iterator/ya.make index edc34cf4c8ca..15a56d65a56c 100644 --- a/ydb/core/tx/datashard/ut_read_iterator/ya.make +++ b/ydb/core/tx/datashard/ut_read_iterator/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_read_table/ya.make b/ydb/core/tx/datashard/ut_read_table/ya.make index 7fb49868a22c..1f27fbb8956f 100644 --- a/ydb/core/tx/datashard/ut_read_table/ya.make +++ b/ydb/core/tx/datashard/ut_read_table/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_reassign/ya.make b/ydb/core/tx/datashard/ut_reassign/ya.make index d429091b6c43..855b74b40c67 100644 --- a/ydb/core/tx/datashard/ut_reassign/ya.make +++ b/ydb/core/tx/datashard/ut_reassign/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_replication/ya.make b/ydb/core/tx/datashard/ut_replication/ya.make index 1646b1fd77f1..cfa48704aab7 100644 --- a/ydb/core/tx/datashard/ut_replication/ya.make +++ b/ydb/core/tx/datashard/ut_replication/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_reshuffle_kmeans/ya.make b/ydb/core/tx/datashard/ut_reshuffle_kmeans/ya.make index df4dbcd4ae47..24a35f2de72e 100644 --- a/ydb/core/tx/datashard/ut_reshuffle_kmeans/ya.make +++ b/ydb/core/tx/datashard/ut_reshuffle_kmeans/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_rs/ya.make b/ydb/core/tx/datashard/ut_rs/ya.make index f99ba9e57724..61d3a3382fb3 100644 --- a/ydb/core/tx/datashard/ut_rs/ya.make +++ b/ydb/core/tx/datashard/ut_rs/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_sample_k/ya.make b/ydb/core/tx/datashard/ut_sample_k/ya.make index cd4f9aec8e10..18713b07ee68 100644 --- a/ydb/core/tx/datashard/ut_sample_k/ya.make +++ b/ydb/core/tx/datashard/ut_sample_k/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_sequence/ya.make b/ydb/core/tx/datashard/ut_sequence/ya.make index 54d3ea3762e5..2c138e3b10b4 100644 --- a/ydb/core/tx/datashard/ut_sequence/ya.make +++ b/ydb/core/tx/datashard/ut_sequence/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ydb/core/tx/schemeshard/ut_helpers ) diff --git a/ydb/core/tx/datashard/ut_snapshot/ya.make b/ydb/core/tx/datashard/ut_snapshot/ya.make index 42c3e8ffa9ca..139143888b4c 100644 --- a/ydb/core/tx/datashard/ut_snapshot/ya.make +++ b/ydb/core/tx/datashard/ut_snapshot/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_stats/ya.make b/ydb/core/tx/datashard/ut_stats/ya.make index 040534377c89..17a3563cbff7 100644 --- a/ydb/core/tx/datashard/ut_stats/ya.make +++ b/ydb/core/tx/datashard/ut_stats/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_trace/ya.make b/ydb/core/tx/datashard/ut_trace/ya.make index a62cb369a4b9..a54611cfb7e7 100644 --- a/ydb/core/tx/datashard/ut_trace/ya.make +++ b/ydb/core/tx/datashard/ut_trace/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_upload_rows/ya.make b/ydb/core/tx/datashard/ut_upload_rows/ya.make index d11d33dfbb5b..3710aaddc20e 100644 --- a/ydb/core/tx/datashard/ut_upload_rows/ya.make +++ b/ydb/core/tx/datashard/ut_upload_rows/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_volatile/ya.make b/ydb/core/tx/datashard/ut_volatile/ya.make index 501a6b4d7c52..977b87383d55 100644 --- a/ydb/core/tx/datashard/ut_volatile/ya.make +++ b/ydb/core/tx/datashard/ut_volatile/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/datashard/ut_write/ya.make b/ydb/core/tx/datashard/ut_write/ya.make index 8d319a0bc9ef..8baa42f56da5 100644 --- a/ydb/core/tx/datashard/ut_write/ya.make +++ b/ydb/core/tx/datashard/ut_write/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/locks/ut_range_treap/ya.make b/ydb/core/tx/locks/ut_range_treap/ya.make index 2b594351ac19..76e7e1559783 100644 --- a/ydb/core/tx/locks/ut_range_treap/ya.make +++ b/ydb/core/tx/locks/ut_range_treap/ya.make @@ -21,7 +21,7 @@ PEERDIR( ydb/core/tx yql/essentials/public/udf/service/exception_policy ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/src/client/result ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/replication/controller/private_events.h b/ydb/core/tx/replication/controller/private_events.h index ebf083f97b0b..76688cd5b56e 100644 --- a/ydb/core/tx/replication/controller/private_events.h +++ b/ydb/core/tx/replication/controller/private_events.h @@ -2,7 +2,7 @@ #include "replication.h" -#include +#include #include #include diff --git a/ydb/core/tx/replication/controller/stream_creator.cpp b/ydb/core/tx/replication/controller/stream_creator.cpp index c1663bf2f956..13017942efb7 100644 --- a/ydb/core/tx/replication/controller/stream_creator.cpp +++ b/ydb/core/tx/replication/controller/stream_creator.cpp @@ -103,9 +103,9 @@ class TStreamCreator: public TActorBootstrapped { TString BuildStreamPath() const { switch (Kind) { case TReplication::ETargetKind::Table: - return CanonizePath(ChildPath(SplitPath(SrcPath), Changefeed.GetName())); + return CanonizePath(ChildPath(SplitPath(SrcPath), TString{Changefeed.GetName()})); case TReplication::ETargetKind::IndexTable: - return CanonizePath(ChildPath(SplitPath(SrcPath), {"indexImplTable", Changefeed.GetName()})); + return CanonizePath(ChildPath(SplitPath(SrcPath), {"indexImplTable", TString{Changefeed.GetName()}})); } } @@ -134,7 +134,7 @@ class TStreamCreator: public TActorBootstrapped { auto& result = ev->Get()->Result; if (result.GetStatus() == NYdb::EStatus::ALREADY_EXISTS) { - return Reply(NYdb::TStatus(NYdb::EStatus::SUCCESS, NYql::TIssues())); + return Reply(NYdb::TStatus(NYdb::EStatus::SUCCESS, NYdb::NIssue::TIssues())); } if (!result.IsSuccess()) { diff --git a/ydb/core/tx/replication/controller/target_discoverer.cpp b/ydb/core/tx/replication/controller/target_discoverer.cpp index a29fafea1ffd..0e253d0699c7 100644 --- a/ydb/core/tx/replication/controller/target_discoverer.cpp +++ b/ydb/core/tx/replication/controller/target_discoverer.cpp @@ -54,7 +54,7 @@ class TTargetDiscoverer: public TActorBootstrapped { << ": path# " << path.first << ", type# " << entry.Type); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Unsupported entry type: " << entry.Type); Failed.emplace_back(path.first, NYdb::TStatus(NYdb::EStatus::UNSUPPORTED, std::move(issues))); } else { @@ -114,8 +114,8 @@ class TTargetDiscoverer: public TActorBootstrapped { } const auto& target = ToAdd.emplace_back( - CanonizePath(ChildPath(SplitPath(path.first), index.GetIndexName())), - CanonizePath(ChildPath(SplitPath(path.second), {index.GetIndexName(), "indexImplTable"})), + CanonizePath(ChildPath(SplitPath(path.first), TString{index.GetIndexName()})), + CanonizePath(ChildPath(SplitPath(path.second), {TString{index.GetIndexName()}, "indexImplTable"})), TReplication::ETargetKind::IndexTable); LOG_I("Add target" << ": srcPath# " << target.SrcPath @@ -170,10 +170,10 @@ class TTargetDiscoverer: public TActorBootstrapped { return false; } - return entry.Name.StartsWith("~") - || entry.Name.StartsWith(".sys") - || entry.Name.StartsWith(".metadata") - || entry.Name.StartsWith("export-"); + return entry.Name.starts_with("~") + || entry.Name.starts_with(".sys") + || entry.Name.starts_with(".metadata") + || entry.Name.starts_with("export-"); } void Handle(TEvYdbProxy::TEvListDirectoryResponse::TPtr& ev) { diff --git a/ydb/core/tx/replication/controller/tx_describe_replication.cpp b/ydb/core/tx/replication/controller/tx_describe_replication.cpp index 1f3ffa70745b..cb63cde0a708 100644 --- a/ydb/core/tx/replication/controller/tx_describe_replication.cpp +++ b/ydb/core/tx/replication/controller/tx_describe_replication.cpp @@ -256,7 +256,7 @@ class TController::TTxDescribeReplication: public TTxBase { break; case TReplication::EState::Error: if (auto issue = state.MutableError()->AddIssues()) { - issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_severity(static_cast(NYdb::NIssue::ESeverity::Error)); issue->set_message(replication->GetIssue()); } break; diff --git a/ydb/core/tx/replication/controller/util.h b/ydb/core/tx/replication/controller/util.h index 66b407633a4b..7fba95897046 100644 --- a/ydb/core/tx/replication/controller/util.h +++ b/ydb/core/tx/replication/controller/util.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/ydb/core/tx/replication/service/topic_reader_ut.cpp b/ydb/core/tx/replication/service/topic_reader_ut.cpp index 5cfcf70a79c7..03c7542d059a 100644 --- a/ydb/core/tx/replication/service/topic_reader_ut.cpp +++ b/ydb/core/tx/replication/service/topic_reader_ut.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/tx/replication/service/ut_topic_reader/ya.make b/ydb/core/tx/replication/service/ut_topic_reader/ya.make index dfb71e2ba255..c1c0e25e800d 100644 --- a/ydb/core/tx/replication/service/ut_topic_reader/ya.make +++ b/ydb/core/tx/replication/service/ut_topic_reader/ya.make @@ -7,7 +7,7 @@ SIZE(MEDIUM) PEERDIR( ydb/core/tx/replication/ut_helpers ydb/core/tx/replication/ydb_proxy - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/testing/unittest ) diff --git a/ydb/core/tx/replication/service/ut_worker/ya.make b/ydb/core/tx/replication/service/ut_worker/ya.make index ea65d311d9f1..a38bf3e895bf 100644 --- a/ydb/core/tx/replication/service/ut_worker/ya.make +++ b/ydb/core/tx/replication/service/ut_worker/ya.make @@ -7,7 +7,7 @@ SIZE(MEDIUM) PEERDIR( ydb/core/tx/replication/ut_helpers ydb/core/tx/replication/ydb_proxy - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/testing/unittest ) diff --git a/ydb/core/tx/replication/service/worker_ut.cpp b/ydb/core/tx/replication/service/worker_ut.cpp index b1c80ed759a9..450df7f3d26e 100644 --- a/ydb/core/tx/replication/service/worker_ut.cpp +++ b/ydb/core/tx/replication/service/worker_ut.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/tx/replication/ut_helpers/write_topic.h b/ydb/core/tx/replication/ut_helpers/write_topic.h index 60bf89eac355..d1c3efd248a0 100644 --- a/ydb/core/tx/replication/ut_helpers/write_topic.h +++ b/ydb/core/tx/replication/ut_helpers/write_topic.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NKikimr::NReplication::NTestHelpers { diff --git a/ydb/core/tx/replication/ut_helpers/ya.make b/ydb/core/tx/replication/ut_helpers/ya.make index 54d74d6da076..b6f954b0da3c 100644 --- a/ydb/core/tx/replication/ut_helpers/ya.make +++ b/ydb/core/tx/replication/ut_helpers/ya.make @@ -6,7 +6,7 @@ PEERDIR( ydb/core/testlib/pg ydb/core/tx/replication/ydb_proxy ydb/library/actors/core - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/testing/unittest ) diff --git a/ydb/core/tx/replication/ydb_proxy/ut/ya.make b/ydb/core/tx/replication/ydb_proxy/ut/ya.make index d48144a6cab4..f64053190212 100644 --- a/ydb/core/tx/replication/ydb_proxy/ut/ya.make +++ b/ydb/core/tx/replication/ydb_proxy/ut/ya.make @@ -6,7 +6,7 @@ SIZE(MEDIUM) PEERDIR( ydb/core/tx/replication/ut_helpers - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/testing/unittest ) diff --git a/ydb/core/tx/replication/ydb_proxy/ya.make b/ydb/core/tx/replication/ydb_proxy/ya.make index 27ced7a7376e..5f1fc8a37ff1 100644 --- a/ydb/core/tx/replication/ydb_proxy/ya.make +++ b/ydb/core/tx/replication/ydb_proxy/ya.make @@ -3,12 +3,12 @@ LIBRARY() PEERDIR( ydb/core/base ydb/core/protos - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types/credentials - ydb/public/sdk/cpp/client/ydb_types/credentials/login + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types/credentials + ydb/public/sdk/cpp/src/client/types/credentials/login ) SRCS( diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp index 40252d2f17da..c95c473e286c 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp @@ -2,8 +2,8 @@ #include "ydb_proxy.h" #include -#include -#include +#include +#include #include #include @@ -230,7 +230,7 @@ class TTopicReader: public TBaseProxyActor { } else if (std::get_if(&*event)) { return WaitEvent(ev->Get()->Sender, ev->Get()->Cookie); } else if (auto* x = std::get_if(&*event)) { - auto status = TStatus(EStatus::UNAVAILABLE, NYql::TIssues{NYql::TIssue(x->DebugString())}); + auto status = TStatus(EStatus::UNAVAILABLE, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue(x->DebugString())}); return Leave(ev->Get()->Sender, std::move(status)); } else if (auto* x = std::get_if(&*event)) { return Leave(ev->Get()->Sender, std::move(*x)); diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h index 17b2e02d41e4..6a5801003c24 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp index 356c5f157929..8fa949792ded 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include @@ -341,7 +341,7 @@ Y_UNIT_TEST_SUITE(YdbProxy) { .Build(); auto ev = env.Send( - new TEvYdbProxy::TEvCreateTableRequest(item.SourcePath(), std::move(schema), {})); + new TEvYdbProxy::TEvCreateTableRequest(TString{item.SourcePath()}, std::move(schema), {})); UNIT_ASSERT(ev); UNIT_ASSERT(ev->Get()->Result.IsSuccess()); } diff --git a/ydb/core/tx/scheme_cache/ya.make b/ydb/core/tx/scheme_cache/ya.make index 71c483618744..2efac664164a 100644 --- a/ydb/core/tx/scheme_cache/ya.make +++ b/ydb/core/tx/scheme_cache/ya.make @@ -4,6 +4,7 @@ PEERDIR( ydb/core/base ydb/core/protos ydb/core/scheme + ydb/core/persqueue/writer ydb/library/aclib ) diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make index 0c4a491d3280..3567f61d6172 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make @@ -14,6 +14,7 @@ PEERDIR( ydb/core/protos ydb/library/actors/wilson ydb/library/formats/arrow + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make index 24a9ac14d105..7c33d02871bf 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( ydb/core/tx/schemeshard/olap/operations/alter/abstract + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make index d048f582fe10..c1e4adba1ab8 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make @@ -8,6 +8,7 @@ PEERDIR( ydb/core/tx/schemeshard/olap/operations/alter/abstract ydb/core/tx/columnshard/data_sharing/initiator/controller ydb/core/tx/columnshard/data_sharing/protos + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/olap/ttl/ya.make b/ydb/core/tx/schemeshard/olap/ttl/ya.make index 3d84cb700fb2..4b0da27c3e3c 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/ya.make +++ b/ydb/core/tx/schemeshard/olap/ttl/ya.make @@ -10,6 +10,8 @@ PEERDIR( ydb/core/base ydb/core/protos ydb/core/tx/tiering/tier + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 23c2308c10e6..e6692fa6fb41 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index 1e236ad6696f..916ad2489532 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/tx/schemeshard/ut_helpers/ya.make b/ydb/core/tx/schemeshard/ut_helpers/ya.make index 7395ee2a7dda..c364a92d5f62 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ya.make +++ b/ydb/core/tx/schemeshard/ut_helpers/ya.make @@ -21,8 +21,8 @@ PEERDIR( ydb/core/tx/tx_proxy ydb/public/lib/scheme_types yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table ) SRCS( diff --git a/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp b/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp index 7337b0a8a71c..cbc0edf272ab 100644 --- a/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp +++ b/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include using namespace NKikimr; using namespace NSchemeShard; diff --git a/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp index 28895166660b..52572b672ef7 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include using namespace NKikimr; using namespace NSchemeShard; diff --git a/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp index 49bd38446dde..73d0902c1b87 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include using namespace NKikimr; using namespace NSchemeShard; diff --git a/ydb/core/tx/schemeshard/ut_index_build/ya.make b/ydb/core/tx/schemeshard/ut_index_build/ya.make index f6050c4acffc..311ccd7078d3 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ya.make +++ b/ydb/core/tx/schemeshard/ut_index_build/ya.make @@ -12,7 +12,7 @@ PEERDIR( ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make b/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make index ac192d26f931..66b488bc955d 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make @@ -22,7 +22,7 @@ PEERDIR( ydb/core/tx ydb/core/tx/schemeshard/ut_helpers yql/essentials/public/udf/service/exception_policy - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) SRCS( diff --git a/ydb/core/tx/tiering/manager.h b/ydb/core/tx/tiering/manager.h index 9c223fbd52a0..5594c8967312 100644 --- a/ydb/core/tx/tiering/manager.h +++ b/ydb/core/tx/tiering/manager.h @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index e451f7150f73..b7507199dcdf 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 36fcb2d9ccf1..fb55bc7c7a61 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -25,7 +25,7 @@ #include #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include diff --git a/ydb/core/tx/tx_proxy/ya.make b/ydb/core/tx/tx_proxy/ya.make index 911b835d3aaf..907eb06dc18d 100644 --- a/ydb/core/tx/tx_proxy/ya.make +++ b/ydb/core/tx/tx_proxy/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( mon.cpp proxy_impl.cpp diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 0600412c3aa6..265626d0b6a1 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -74,6 +74,7 @@ PEERDIR( library/cpp/random_provider ydb/core/base ydb/core/protos + ydb/core/mon library/cpp/deprecated/atomic ydb/library/yverify_stream ) diff --git a/ydb/core/viewer/json_local_rpc.h b/ydb/core/viewer/json_local_rpc.h index 139848dc2f1e..59c25ef62f64 100644 --- a/ydb/core/viewer/json_local_rpc.h +++ b/ydb/core/viewer/json_local_rpc.h @@ -1,6 +1,7 @@ #pragma once #include "json_pipe_req.h" #include +#include #include namespace NKikimr::NViewer { @@ -156,12 +157,12 @@ class TJsonLocalRpc : public TViewerPipeClient { } NYql::TIssues issues; NYql::IssuesFromMessage(response.operation().issues(), issues); - result->Status = NYdb::TStatus(NYdb::EStatus(response.operation().status()), std::move(issues)); + result->Status = NYdb::TStatus(NYdb::EStatus(response.operation().status()), NYdb::NAdapters::ToSdkIssues(std::move(issues))); } else { result->Message = response; NYql::TIssues issues; NYql::IssuesFromMessage(response.issues(), issues); - result->Status = NYdb::TStatus(NYdb::EStatus(response.status()), std::move(issues)); + result->Status = NYdb::TStatus(NYdb::EStatus(response.status()), NYdb::NAdapters::ToSdkIssues(std::move(issues))); } actorSystem->Send(actorId, result.Release()); diff --git a/ydb/core/viewer/operation_list.h b/ydb/core/viewer/operation_list.h index e72925d302de..201cc21b3150 100644 --- a/ydb/core/viewer/operation_list.h +++ b/ydb/core/viewer/operation_list.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace NKikimr::NViewer { diff --git a/ydb/core/viewer/viewer.h b/ydb/core/viewer/viewer.h index 355a4029dae5..d1920c72ab58 100644 --- a/ydb/core/viewer/viewer.h +++ b/ydb/core/viewer/viewer.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include namespace NKikimr::NViewer { diff --git a/ydb/core/viewer/viewer_labeled_counters.h b/ydb/core/viewer/viewer_labeled_counters.h index f2a9272d30c1..0a55605caa6f 100644 --- a/ydb/core/viewer/viewer_labeled_counters.h +++ b/ydb/core/viewer/viewer_labeled_counters.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace NKikimr::NViewer { diff --git a/ydb/core/viewer/viewer_query.h b/ydb/core/viewer/viewer_query.h index c153b36696d7..d94ed998c553 100644 --- a/ydb/core/viewer/viewer_query.h +++ b/ydb/core/viewer/viewer_query.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace NKikimr::NViewer { @@ -257,12 +257,12 @@ class TJsonQuery : public TViewerPipeClient { CreateSessionResponse.Set(std::move(ev)); } else { CreateSessionResponse.Error("FailedToCreateSession"); - NYql::TIssue issue; + NYdb::NIssue::TIssue issue; issue.SetMessage("Failed to create session"); - issue.SetCode(ev->Get()->Record.GetYdbStatus(), NYql::TSeverityIds::S_ERROR); + issue.SetCode(ev->Get()->Record.GetYdbStatus(), NYdb::NIssue::ESeverity::Error); NJson::TJsonValue json; TString message; - MakeJsonErrorReply(json, message, NYdb::TStatus(NYdb::EStatus(ev->Get()->Record.GetYdbStatus()), NYql::TIssues{issue})); + MakeJsonErrorReply(json, message, NYdb::TStatus(NYdb::EStatus(ev->Get()->Record.GetYdbStatus()), NYdb::NIssue::TIssues{issue})); return ReplyWithJsonAndPassAway(json, message); } @@ -551,7 +551,7 @@ class TJsonQuery : public TViewerPipeClient { QueryResponse.Error("QueryError"); NYql::TIssues issues; NYql::IssuesFromMessage(ev->Get()->Record.GetResponse().GetQueryIssues(), issues); - MakeErrorReply(jsonResponse, NYdb::TStatus(NYdb::EStatus(ev->Get()->Record.GetYdbStatus()), std::move(issues))); + MakeErrorReply(jsonResponse, NYdb::TStatus(NYdb::EStatus(ev->Get()->Record.GetYdbStatus()), NYdb::NAdapters::ToSdkIssues(std::move(issues)))); } ReplyWithJsonAndPassAway(jsonResponse); } @@ -563,7 +563,7 @@ class TJsonQuery : public TViewerPipeClient { if (record.IssuesSize() > 0) { NYql::TIssues issues; NYql::IssuesFromMessage(record.GetIssues(), issues); - MakeErrorReply(jsonResponse, NYdb::TStatus(NYdb::EStatus(record.GetStatusCode()), std::move(issues))); + MakeErrorReply(jsonResponse, NYdb::TStatus(NYdb::EStatus(record.GetStatusCode()), NYdb::NAdapters::ToSdkIssues(std::move(issues)))); } ReplyWithJsonAndPassAway(jsonResponse); } @@ -685,7 +685,7 @@ class TJsonQuery : public TViewerPipeClient { } } catch (const std::exception& ex) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Convert error: " << ex.what()); MakeErrorReply(jsonResponse, NYdb::TStatus(NYdb::EStatus::BAD_REQUEST, std::move(issues))); return; diff --git a/ydb/core/viewer/viewer_query_old.h b/ydb/core/viewer/viewer_query_old.h index ff592792d033..5092990d4b55 100644 --- a/ydb/core/viewer/viewer_query_old.h +++ b/ydb/core/viewer/viewer_query_old.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace NKikimr::NViewer { diff --git a/ydb/core/viewer/ya.make b/ydb/core/viewer/ya.make index bdab7651d3ea..ba49857ccabd 100644 --- a/ydb/core/viewer/ya.make +++ b/ydb/core/viewer/ya.make @@ -585,7 +585,8 @@ PEERDIR( ydb/public/lib/json_value ydb/public/lib/ydb_cli/common ydb/public/api/grpc - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/types contrib/libs/yaml-cpp ) diff --git a/ydb/core/viewer/yaml/yaml.h b/ydb/core/viewer/yaml/yaml.h index 8415cce3c57e..9fd3a056427f 100644 --- a/ydb/core/viewer/yaml/yaml.h +++ b/ydb/core/viewer/yaml/yaml.h @@ -15,6 +15,7 @@ class TProtoToYaml { static YAML::Node ProtoToYamlSchema(const ::google::protobuf::Descriptor* descriptor); static void FillEnum(YAML::Node property, const ::google::protobuf::EnumDescriptor* enumDescriptor, const TEnumSettings& enumSettings = TEnumSettings()); + static void FillEnum(YAML::Node property, const std::vector& enumNames, const TEnumSettings& enumSettings); template static YAML::Node ProtoToYamlSchema() { diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp index 88657dfe51d4..0d049567c395 100644 --- a/ydb/core/ydb_convert/ydb_convert.cpp +++ b/ydb/core/ydb_convert/ydb_convert.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/ydb_convert/ydb_convert.h b/ydb/core/ydb_convert/ydb_convert.h index 77ad84b70c02..3752fcae9131 100644 --- a/ydb/core/ydb_convert/ydb_convert.h +++ b/ydb/core/ydb_convert/ydb_convert.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/ymq/actor/auth_factory.h b/ydb/core/ymq/actor/auth_factory.h index 18670fa37e23..b7092323d004 100644 --- a/ydb/core/ymq/actor/auth_factory.h +++ b/ydb/core/ymq/actor/auth_factory.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace NKikimr::NSQS { diff --git a/ydb/core/ymq/actor/auth_mocks.cpp b/ydb/core/ymq/actor/auth_mocks.cpp index 798996947929..dc298ccacee6 100644 --- a/ydb/core/ymq/actor/auth_mocks.cpp +++ b/ydb/core/ymq/actor/auth_mocks.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/ymq/actor/auth_multi_factory.cpp b/ydb/core/ymq/actor/auth_multi_factory.cpp index 165bb3be44f1..3042009b3ce0 100644 --- a/ydb/core/ymq/actor/auth_multi_factory.cpp +++ b/ydb/core/ymq/actor/auth_multi_factory.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -588,7 +588,7 @@ void TMultiAuthFactory::RegisterAuthActor(NActors::TActorSystem& system, TAuthAc const ui32 poolID = data.ExecutorPoolID; // token needed only for ResourceManager - const auto token = UseResourceManagerFolderService_ ? CredentialsProvider_->GetAuthInfo() : ""; + const TString token = UseResourceManagerFolderService_ ? CredentialsProvider_->GetAuthInfo() : ""; if (data.RequestFormat == NSQS::TAuthActorData::Json) { system.Register( diff --git a/ydb/core/ymq/actor/auth_multi_factory.h b/ydb/core/ymq/actor/auth_multi_factory.h index 725cd6bc403a..5965b2988207 100644 --- a/ydb/core/ymq/actor/auth_multi_factory.h +++ b/ydb/core/ymq/actor/auth_multi_factory.h @@ -2,7 +2,7 @@ #include #include "ydb/core/protos/sqs.pb.h" -#include "ydb/library/grpc/client/grpc_client_low.h" +#include "ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.h" #include "ydb/library/http_proxy/error/error.h" #include #include diff --git a/ydb/core/ymq/actor/cleanup_queue_data.h b/ydb/core/ymq/actor/cleanup_queue_data.h index 25d3c73d905e..24aaf840db92 100644 --- a/ydb/core/ymq/actor/cleanup_queue_data.h +++ b/ydb/core/ymq/actor/cleanup_queue_data.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/ymq/actor/index_events_processor.cpp b/ydb/core/ymq/actor/index_events_processor.cpp index f5c441b13238..d5a601be6b72 100644 --- a/ydb/core/ymq/actor/index_events_processor.cpp +++ b/ydb/core/ymq/actor/index_events_processor.cpp @@ -1,5 +1,5 @@ #include "index_events_processor.h" -#include +#include namespace NKikimr::NSQS { using namespace NActors; diff --git a/ydb/core/ymq/actor/index_events_processor.h b/ydb/core/ymq/actor/index_events_processor.h index 41260a768b01..dda27b765ec3 100644 --- a/ydb/core/ymq/actor/index_events_processor.h +++ b/ydb/core/ymq/actor/index_events_processor.h @@ -5,8 +5,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -56,10 +56,10 @@ friend class TIndexProcesorTests; struct TQueueEvent { EQueueEventType Type; ui64 Timestamp; - TString CustomName; - TString CloudId; - TString FolderId; - TString Labels; + std::string CustomName; + std::string CloudId; + std::string FolderId; + std::string Labels; }; struct TQueueTableKey { TString Account; diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp index 5bb74f46c75d..456b0cf45b77 100644 --- a/ydb/core/ymq/actor/service.cpp +++ b/ydb/core/ymq/actor/service.cpp @@ -17,7 +17,7 @@ #include "cleanup_queue_data.h" #include -#include +#include #include #include #include diff --git a/ydb/core/ymq/actor/service.h b/ydb/core/ymq/actor/service.h index 5a684e4f5046..93ce333da645 100644 --- a/ydb/core/ymq/actor/service.h +++ b/ydb/core/ymq/actor/service.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/ymq/actor/ya.make b/ydb/core/ymq/actor/ya.make index abd995f4d597..6d1af352d98c 100644 --- a/ydb/core/ymq/actor/ya.make +++ b/ydb/core/ymq/actor/ya.make @@ -61,7 +61,7 @@ PEERDIR( ydb/library/actors/core library/cpp/containers/intrusive_rb_tree library/cpp/digest/md5 - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/ycloud/impl library/cpp/logger library/cpp/lwtrace/mon @@ -91,7 +91,7 @@ PEERDIR( ydb/library/mkql_proto/protos ydb/public/lib/scheme_types ydb/public/lib/value - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials yql/essentials/minikql ydb/public/lib/deprecated/client ) diff --git a/ydb/core/ymq/base/run_query.h b/ydb/core/ymq/base/run_query.h index 56e12bccf3d7..84f6b9024b83 100644 --- a/ydb/core/ymq/base/run_query.h +++ b/ydb/core/ymq/base/run_query.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/ydb/core/yql_testlib/ya.make b/ydb/core/yql_testlib/ya.make index e50fdeeaf7f3..b03a299ba7c0 100644 --- a/ydb/core/yql_testlib/ya.make +++ b/ydb/core/yql_testlib/ya.make @@ -6,7 +6,7 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/testing/unittest ydb/core/base diff --git a/ydb/library/arrow_parquet/result_set_parquet_printer.cpp b/ydb/library/arrow_parquet/result_set_parquet_printer.cpp index 1d5d0a762899..088379bcccf4 100644 --- a/ydb/library/arrow_parquet/result_set_parquet_printer.cpp +++ b/ydb/library/arrow_parquet/result_set_parquet_printer.cpp @@ -1,7 +1,7 @@ #include "result_set_parquet_printer.h" -#include -#include +#include +#include #include #include diff --git a/ydb/library/arrow_parquet/result_set_parquet_printer.h b/ydb/library/arrow_parquet/result_set_parquet_printer.h index 4d67cde2e13d..c1ebc7168b5d 100644 --- a/ydb/library/arrow_parquet/result_set_parquet_printer.h +++ b/ydb/library/arrow_parquet/result_set_parquet_printer.h @@ -6,8 +6,9 @@ #include namespace NYdb { - - class TResultSet; + inline namespace V3 { + class TResultSet; + } class TResultSetParquetPrinter { public: diff --git a/ydb/library/arrow_parquet/ya.make b/ydb/library/arrow_parquet/ya.make index e20b634cba26..28364f4125f7 100644 --- a/ydb/library/arrow_parquet/ya.make +++ b/ydb/library/arrow_parquet/ya.make @@ -5,7 +5,7 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/value contrib/libs/apache/arrow ) diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 3a811a24c170..86489ff290b8 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -10,13 +10,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -39,8 +39,11 @@ #include #include +#include + #include + namespace NYdb::NBackup { static constexpr size_t IO_BUFFER_SIZE = 2 << 20; // 2 MiB @@ -103,7 +106,7 @@ case EPrimitiveType::type: \ #define CASE_PRINT_PRIMITIVE_STRING_TYPE(out, type) \ case EPrimitiveType::type: { \ - TString str = parser.Get##type(); \ + auto str = TString{parser.Get##type()}; \ CGIEscape(str); \ out << '"' << str << '"'; \ } \ @@ -348,7 +351,7 @@ void ReadTable(TDriver driver, const NTable::TTableDescription& desc, const TStr do { lastWrittenPK = TryReadTable(driver, desc, fullTablePath, folderPath, lastWrittenPK, &fileCounter, ordered); if (lastWrittenPK && retries) { - LOG_D("Retry read table from key: " << FormatValueYson(*lastWrittenPK).Quote()); + LOG_D("Retry read table from key: " << TString{FormatValueYson(*lastWrittenPK)}.Quote()); } } while (lastWrittenPK && retries--); @@ -502,10 +505,10 @@ void BackupChangefeeds(TDriver driver, const TString& tablePath, const TFsPath& auto desc = DescribeTable(driver, tablePath); for (const auto& changefeedDesc : desc.GetChangefeedDescriptions()) { - TFsPath changefeedDirPath = CreateDirectory(folderPath, changefeedDesc.GetName()); + TFsPath changefeedDirPath = CreateDirectory(folderPath, TString{changefeedDesc.GetName()}); auto protoChangeFeedDesc = ProtoFromChangefeedDesc(changefeedDesc); - const auto descTopicResult = DescribeTopic(driver, JoinDatabasePath(tablePath, changefeedDesc.GetName())); + const auto descTopicResult = DescribeTopic(driver, JoinDatabasePath(tablePath, TString{changefeedDesc.GetName()})); VerifyStatus(descTopicResult); const auto& topicDescription = descTopicResult.GetTopicDescription(); auto protoTopicDescription = NYdb::TProtoAccessor::GetProto(topicDescription); @@ -587,7 +590,8 @@ void ValidateViewQuery(const TString& query, const TString& dbPath, NYql::TIssue } TString BuildCreateViewQuery(TStringBuf name, const NView::TViewDescription& description, TStringBuf backupRoot) { - auto [contextRecreation, select] = SplitViewQuery(description.GetQueryText()); + auto queryText = TString{description.GetQueryText()}; + auto [contextRecreation, select] = SplitViewQuery(queryText); const TString query = std::format( "-- backup root: \"{}\"\n" @@ -633,7 +637,7 @@ void BackupView(TDriver driver, const TString& dbBackupRoot, const TString& dbPa NView::TViewClient client(driver); auto viewDescription = DescribeView(client, dbPath); - ValidateViewQuery(viewDescription.GetQueryText(), dbPath, issues); + ValidateViewQuery(TString{viewDescription.GetQueryText()}, dbPath, issues); const auto fsPath = fsBackupDir.Child(NDump::NFiles::CreateView().FileName); LOG_D("Write view creation query to " << fsPath.GetPath().Quote()); diff --git a/ydb/library/backup/backup.h b/ydb/library/backup/backup.h index 9da17496f443..dfd96b40b3df 100644 --- a/ydb/library/backup/backup.h +++ b/ydb/library/backup/backup.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -11,6 +11,7 @@ class TRegExMatch; namespace NYdb { +inline namespace V3 { class TDriver; class TResultSetParser; class TValue; @@ -18,6 +19,7 @@ class TValue; namespace NTable { class TTableDescription; } +} namespace NBackup { diff --git a/ydb/library/backup/db_iterator.h b/ydb/library/backup/db_iterator.h index e0f773d25bcc..6b94f68919a7 100644 --- a/ydb/library/backup/db_iterator.h +++ b/ydb/library/backup/db_iterator.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include #include @@ -76,7 +76,7 @@ class TDbIterator { NextNodes.front().IsListed = true; const auto& children = childList.GetChildren(); - if (!children) { + if (children.empty()) { break; } const auto& currRelPath = GetRelPath(); diff --git a/ydb/library/backup/query_builder.cpp b/ydb/library/backup/query_builder.cpp index c10386ffb8fe..554409ea4c1c 100644 --- a/ydb/library/backup/query_builder.cpp +++ b/ydb/library/backup/query_builder.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include @@ -260,7 +260,7 @@ void TQueryBuilder::AddLine(TStringBuf line) { Y_ENSURE(tok, "Empty token on line"); TTypeParser type(col.Type); Value.AddMember(col.Name); - AddMemberFromString(type, col.Name, tok); + AddMemberFromString(type, TString{col.Name}, tok); } Value.EndStruct(); } diff --git a/ydb/library/backup/query_builder.h b/ydb/library/backup/query_builder.h index 05f57d1d3f31..9ff2fe8bb7cf 100644 --- a/ydb/library/backup/query_builder.h +++ b/ydb/library/backup/query_builder.h @@ -1,9 +1,9 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -13,7 +13,7 @@ namespace NYdb::NBackup { class TQueryBuilder { - TVector Columns; + std::vector Columns; const TString Query; TValueBuilder Value; @@ -25,7 +25,7 @@ class TQueryBuilder { static TType GetType(TTypeParser& typeParser, const TString& name); public: - TQueryBuilder(const TString& path, TVector columns) + TQueryBuilder(const TString& path, std::vector columns) : Columns(std::move(columns)) , Query(BuildQuery(path)) {} @@ -57,7 +57,7 @@ class TQueryFromFileIterator { std::conditional_t ReadNext(); public: - TQueryFromFileIterator(const TString& path, const TString& dataFileName, TVector columns, i64 buffSize, + TQueryFromFileIterator(const TString& path, const TString& dataFileName, std::vector columns, i64 buffSize, i64 maxRowsPerQuery, i64 maxBytesPerQuery) : DataFile(dataFileName, OpenExisting | RdOnly) , Query(path, std::move(columns)) diff --git a/ydb/library/backup/query_uploader.h b/ydb/library/backup/query_uploader.h index b72a31cea266..ecde61198204 100644 --- a/ydb/library/backup/query_uploader.h +++ b/ydb/library/backup/query_uploader.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/ydb/library/backup/ut/ut.cpp b/ydb/library/backup/ut/ut.cpp index ef6bb5bfccac..fb1d2aba7dcc 100644 --- a/ydb/library/backup/ut/ut.cpp +++ b/ydb/library/backup/ut/ut.cpp @@ -3,8 +3,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -12,6 +12,10 @@ #include #include +#include + +#include + namespace NYdb { Y_UNIT_TEST_SUITE(BackupToolValuePrintParse) { @@ -126,7 +130,7 @@ Y_UNIT_TEST(ParseValuesFromFile) { case 0: { UNIT_ASSERT(parser.GetPrimitiveType() == EPrimitiveType::Uint32); parser.CloseOptional(); - const TMaybe val = parser.GetOptionalUint32(); + const std::optional val = parser.GetOptionalUint32(); if (rowsRead % 2 == 0) { UNIT_ASSERT(!val); } else { @@ -139,7 +143,7 @@ Y_UNIT_TEST(ParseValuesFromFile) { UNIT_ASSERT(parser.GetPrimitiveType() == EPrimitiveType::String); parser.CloseOptional(); TString col2str = TStringBuilder() << "TestString" << 2 * rowsRead << "with number"; - const TMaybe val = parser.GetOptionalString(); + const std::optional val = parser.GetOptionalString(); if (rowsRead % 2 == 0) { UNIT_ASSERT(val); UNIT_ASSERT_STRINGS_EQUAL(*val, col2str); @@ -151,7 +155,7 @@ Y_UNIT_TEST(ParseValuesFromFile) { case 2: { UNIT_ASSERT(parser.GetPrimitiveType() == EPrimitiveType::Int64); parser.CloseOptional(); - const TMaybe val = parser.GetOptionalInt64(); + const std::optional val = parser.GetOptionalInt64(); UNIT_ASSERT(val); UNIT_ASSERT(*val == rowsRead*rowsRead); break; diff --git a/ydb/library/backup/ya.make b/ydb/library/backup/ya.make index d2afef11ffa5..9aac8674632f 100644 --- a/ydb/library/backup/ya.make +++ b/ydb/library/backup/ya.make @@ -12,13 +12,13 @@ PEERDIR( ydb/public/lib/ydb_cli/dump/util ydb/public/lib/yson_value ydb/public/lib/ydb_cli/dump/files - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/value ) SRCS( diff --git a/ydb/library/db_pool/db_pool.cpp b/ydb/library/db_pool/db_pool.cpp index 883908ca2ab2..3397daed5fa4 100644 --- a/ydb/library/db_pool/db_pool.cpp +++ b/ydb/library/db_pool/db_pool.cpp @@ -12,7 +12,6 @@ namespace NDbPool { using namespace NActors; -using namespace NYql; class TDbPoolActor : public NActors::TActor { @@ -62,7 +61,7 @@ class TDbPoolActor : public NActors::TActor { ) void PassAway() override { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("DB connection closed"); auto cancelled = NYdb::TStatus(NYdb::EStatus::CANCELLED, std::move(issues)); for (const auto& x : Requests) { @@ -93,7 +92,7 @@ class TDbPoolActor : public NActors::TActor { if (auto pRequest = std::get_if(&requestVariant)) { auto& request = *pRequest; auto cookie = request.Cookie; - auto sharedResult = std::make_shared>(); + auto sharedResult = std::make_shared>(); NYdb::NTable::TRetryOperationSettings settings; settings.Idempotent(request.Idempotent); TableClient.RetryOperation([sharedResult, request](NYdb::NTable::TSession session) { diff --git a/ydb/library/db_pool/db_pool.h b/ydb/library/db_pool/db_pool.h index 8c207b4fa5ff..0f6ed063b43f 100644 --- a/ydb/library/db_pool/db_pool.h +++ b/ydb/library/db_pool/db_pool.h @@ -2,7 +2,7 @@ #include "events.h" -#include +#include #include #include diff --git a/ydb/library/db_pool/events.h b/ydb/library/db_pool/events.h index 2f9c683a74dd..6f06f40888b8 100644 --- a/ydb/library/db_pool/events.h +++ b/ydb/library/db_pool/events.h @@ -3,7 +3,7 @@ #include #include -#include +#include namespace NDbPool { @@ -33,9 +33,9 @@ struct TEvents { struct TEvDbResponse : NActors::TEventLocal { NYdb::TStatus Status; - TVector ResultSets; + std::vector ResultSets; - TEvDbResponse(NYdb::TStatus status, const TVector& resultSets) + TEvDbResponse(NYdb::TStatus status, const std::vector& resultSets) : Status(status) , ResultSets(resultSets) {} diff --git a/ydb/library/db_pool/ya.make b/ydb/library/db_pool/ya.make index 51269617e9f9..ed2036a433fb 100644 --- a/ydb/library/db_pool/ya.make +++ b/ydb/library/db_pool/ya.make @@ -9,8 +9,8 @@ PEERDIR( library/cpp/monlib/dynamic_counters ydb/library/db_pool/protos ydb/library/security - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/library/folder_service/events.h b/ydb/library/folder_service/events.h index 416445b930fb..4df0d45e60ff 100644 --- a/ydb/library/folder_service/events.h +++ b/ydb/library/folder_service/events.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NKikimr::NFolderService { diff --git a/ydb/library/folder_service/mock/ya.make b/ydb/library/folder_service/mock/ya.make index a62e26a4b9f4..741edf41b24c 100644 --- a/ydb/library/folder_service/mock/ya.make +++ b/ydb/library/folder_service/mock/ya.make @@ -8,6 +8,7 @@ PEERDIR( ydb/library/actors/core ydb/library/folder_service ydb/library/folder_service/proto + ydb/public/sdk/cpp/src/library/grpc/client ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/formats/arrow/csv/table/table.cpp b/ydb/library/formats/arrow/csv/table/table.cpp index 0364acfc70ed..f6dda086d609 100644 --- a/ydb/library/formats/arrow/csv/table/table.cpp +++ b/ydb/library/formats/arrow/csv/table/table.cpp @@ -3,7 +3,7 @@ namespace NKikimr::NFormats { -arrow::Result TArrowCSVTable::Create(const TVector& columns, bool header) { +arrow::Result TArrowCSVTable::Create(const std::vector& columns, bool header) { TVector errors; TColummns convertedColumns; convertedColumns.reserve(columns.size()); @@ -19,7 +19,7 @@ arrow::Result TArrowCSVTable::Create(const TVector -#include +#include namespace NKikimr::NFormats { @@ -10,7 +10,7 @@ class TArrowCSVTable: public TArrowCSV { public: /// If header is true read column names from first line after skipRows. Parse columns as strings in this case. /// @note It's possible to skip header with skipRows and use typed columns instead. - static arrow::Result Create(const TVector& columns, bool header = false); + static arrow::Result Create(const std::vector& columns, bool header = false); private: static NYdb::TTypeParser ExtractType(const NYdb::TType& type); diff --git a/ydb/library/formats/arrow/csv/table/ya.make b/ydb/library/formats/arrow/csv/table/ya.make index 26278cef8ff7..4d996a7e9a47 100644 --- a/ydb/library/formats/arrow/csv/table/ya.make +++ b/ydb/library/formats/arrow/csv/table/ya.make @@ -6,7 +6,7 @@ SRCS( PEERDIR( ydb/library/formats/arrow/csv/converter - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/library/grpc/actor_client/grpc_service_client.h b/ydb/library/grpc/actor_client/grpc_service_client.h index 4c957a505e79..f661a01a512f 100644 --- a/ydb/library/grpc/actor_client/grpc_service_client.h +++ b/ydb/library/grpc/actor_client/grpc_service_client.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include "grpc_service_settings.h" diff --git a/ydb/library/grpc/actor_client/ya.make b/ydb/library/grpc/actor_client/ya.make index c1f69c7bf3d7..d04eae019f20 100644 --- a/ydb/library/grpc/actor_client/ya.make +++ b/ydb/library/grpc/actor_client/ya.make @@ -10,7 +10,7 @@ PEERDIR( ydb/library/actors/core ydb/core/util library/cpp/digest/crc32c - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/services @@ -19,7 +19,7 @@ PEERDIR( #ydb/library/ycloud/api #ydb/library/actors/core #library/cpp/digest/crc32c - #ydb/library/grpc/client + #ydb/public/sdk/cpp/src/library/grpc/client #library/cpp/json #ydb/core/base #ydb/library/services diff --git a/ydb/library/grpc/client/grpc_client_low.h b/ydb/library/grpc/client/grpc_client_low.h index 233f9fbab39b..9c9fbc422975 100644 --- a/ydb/library/grpc/client/grpc_client_low.h +++ b/ydb/library/grpc/client/grpc_client_low.h @@ -1,1434 +1 @@ -#pragma once - -#include "grpc_common.h" - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * This file contains low level logic for grpc - * This file should not be used in high level code without special reason - */ -namespace NYdbGrpc { - -const size_t DEFAULT_NUM_THREADS = 2; - -//////////////////////////////////////////////////////////////////////////////// - -void EnableGRpcTracing(); - -//////////////////////////////////////////////////////////////////////////////// - -struct TTcpKeepAliveSettings { - bool Enabled; - size_t Idle; - size_t Count; - size_t Interval; -}; - -//////////////////////////////////////////////////////////////////////////////// - -// Common interface used to execute action from grpc cq routine -class IQueueClientEvent { -public: - virtual ~IQueueClientEvent() = default; - - //! Execute an action defined by implementation - virtual bool Execute(bool ok) = 0; - - //! Finish and destroy event - virtual void Destroy() = 0; -}; - -// Implementation of IQueueClientEvent that reduces allocations -template -class TQueueClientFixedEvent : private IQueueClientEvent { - using TCallback = void (TSelf::*)(bool); - -public: - TQueueClientFixedEvent(TSelf* self, TCallback callback) - : Self(self) - , Callback(callback) - { } - - IQueueClientEvent* Prepare() { - Self->Ref(); - return this; - } - -private: - bool Execute(bool ok) override { - ((*Self).*Callback)(ok); - return false; - } - - void Destroy() override { - Self->UnRef(); - } - -private: - TSelf* const Self; - TCallback const Callback; -}; - -class IQueueClientContext; -using IQueueClientContextPtr = std::shared_ptr; - -// Provider of IQueueClientContext instances -class IQueueClientContextProvider { -public: - virtual ~IQueueClientContextProvider() = default; - - virtual IQueueClientContextPtr CreateContext() = 0; -}; - -// Activity context for a low-level client -class IQueueClientContext : public IQueueClientContextProvider { -public: - virtual ~IQueueClientContext() = default; - - //! Returns CompletionQueue associated with the client - virtual grpc::CompletionQueue* CompletionQueue() = 0; - - //! Returns true if context has been cancelled - virtual bool IsCancelled() const = 0; - - //! Tries to cancel context, calling all registered callbacks - virtual bool Cancel() = 0; - - //! Subscribes callback to cancellation - // - // Note there's no way to unsubscribe, if subscription is temporary - // make sure you create a new context with CreateContext and release - // it as soon as it's no longer needed. - virtual void SubscribeCancel(std::function callback) = 0; - - //! Subscribes callback to cancellation - // - // This alias is for compatibility with older code. - void SubscribeStop(std::function callback) { - SubscribeCancel(std::move(callback)); - } -}; - -// Represents grpc status and error message string -struct TGrpcStatus { - TString Msg; - TString Details; - int GRpcStatusCode; - bool InternalError; - std::multimap ServerTrailingMetadata; - - TGrpcStatus() - : GRpcStatusCode(grpc::StatusCode::OK) - , InternalError(false) - { } - - TGrpcStatus(TString msg, int statusCode, bool internalError) - : Msg(std::move(msg)) - , GRpcStatusCode(statusCode) - , InternalError(internalError) - { } - - TGrpcStatus(grpc::StatusCode status, TString msg, TString details = {}) - : Msg(std::move(msg)) - , Details(std::move(details)) - , GRpcStatusCode(status) - , InternalError(false) - { } - - TGrpcStatus(const grpc::Status& status) - : TGrpcStatus(status.error_code(), TString(status.error_message()), TString(status.error_details())) - { } - - TGrpcStatus& operator=(const grpc::Status& status) { - Msg = TString(status.error_message()); - Details = TString(status.error_details()); - GRpcStatusCode = status.error_code(); - InternalError = false; - return *this; - } - - static TGrpcStatus Internal(TString msg) { - return { std::move(msg), -1, true }; - } - - bool Ok() const { - return !InternalError && GRpcStatusCode == grpc::StatusCode::OK; - } - - TStringBuilder ToDebugString() const { - TStringBuilder ret; - ret << "gRpcStatusCode: " << GRpcStatusCode; - if(!Ok()) - ret << ", Msg: " << Msg << ", Details: " << Details << ", InternalError: " << InternalError; - return ret; - } -}; - -bool inline IsGRpcStatusGood(const TGrpcStatus& status) { - return status.Ok(); -} - -// Response callback type - this callback will be called when request is finished -// (or after getting each chunk in case of streaming mode) -template -using TResponseCallback = std::function; - -template -using TAdvancedResponseCallback = std::function; - -// Call associated metadata -struct TCallMeta { - std::shared_ptr CallCredentials; - std::vector> Aux; - std::variant Timeout; // timeout as duration from now or time point in future -}; - -class TGRpcRequestProcessorCommon { -protected: - void ApplyMeta(const TCallMeta& meta) { - for (const auto& rec : meta.Aux) { - Context.AddMetadata(rec.first, rec.second); - } - if (meta.CallCredentials) { - Context.set_credentials(meta.CallCredentials); - } - if (const TDuration* timeout = std::get_if(&meta.Timeout)) { - if (*timeout) { - auto deadline = gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(timeout->MicroSeconds(), GPR_TIMESPAN)); - Context.set_deadline(deadline); - } - } else if (const TInstant* deadline = std::get_if(&meta.Timeout)) { - if (*deadline) { - Context.set_deadline(gpr_time_from_micros(deadline->MicroSeconds(), GPR_CLOCK_MONOTONIC)); - } - } - } - - void GetInitialMetadata(std::unordered_multimap* metadata) { - for (const auto& [key, value] : Context.GetServerInitialMetadata()) { - metadata->emplace( - TString(key.begin(), key.end()), - TString(value.begin(), value.end()) - ); - } - } - - grpc::Status Status; - grpc::ClientContext Context; - std::shared_ptr LocalContext; -}; - -template -class TSimpleRequestProcessor - : public TThrRefBase - , public IQueueClientEvent - , public TGRpcRequestProcessorCommon { - using TAsyncReaderPtr = std::unique_ptr>; - template friend class TServiceConnection; -public: - using TPtr = TIntrusivePtr; - using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); - - explicit TSimpleRequestProcessor(TResponseCallback&& callback) - : Callback_(std::move(callback)) - { } - - ~TSimpleRequestProcessor() { - if (!Replied_ && Callback_) { - Callback_(TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); - Callback_ = nullptr; // free resources as early as possible - } - } - - bool Execute(bool ok) override { - { - std::unique_lock guard(Mutex_); - LocalContext.reset(); - } - TGrpcStatus status; - if (ok) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - Replied_ = true; - Callback_(std::move(status), std::move(Reply_)); - Callback_ = nullptr; // free resources as early as possible - return false; - } - - void Destroy() override { - UnRef(); - } - -private: - IQueueClientEvent* FinishedEvent() { - Ref(); - return this; - } - - void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - Replied_ = true; - Callback_(TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); - Callback_ = nullptr; - return; - } - { - std::unique_lock guard(Mutex_); - LocalContext = context; - Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); - Reader_->Finish(&Reply_, &Status, FinishedEvent()); - } - context->SubscribeStop([self = TPtr(this)] { - self->Stop(); - }); - } - - void Stop() { - Context.TryCancel(); - } - - TResponseCallback Callback_; - TResponse Reply_; - std::mutex Mutex_; - TAsyncReaderPtr Reader_; - - bool Replied_ = false; -}; - -template -class TAdvancedRequestProcessor - : public TThrRefBase - , public IQueueClientEvent - , public TGRpcRequestProcessorCommon { - using TAsyncReaderPtr = std::unique_ptr>; - template friend class TServiceConnection; -public: - using TPtr = TIntrusivePtr; - using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); - - explicit TAdvancedRequestProcessor(TAdvancedResponseCallback&& callback) - : Callback_(std::move(callback)) - { } - - ~TAdvancedRequestProcessor() { - if (!Replied_ && Callback_) { - Callback_(Context, TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); - Callback_ = nullptr; // free resources as early as possible - } - } - - bool Execute(bool ok) override { - { - std::unique_lock guard(Mutex_); - LocalContext.reset(); - } - TGrpcStatus status; - if (ok) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - Replied_ = true; - Callback_(Context, std::move(status), std::move(Reply_)); - Callback_ = nullptr; // free resources as early as possible - return false; - } - - void Destroy() override { - UnRef(); - } - -private: - IQueueClientEvent* FinishedEvent() { - Ref(); - return this; - } - - void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - Replied_ = true; - Callback_(Context, TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); - Callback_ = nullptr; - return; - } - { - std::unique_lock guard(Mutex_); - LocalContext = context; - Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); - Reader_->Finish(&Reply_, &Status, FinishedEvent()); - } - context->SubscribeStop([self = TPtr(this)] { - self->Stop(); - }); - } - - void Stop() { - Context.TryCancel(); - } - - TAdvancedResponseCallback Callback_; - TResponse Reply_; - std::mutex Mutex_; - TAsyncReaderPtr Reader_; - - bool Replied_ = false; -}; - -class IStreamRequestCtrl : public TThrRefBase { -public: - using TPtr = TIntrusivePtr; - - /** - * Asynchronously cancel the request - */ - virtual void Cancel() = 0; -}; - -template -class IStreamRequestReadProcessor : public IStreamRequestCtrl { -public: - using TPtr = TIntrusivePtr; - using TReadCallback = std::function; - - /** - * Scheduled initial server metadata read from the stream - */ - virtual void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) = 0; - - /** - * Scheduled response read from the stream - * Callback will be called with the status if it failed - * Only one Read or Finish call may be active at a time - */ - virtual void Read(TResponse* response, TReadCallback callback) = 0; - - /** - * Stop reading and gracefully finish the stream - * Only one Read or Finish call may be active at a time - */ - virtual void Finish(TReadCallback callback) = 0; - - /** - * Additional callback to be called when stream has finished - */ - virtual void AddFinishedCallback(TReadCallback callback) = 0; -}; - -template -class IStreamRequestReadWriteProcessor : public IStreamRequestReadProcessor { -public: - using TPtr = TIntrusivePtr; - using TWriteCallback = std::function; - - /** - * Scheduled request write to the stream - */ - virtual void Write(TRequest&& request, TWriteCallback callback = { }) = 0; -}; - -class TGRpcKeepAliveSocketMutator; - -namespace NImpl { -grpc_socket_mutator* CreateGRpcKeepAliveSocketMutator(const TTcpKeepAliveSettings& TcpKeepAliveSettings_); -} // NImpl - -// Class to hold stubs allocated on channel. -// It is poor documented part of grpc. See KIKIMR-6109 and comment to this commit - -// Stub holds shared_ptr, so we can destroy this holder even if -// request processor using stub -class TStubsHolder : public TNonCopyable { - using TypeInfoRef = std::reference_wrapper; - - struct THasher { - std::size_t operator()(TypeInfoRef code) const { - return code.get().hash_code(); - } - }; - - struct TEqualTo { - bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { - return lhs.get() == rhs.get(); - } - }; -public: - TStubsHolder(std::shared_ptr channel) - : ChannelInterface_(channel) - {} - - // Returns true if channel can't be used to perform request now - bool IsChannelBroken() const { - auto state = ChannelInterface_->GetState(false); - return state == GRPC_CHANNEL_SHUTDOWN || - state == GRPC_CHANNEL_TRANSIENT_FAILURE; - } - - template - std::shared_ptr GetOrCreateStub() { - const auto& stubId = typeid(TStub); - { - std::shared_lock readGuard(RWMutex_); - const auto it = Stubs_.find(stubId); - if (it != Stubs_.end()) { - return std::static_pointer_cast(it->second); - } - } - { - std::unique_lock writeGuard(RWMutex_); - auto it = Stubs_.emplace(stubId, nullptr); - if (!it.second) { - return std::static_pointer_cast(it.first->second); - } else { - it.first->second = std::make_shared(ChannelInterface_); - return std::static_pointer_cast(it.first->second); - } - } - } - - const TInstant& GetLastUseTime() const { - return LastUsed_; - } - - void SetLastUseTime(const TInstant& time) { - LastUsed_ = time; - } -private: - TInstant LastUsed_ = Now(); - std::shared_mutex RWMutex_; - std::unordered_map, THasher, TEqualTo> Stubs_; - std::shared_ptr ChannelInterface_; -}; - -class TChannelPool { -public: - TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, const TDuration& expireTime = TDuration::Minutes(6)); - //Allows to CreateStub from TStubsHolder under lock - //The callback will be called just during GetStubsHolderLocked call - void GetStubsHolderLocked(const TString& channelId, const TGRpcClientConfig& config, std::function cb); - void DeleteChannel(const TString& channelId); - void DeleteExpiredStubsHolders(); -private: - std::shared_mutex RWMutex_; - std::unordered_map Pool_; - std::multimap LastUsedQueue_; - TTcpKeepAliveSettings TcpKeepAliveSettings_; - TDuration ExpireTime_; - TDuration UpdateReUseTime_; - void EraseFromQueueByTime(const TInstant& lastUseTime, const TString& channelId); -}; - -template -using TStreamReaderCallback = std::function::TPtr)>; - -template -class TStreamRequestReadProcessor - : public IStreamRequestReadProcessor - , public TGRpcRequestProcessorCommon { - template friend class TServiceConnection; -public: - using TSelf = TStreamRequestReadProcessor; - using TAsyncReaderPtr = std::unique_ptr>; - using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*, void*); - using TReaderCallback = TStreamReaderCallback; - using TPtr = TIntrusivePtr; - using TBase = IStreamRequestReadProcessor; - using TReadCallback = typename TBase::TReadCallback; - - explicit TStreamRequestReadProcessor(TReaderCallback&& callback) - : Callback(std::move(callback)) - { - Y_ABORT_UNLESS(Callback, "Missing connected callback"); - } - - void Cancel() override { - Context.TryCancel(); - - { - std::unique_lock guard(Mutex); - Cancelled = true; - if (Started && !ReadFinished) { - if (!ReadActive) { - ReadFinished = true; - } - if (ReadFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } - } - } - - void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished && !HasInitialMetadata) { - ReadActive = true; - ReadCallback = std::move(callback); - InitialMetadata = metadata; - if (!ReadFinished) { - Stream->ReadInitialMetadata(OnReadDoneTag.Prepare()); - } - return; - } - if (!HasInitialMetadata) { - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } else { - GetInitialMetadata(metadata); - } - } - - callback(std::move(status)); - } - - void Read(TResponse* message, TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - ReadCallback = std::move(callback); - if (!ReadFinished) { - Stream->Read(message, OnReadDoneTag.Prepare()); - } - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - } - - callback(std::move(status)); - } - - void Finish(TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - FinishCallback = std::move(callback); - if (!ReadFinished) { - ReadFinished = true; - } - Stream->Finish(&Status, OnFinishedTag.Prepare()); - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - callback(std::move(status)); - } - - void AddFinishedCallback(TReadCallback callback) override { - Y_ABORT_UNLESS(callback, "Unexpected empty callback"); - - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - if (!Finished) { - FinishedCallbacks.emplace_back().swap(callback); - return; - } - - if (FinishedOk) { - status = Status; - } else if (Cancelled) { - status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - callback(std::move(status)); - } - -private: - void Start(TStub& stub, const TRequest& request, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - auto callback = std::move(Callback); - TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); - callback(std::move(status), nullptr); - return; - } - - { - std::unique_lock guard(Mutex); - LocalContext = context; - Stream = (stub.*asyncRequest)(&Context, request, context->CompletionQueue(), OnStartDoneTag.Prepare()); - } - - context->SubscribeStop([self = TPtr(this)] { - self->Cancel(); - }); - } - - void OnReadDone(bool ok) { - TGrpcStatus status; - TReadCallback callback; - std::unordered_multimap* initialMetadata = nullptr; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(ReadActive, "Unexpected Read done callback"); - Y_ABORT_UNLESS(!ReadFinished, "Unexpected ReadFinished flag"); - - if (!ok || Cancelled) { - ReadFinished = true; - - Stream->Finish(&Status, OnFinishedTag.Prepare()); - if (!ok) { - // Keep ReadActive=true, so callback is called - // after the call is finished with an error - return; - } - } - - callback = std::move(ReadCallback); - ReadCallback = nullptr; - ReadActive = false; - initialMetadata = InitialMetadata; - InitialMetadata = nullptr; - HasInitialMetadata = true; - } - - if (initialMetadata) { - GetInitialMetadata(initialMetadata); - } - - callback(std::move(status)); - } - - void OnStartDone(bool ok) { - TReaderCallback callback; - - { - std::unique_lock guard(Mutex); - Started = true; - if (!ok || Cancelled) { - ReadFinished = true; - Stream->Finish(&Status, OnFinishedTag.Prepare()); - return; - } - callback = std::move(Callback); - Callback = nullptr; - } - - callback({ }, typename TBase::TPtr(this)); - } - - void OnFinished(bool ok) { - TGrpcStatus status; - std::vector finishedCallbacks; - TReaderCallback startCallback; - TReadCallback readCallback; - TReadCallback finishCallback; - - { - std::unique_lock guard(Mutex); - - Finished = true; - FinishedOk = ok; - LocalContext.reset(); - - if (ok) { - status = Status; - } else if (Cancelled) { - status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - - finishedCallbacks.swap(FinishedCallbacks); - - if (Callback) { - Y_ABORT_UNLESS(!ReadActive); - startCallback = std::move(Callback); - Callback = nullptr; - } else if (ReadActive) { - if (ReadCallback) { - readCallback = std::move(ReadCallback); - ReadCallback = nullptr; - } else { - finishCallback = std::move(FinishCallback); - FinishCallback = nullptr; - } - ReadActive = false; - } - } - - for (auto& finishedCallback : finishedCallbacks) { - auto statusCopy = status; - finishedCallback(std::move(statusCopy)); - } - - if (startCallback) { - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); - } - startCallback(std::move(status), nullptr); - } else if (readCallback) { - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - for (const auto& [name, value] : Context.GetServerTrailingMetadata()) { - status.ServerTrailingMetadata.emplace( - TString(name.begin(), name.end()), - TString(value.begin(), value.end())); - } - } - readCallback(std::move(status)); - } else if (finishCallback) { - finishCallback(std::move(status)); - } - } - - TReaderCallback Callback; - TAsyncReaderPtr Stream; - using TFixedEvent = TQueueClientFixedEvent; - std::mutex Mutex; - TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; - TFixedEvent OnStartDoneTag = { this, &TSelf::OnStartDone }; - TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; - - TReadCallback ReadCallback; - TReadCallback FinishCallback; - std::vector FinishedCallbacks; - std::unordered_multimap* InitialMetadata = nullptr; - bool Started = false; - bool HasInitialMetadata = false; - bool ReadActive = false; - bool ReadFinished = false; - bool Finished = false; - bool Cancelled = false; - bool FinishedOk = false; -}; - -template -using TStreamConnectedCallback = std::function::TPtr)>; - -template -class TStreamRequestReadWriteProcessor - : public IStreamRequestReadWriteProcessor - , public TGRpcRequestProcessorCommon { -public: - using TSelf = TStreamRequestReadWriteProcessor; - using TBase = IStreamRequestReadWriteProcessor; - using TPtr = TIntrusivePtr; - using TConnectedCallback = TStreamConnectedCallback; - using TReadCallback = typename TBase::TReadCallback; - using TWriteCallback = typename TBase::TWriteCallback; - using TAsyncReaderWriterPtr = std::unique_ptr>; - using TAsyncRequest = TAsyncReaderWriterPtr (TStub::*)(grpc::ClientContext*, grpc::CompletionQueue*, void*); - - explicit TStreamRequestReadWriteProcessor(TConnectedCallback&& callback) - : ConnectedCallback(std::move(callback)) - { - Y_ABORT_UNLESS(ConnectedCallback, "Missing connected callback"); - } - - void Cancel() override { - Context.TryCancel(); - - { - std::unique_lock guard(Mutex); - Cancelled = true; - if (Started && !(ReadFinished && WriteFinished)) { - if (!ReadActive) { - ReadFinished = true; - } - if (!WriteActive) { - WriteFinished = true; - } - if (ReadFinished && WriteFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } - } - } - - void Write(TRequest&& request, TWriteCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - if (Cancelled || ReadFinished || WriteFinished) { - status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); - } else if (WriteActive) { - auto& item = WriteQueue.emplace_back(); - item.Callback.swap(callback); - item.Request.Swap(&request); - } else { - WriteActive = true; - WriteCallback.swap(callback); - Stream->Write(request, OnWriteDoneTag.Prepare()); - } - } - - if (!status.Ok() && callback) { - callback(std::move(status)); - } - } - - void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished && !HasInitialMetadata) { - ReadActive = true; - ReadCallback = std::move(callback); - InitialMetadata = metadata; - if (!ReadFinished) { - Stream->ReadInitialMetadata(OnReadDoneTag.Prepare()); - } - return; - } - if (!HasInitialMetadata) { - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } else { - GetInitialMetadata(metadata); - } - } - - callback(std::move(status)); - } - - void Read(TResponse* message, TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - ReadCallback = std::move(callback); - if (!ReadFinished) { - Stream->Read(message, OnReadDoneTag.Prepare()); - } - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - } - - callback(std::move(status)); - } - - void Finish(TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - FinishCallback = std::move(callback); - if (!ReadFinished) { - ReadFinished = true; - if (!WriteActive) { - WriteFinished = true; - } - if (WriteFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - callback(std::move(status)); - } - - void AddFinishedCallback(TReadCallback callback) override { - Y_ABORT_UNLESS(callback, "Unexpected empty callback"); - - TGrpcStatus status; - - { - std::unique_lock guard(Mutex); - if (!Finished) { - FinishedCallbacks.emplace_back().swap(callback); - return; - } - - if (FinishedOk) { - status = Status; - } else if (Cancelled) { - status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - callback(std::move(status)); - } - -private: - template friend class TServiceConnection; - - void Start(TStub& stub, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - auto callback = std::move(ConnectedCallback); - TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); - callback(std::move(status), nullptr); - return; - } - - { - std::unique_lock guard(Mutex); - LocalContext = context; - Stream = (stub.*asyncRequest)(&Context, context->CompletionQueue(), OnConnectedTag.Prepare()); - } - - context->SubscribeStop([self = TPtr(this)] { - self->Cancel(); - }); - } - -private: - void OnConnected(bool ok) { - TConnectedCallback callback; - - { - std::unique_lock guard(Mutex); - Started = true; - if (!ok || Cancelled) { - ReadFinished = true; - WriteFinished = true; - Stream->Finish(&Status, OnFinishedTag.Prepare()); - return; - } - - callback = std::move(ConnectedCallback); - ConnectedCallback = nullptr; - } - - callback({ }, typename TBase::TPtr(this)); - } - - void OnReadDone(bool ok) { - TGrpcStatus status; - TReadCallback callback; - std::unordered_multimap* initialMetadata = nullptr; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(ReadActive, "Unexpected Read done callback"); - Y_ABORT_UNLESS(!ReadFinished, "Unexpected ReadFinished flag"); - - if (!ok || Cancelled) { - ReadFinished = true; - if (!WriteActive) { - WriteFinished = true; - } - if (WriteFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - if (!ok) { - // Keep ReadActive=true, so callback is called - // after the call is finished with an error - return; - } - } - - callback = std::move(ReadCallback); - ReadCallback = nullptr; - ReadActive = false; - initialMetadata = InitialMetadata; - InitialMetadata = nullptr; - HasInitialMetadata = true; - } - - if (initialMetadata) { - GetInitialMetadata(initialMetadata); - } - - callback(std::move(status)); - } - - void OnWriteDone(bool ok) { - TWriteCallback okCallback; - - { - std::unique_lock guard(Mutex); - Y_ABORT_UNLESS(WriteActive, "Unexpected Write done callback"); - Y_ABORT_UNLESS(!WriteFinished, "Unexpected WriteFinished flag"); - - if (ok) { - okCallback.swap(WriteCallback); - } else if (WriteCallback) { - // Put callback back on the queue until OnFinished - auto& item = WriteQueue.emplace_front(); - item.Callback.swap(WriteCallback); - } - - if (!ok || Cancelled) { - WriteActive = false; - WriteFinished = true; - if (ReadFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } else if (!WriteQueue.empty()) { - WriteCallback.swap(WriteQueue.front().Callback); - Stream->Write(WriteQueue.front().Request, OnWriteDoneTag.Prepare()); - WriteQueue.pop_front(); - } else { - WriteActive = false; - if (ReadFinished) { - WriteFinished = true; - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } - } - - if (okCallback) { - okCallback(TGrpcStatus()); - } - } - - void OnFinished(bool ok) { - TGrpcStatus status; - std::deque writesDropped; - std::vector finishedCallbacks; - TConnectedCallback connectedCallback; - TReadCallback readCallback; - TReadCallback finishCallback; - - { - std::unique_lock guard(Mutex); - Finished = true; - FinishedOk = ok; - LocalContext.reset(); - - if (ok) { - status = Status; - } else if (Cancelled) { - status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - - writesDropped.swap(WriteQueue); - finishedCallbacks.swap(FinishedCallbacks); - - if (ConnectedCallback) { - Y_ABORT_UNLESS(!ReadActive); - connectedCallback = std::move(ConnectedCallback); - ConnectedCallback = nullptr; - } else if (ReadActive) { - if (ReadCallback) { - readCallback = std::move(ReadCallback); - ReadCallback = nullptr; - } else { - finishCallback = std::move(FinishCallback); - FinishCallback = nullptr; - } - ReadActive = false; - } - } - - for (auto& item : writesDropped) { - if (item.Callback) { - TGrpcStatus writeStatus = status; - if (writeStatus.Ok()) { - writeStatus = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); - } - item.Callback(std::move(writeStatus)); - } - } - - for (auto& finishedCallback : finishedCallbacks) { - TGrpcStatus statusCopy = status; - finishedCallback(std::move(statusCopy)); - } - - if (connectedCallback) { - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); - } - connectedCallback(std::move(status), nullptr); - } else if (readCallback) { - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - for (const auto& [name, value] : Context.GetServerTrailingMetadata()) { - status.ServerTrailingMetadata.emplace( - TString(name.begin(), name.end()), - TString(value.begin(), value.end())); - } - } - readCallback(std::move(status)); - } else if (finishCallback) { - finishCallback(std::move(status)); - } - } - -private: - struct TWriteItem { - TWriteCallback Callback; - TRequest Request; - }; - -private: - using TFixedEvent = TQueueClientFixedEvent; - - TFixedEvent OnConnectedTag = { this, &TSelf::OnConnected }; - TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; - TFixedEvent OnWriteDoneTag = { this, &TSelf::OnWriteDone }; - TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; - -private: - std::mutex Mutex; - TAsyncReaderWriterPtr Stream; - TConnectedCallback ConnectedCallback; - TReadCallback ReadCallback; - TReadCallback FinishCallback; - std::vector FinishedCallbacks; - std::deque WriteQueue; - TWriteCallback WriteCallback; - std::unordered_multimap* InitialMetadata = nullptr; - bool Started = false; - bool HasInitialMetadata = false; - bool ReadActive = false; - bool ReadFinished = false; - bool WriteActive = false; - bool WriteFinished = false; - bool Finished = false; - bool Cancelled = false; - bool FinishedOk = false; -}; - -class TGRpcClientLow; - -template -class TServiceConnection { - using TStub = typename TGRpcService::Stub; - friend class TGRpcClientLow; - -public: - /* - * Start simple request - */ - template - void DoRequest(const TRequest& request, - TResponseCallback callback, - typename TSimpleRequestProcessor::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, - IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); - } - - /* - * Start simple request - */ - template - void DoAdvancedRequest(const TRequest& request, - TAdvancedResponseCallback callback, - typename TAdvancedRequestProcessor::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, - IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); - } - - /* - * Start bidirectional streamming - */ - template - void DoStreamRequest(TStreamConnectedCallback callback, - typename TStreamRequestReadWriteProcessor::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, - IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, std::move(asyncRequest), provider ? provider : Provider_); - } - - /* - * Start streaming response reading (one request, many responses) - */ - template - void DoStreamRequest(const TRequest& request, - TStreamReaderCallback callback, - typename TStreamRequestReadProcessor::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, - IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, request, std::move(asyncRequest), provider ? provider : Provider_); - } - -private: - TServiceConnection(std::shared_ptr ci, - IQueueClientContextProvider* provider) - : Stub_(TGRpcService::NewStub(ci)) - , Provider_(provider) - { - Y_ABORT_UNLESS(Provider_, "Connection does not have a queue provider"); - } - - TServiceConnection(TStubsHolder& holder, - IQueueClientContextProvider* provider) - : Stub_(holder.GetOrCreateStub()) - , Provider_(provider) - { - Y_ABORT_UNLESS(Provider_, "Connection does not have a queue provider"); - } - - std::shared_ptr Stub_; - IQueueClientContextProvider* Provider_; -}; - -class TGRpcClientLow - : public IQueueClientContextProvider -{ - class TContextImpl; - friend class TContextImpl; - - enum ECqState : TAtomicBase { - WORKING = 0, - STOP_SILENT = 1, - STOP_EXPLICIT = 2, - }; - -public: - explicit TGRpcClientLow(size_t numWorkerThread = DEFAULT_NUM_THREADS, bool useCompletionQueuePerThread = false); - ~TGRpcClientLow(); - - // Tries to stop all currently running requests (via their stop callbacks) - // Will shutdown CQ and drain events once all requests have finished - // No new requests may be started after this call - void Stop(bool wait = false); - - // Waits until all currently running requests finish execution - void WaitIdle(); - - inline bool IsStopping() const { - switch (GetCqState()) { - case WORKING: - return false; - case STOP_SILENT: - case STOP_EXPLICIT: - return true; - } - - Y_UNREACHABLE(); - } - - IQueueClientContextPtr CreateContext() override; - - template - std::unique_ptr> CreateGRpcServiceConnection(const TGRpcClientConfig& config) { - return std::unique_ptr>(new TServiceConnection(CreateChannelInterface(config), this)); - } - - template - std::unique_ptr> CreateGRpcServiceConnection(const TGRpcClientConfig& config, const TTcpKeepAliveSettings& keepAlive) { - auto mutator = NImpl::CreateGRpcKeepAliveSocketMutator(keepAlive); - // will be destroyed inside grpc - return std::unique_ptr>(new TServiceConnection(CreateChannelInterface(config, mutator), this)); - } - - template - std::unique_ptr> CreateGRpcServiceConnection(TStubsHolder& holder) { - return std::unique_ptr>(new TServiceConnection(holder, this)); - } - - // Tests only, not thread-safe - void AddWorkerThreadForTest(); - -private: - using IThreadRef = std::unique_ptr; - using CompletionQueueRef = std::unique_ptr; - void Init(size_t numWorkerThread); - - inline ECqState GetCqState() const { return (ECqState) AtomicGet(CqState_); } - inline void SetCqState(ECqState state) { AtomicSet(CqState_, state); } - - void StopInternal(bool silent); - void WaitInternal(); - - void ForgetContext(TContextImpl* context); - -private: - bool UseCompletionQueuePerThread_; - std::vector CQS_; - std::vector WorkerThreads_; - TAtomic CqState_ = -1; - - std::mutex Mtx_; - std::condition_variable ContextsEmpty_; - std::unordered_set Contexts_; - - std::mutex JoinMutex_; -}; - -} // namespace NGRpc +#include diff --git a/ydb/library/grpc/client/grpc_common.h b/ydb/library/grpc/client/grpc_common.h index bf009d86b126..d341b96ca30d 100644 --- a/ydb/library/grpc/client/grpc_common.h +++ b/ydb/library/grpc/client/grpc_common.h @@ -1,83 +1 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include - -namespace NYdbGrpc { - -struct TGRpcClientConfig { - TString Locator; // format host:port - TDuration Timeout = TDuration::Max(); // request timeout - ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size - ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests - ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests - ui32 MaxInFlight = 0; - bool EnableSsl = false; - grpc::SslCredentialsOptions SslCredentials; - grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; - ui64 MemQuota = 0; - std::unordered_map StringChannelParams; - std::unordered_map IntChannelParams; - TString LoadBalancingPolicy = { }; - TString SslTargetNameOverride = { }; - - TGRpcClientConfig() = default; - TGRpcClientConfig(const TGRpcClientConfig&) = default; - TGRpcClientConfig(TGRpcClientConfig&&) = default; - TGRpcClientConfig& operator=(const TGRpcClientConfig&) = default; - TGRpcClientConfig& operator=(TGRpcClientConfig&&) = default; - - TGRpcClientConfig(const TString& locator, TDuration timeout = TDuration::Max(), - ui64 maxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, const TString& caCert = "", const TString& clientCert = "", - const TString& clientPrivateKey = "", grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) - : Locator(locator) - , Timeout(timeout) - , MaxMessageSize(maxMessageSize) - , MaxInFlight(maxInFlight) - , EnableSsl(enableSsl) - , SslCredentials{.pem_root_certs = caCert, .pem_private_key = clientPrivateKey, .pem_cert_chain = clientCert} - , CompressionAlgoritm(compressionAlgorithm) - {} -}; - -inline std::shared_ptr CreateChannelInterface(const TGRpcClientConfig& config, grpc_socket_mutator* mutator = nullptr){ - grpc::ChannelArguments args; - args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); - args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); - args.SetCompressionAlgorithm(config.CompressionAlgoritm); - - for (const auto& kvp: config.StringChannelParams) { - args.SetString(kvp.first, kvp.second); - } - - for (const auto& kvp: config.IntChannelParams) { - args.SetInt(kvp.first, kvp.second); - } - - if (config.MemQuota) { - grpc::ResourceQuota quota; - quota.Resize(config.MemQuota); - args.SetResourceQuota(quota); - } - if (mutator) { - args.SetSocketMutator(mutator); - } - if (!config.LoadBalancingPolicy.empty()) { - args.SetLoadBalancingPolicyName(config.LoadBalancingPolicy); - } - if (!config.SslTargetNameOverride.empty()) { - args.SetSslTargetNameOverride(config.SslTargetNameOverride); - } - if (config.EnableSsl || config.SslCredentials.pem_root_certs) { - return grpc::CreateCustomChannel(config.Locator, grpc::SslCredentials(config.SslCredentials), args); - } else { - return grpc::CreateCustomChannel(config.Locator, grpc::InsecureChannelCredentials(), args); - } -} - -} // namespace NGRpc +#include diff --git a/ydb/library/grpc/client/ut/ya.make b/ydb/library/grpc/client/ut/ya.make deleted file mode 100644 index a896a7d567bd..000000000000 --- a/ydb/library/grpc/client/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(ydb/library/grpc/client) - -SRCS( - grpc_client_low_ut.cpp -) - -END() diff --git a/ydb/library/grpc/client/ya.make b/ydb/library/grpc/client/ya.make index a963a42abef3..757b54f44b65 100644 --- a/ydb/library/grpc/client/ya.make +++ b/ydb/library/grpc/client/ya.make @@ -1,16 +1,12 @@ LIBRARY() SRCS( - grpc_client_low.cpp + grpc_client_low.h + grpc_common.h ) PEERDIR( - contrib/libs/grpc - library/cpp/deprecated/atomic + ydb/public/sdk/cpp/src/library/grpc/client ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/library/grpc/common/constants.h b/ydb/library/grpc/common/constants.h index cf90c9589d4e..84c96b6302e1 100644 --- a/ydb/library/grpc/common/constants.h +++ b/ydb/library/grpc/common/constants.h @@ -1,7 +1 @@ -#pragma once - -namespace NYdbGrpc { - -constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; - -} +#include diff --git a/ydb/library/grpc/common/ya.make b/ydb/library/grpc/common/ya.make index 63b8acd0a2c3..6126c52bb3ce 100644 --- a/ydb/library/grpc/common/ya.make +++ b/ydb/library/grpc/common/ya.make @@ -1,8 +1,8 @@ LIBRARY() SRCS( - time_point.h constants.h + time_point.h ) PEERDIR( diff --git a/ydb/library/grpc/server/grpc_server.h b/ydb/library/grpc/server/grpc_server.h index 459c94d6167d..f09dc3ecaf6f 100644 --- a/ydb/library/grpc/server/grpc_server.h +++ b/ydb/library/grpc/server/grpc_server.h @@ -3,7 +3,7 @@ #include "grpc_request_base.h" #include "logger.h" -#include +#include #include #include diff --git a/ydb/library/ncloud/api/events.h b/ydb/library/ncloud/api/events.h index dd9e71e50045..6dbaad7c4688 100644 --- a/ydb/library/ncloud/api/events.h +++ b/ydb/library/ncloud/api/events.h @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace NNebiusCloud { diff --git a/ydb/library/ncloud/api/ya.make b/ydb/library/ncloud/api/ya.make index 13ef0c6858ef..2d66f104ec3b 100644 --- a/ydb/library/ncloud/api/ya.make +++ b/ydb/library/ncloud/api/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( ydb/public/api/client/nc_private/accessservice ydb/library/actors/core - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/base ) diff --git a/ydb/library/persqueue/obfuscate/obfuscate.h b/ydb/library/persqueue/obfuscate/obfuscate.h index 5eb7302fe109..9cc14424d1e7 100644 --- a/ydb/library/persqueue/obfuscate/obfuscate.h +++ b/ydb/library/persqueue/obfuscate/obfuscate.h @@ -1,9 +1 @@ -#pragma once - -#include - -namespace NPersQueue { - -TString ObfuscateString(TString str); - -} // namespace NPersQueue +#include diff --git a/ydb/library/persqueue/obfuscate/ya.make b/ydb/library/persqueue/obfuscate/ya.make index f374b14e264d..3685cb4218b7 100644 --- a/ydb/library/persqueue/obfuscate/ya.make +++ b/ydb/library/persqueue/obfuscate/ya.make @@ -2,7 +2,10 @@ LIBRARY() SRCS( obfuscate.h - obfuscate.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/persqueue/obfuscate ) END() diff --git a/ydb/library/persqueue/topic_parser/topic_parser.h b/ydb/library/persqueue/topic_parser/topic_parser.h index 9ca5809460b2..68c6c489794a 100644 --- a/ydb/library/persqueue/topic_parser/topic_parser.h +++ b/ydb/library/persqueue/topic_parser/topic_parser.h @@ -8,7 +8,7 @@ #include #include -#include +#include namespace NKikimr::NMsgBusProxy::NPqMetaCacheV2 { diff --git a/ydb/library/persqueue/topic_parser/ya.make b/ydb/library/persqueue/topic_parser/ya.make index ef7675c39ee7..c0add3b8f4f0 100644 --- a/ydb/library/persqueue/topic_parser/ya.make +++ b/ydb/library/persqueue/topic_parser/ya.make @@ -2,7 +2,7 @@ LIBRARY() PEERDIR( ydb/core/base - ydb/library/persqueue/topic_parser_public + ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public ydb/public/api/protos ) diff --git a/ydb/library/persqueue/topic_parser_public/topic_parser.h b/ydb/library/persqueue/topic_parser_public/topic_parser.h index 2a5a438722c6..a7f034368395 100644 --- a/ydb/library/persqueue/topic_parser_public/topic_parser.h +++ b/ydb/library/persqueue/topic_parser_public/topic_parser.h @@ -1,33 +1 @@ -#pragma once - -#include -#include - -namespace NPersQueue { - -TString GetDC(const TString& topic); - -TString GetRealTopic(const TString& topic); - -TString BuildFullTopicName(const TString& topicPath, const TString& topicDC); - -TString GetProducer(const TString& topic); -TString GetAccount(const TString& topic); -TString GetTopicPath(const TString& topic); - -TString NormalizePath(const TString& path); - -bool CorrectName(const TString& topic); - -TString ConvertNewTopicName(const TString& topic); - -TString ConvertNewConsumerName(const TString& consumer); -TString ConvertNewProducerName(const TString& consumer); - - -TString ConvertOldTopicName(const TString& topic); -TString ConvertOldProducerName(const TString& producer); -TString ConvertOldConsumerName(const TString& consumer); - - -} // namespace NPersQueue +#include diff --git a/ydb/library/persqueue/topic_parser_public/ya.make b/ydb/library/persqueue/topic_parser_public/ya.make index 8c4eb032533b..f8147d35166d 100644 --- a/ydb/library/persqueue/topic_parser_public/ya.make +++ b/ydb/library/persqueue/topic_parser_public/ya.make @@ -2,7 +2,10 @@ LIBRARY() SRCS( topic_parser.h - topic_parser.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public ) END() diff --git a/ydb/library/persqueue/ya.make b/ydb/library/persqueue/ya.make index 143a827442e3..81d243036ce8 100644 --- a/ydb/library/persqueue/ya.make +++ b/ydb/library/persqueue/ya.make @@ -1,5 +1,4 @@ RECURSE( - counter_time_keeper deprecated obfuscate diff --git a/ydb/library/query_actor/query_actor.h b/ydb/library/query_actor/query_actor.h index a7318c18d23a..4ede865a73a3 100644 --- a/ydb/library/query_actor/query_actor.h +++ b/ydb/library/query_actor/query_actor.h @@ -17,10 +17,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include namespace NKikimr { diff --git a/ydb/library/query_actor/query_actor_ut.cpp b/ydb/library/query_actor/query_actor_ut.cpp index dab5b61c68b3..73b8609f70b1 100644 --- a/ydb/library/query_actor/query_actor_ut.cpp +++ b/ydb/library/query_actor/query_actor_ut.cpp @@ -1,6 +1,6 @@ #include "query_actor.h" #include -#include +#include #include #include @@ -233,8 +233,8 @@ Y_UNIT_TEST_SUITE(QueryActorTest) { UNIT_ASSERT_VALUES_EQUAL_C(result.ColumnsCount(), 1, "Invalid number of columns"); while (result.TryNextRow()) { - const TString& row = result.ColumnParser(0).GetUtf8(); - UNIT_ASSERT_VALUES_EQUAL_C(row, Value, "Ivalid row value"); + const std::string& row = result.ColumnParser(0).GetUtf8(); + UNIT_ASSERT_VALUES_EQUAL_C(TString{row}, Value, "Ivalid row value"); if (ReadedRows >= RowsToRead) { CancelStreamQuery(); diff --git a/ydb/library/query_actor/ya.make b/ydb/library/query_actor/ya.make index 7effb6ccc908..3d762de87604 100644 --- a/ydb/library/query_actor/ya.make +++ b/ydb/library/query_actor/ya.make @@ -11,9 +11,9 @@ PEERDIR( ydb/core/grpc_services/local_rpc yql/essentials/public/issue ydb/public/api/protos - ydb/public/sdk/cpp/client/ydb_params - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/proto ) END() diff --git a/ydb/library/security/ya.make b/ydb/library/security/ya.make index 1dbb4cf076db..9117800cbff7 100644 --- a/ydb/library/security/ya.make +++ b/ydb/library/security/ya.make @@ -1,9 +1,9 @@ LIBRARY() PEERDIR( - ydb/public/sdk/cpp/client/iam/common + ydb/public/sdk/cpp/src/client/iam library/cpp/digest/crc32c - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) SRCS( diff --git a/ydb/library/security/ydb_credentials_provider_factory.cpp b/ydb/library/security/ydb_credentials_provider_factory.cpp index 1837d568302d..1d2b0d1acfdd 100644 --- a/ydb/library/security/ydb_credentials_provider_factory.cpp +++ b/ydb/library/security/ydb_credentials_provider_factory.cpp @@ -1,6 +1,6 @@ #include "ydb_credentials_provider_factory.h" -#include +#include namespace NKikimr { diff --git a/ydb/library/security/ydb_credentials_provider_factory.h b/ydb/library/security/ydb_credentials_provider_factory.h index 92c986e2490e..b0168a1128eb 100644 --- a/ydb/library/security/ydb_credentials_provider_factory.h +++ b/ydb/library/security/ydb_credentials_provider_factory.h @@ -1,6 +1,8 @@ #pragma once -#include +#include + +#include #include diff --git a/ydb/library/table_creator/table_creator_ut.cpp b/ydb/library/table_creator/table_creator_ut.cpp index 33b1f2733089..62a7a406c55d 100644 --- a/ydb/library/table_creator/table_creator_ut.cpp +++ b/ydb/library/table_creator/table_creator_ut.cpp @@ -3,8 +3,8 @@ #include #include -#include -#include +#include +#include namespace NKikimr { diff --git a/ydb/library/table_creator/ut/ya.make b/ydb/library/table_creator/ut/ya.make index 5cd84f862aab..477543563ebf 100644 --- a/ydb/library/table_creator/ut/ya.make +++ b/ydb/library/table_creator/ut/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/core/testlib/default yql/essentials/public/udf/service/exception_policy yql/essentials/sql/pg_dummy - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 4ddc6c8d3dfc..45b9f85df7ec 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -1,9 +1,9 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/library/workload/abstract/ya.make b/ydb/library/workload/abstract/ya.make index 44347e34c610..186533748ae9 100644 --- a/ydb/library/workload/abstract/ya.make +++ b/ydb/library/workload/abstract/ya.make @@ -7,10 +7,10 @@ SRCS( PEERDIR( library/cpp/getopt ydb/library/accessor - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/value ) END() diff --git a/ydb/library/workload/clickbench/data_generator.cpp b/ydb/library/workload/clickbench/data_generator.cpp index 485c1a46d870..5ef975d968b4 100644 --- a/ydb/library/workload/clickbench/data_generator.cpp +++ b/ydb/library/workload/clickbench/data_generator.cpp @@ -1,8 +1,10 @@ #include "data_generator.h" #include +#include #include #include #include + #include namespace NYdbWorkload { diff --git a/ydb/library/workload/kv/kv.h b/ydb/library/workload/kv/kv.h index f270152a8e83..79685f5d1722 100644 --- a/ydb/library/workload/kv/kv.h +++ b/ydb/library/workload/kv/kv.h @@ -64,8 +64,8 @@ class TKvWorkloadGenerator final: public TWorkloadQueryGeneratorBase; struct TRow { - TVector Ints; - TVector Strings; + std::vector Ints; + std::vector Strings; TString ToString() const { std::stringstream ss; diff --git a/ydb/library/ycloud/api/events.h b/ydb/library/ycloud/api/events.h index 5b4c438c2d8f..4780aef3fe0a 100644 --- a/ydb/library/ycloud/api/events.h +++ b/ydb/library/ycloud/api/events.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace NCloud { diff --git a/ydb/library/ycloud/api/ya.make b/ydb/library/ycloud/api/ya.make index bec439527582..ca9ce8d233d1 100644 --- a/ydb/library/ycloud/api/ya.make +++ b/ydb/library/ycloud/api/ya.make @@ -14,7 +14,7 @@ PEERDIR( ydb/public/api/client/yc_private/accessservice ydb/public/api/client/yc_private/resourcemanager ydb/library/actors/core - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/base ydb/core/grpc_caching ) diff --git a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp index fd0554a3150d..affbf34f51cd 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp @@ -10,7 +10,7 @@ class TBearerCredentialsProvider : public NYdb::ICredentialsProvider { : Delegatee(delegatee) { } - TString GetAuthInfo() const override { + std::string GetAuthInfo() const override { TString result = Delegatee->GetAuthInfo(); if (!result || result.StartsWith("Bearer ")) { return result; @@ -33,7 +33,7 @@ class TBearerCredentialsProviderFactory : public NYdb::ICredentialsProviderFacto : Delegatee(delegatee) { } - TString GetClientIdentity() const override { + std::string GetClientIdentity() const override { return "BEARER_CRED_PROV_FACTORY" + ToString((ui64)this); } diff --git a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h index 87034a9da2ed..9f7aee6f5c3c 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h +++ b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NYql { diff --git a/ydb/library/yql/providers/common/token_accessor/client/factory.h b/ydb/library/yql/providers/common/token_accessor/client/factory.h index 0d8aa39d72b7..e957dcc46242 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/factory.h +++ b/ydb/library/yql/providers/common/token_accessor/client/factory.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace NYql { diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp index bc47b395f28b..3b0fea9fc8bb 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -160,7 +160,7 @@ class TTokenAccessorCredentialsProvider : public NYdb::ICredentialsProvider { Impl->Stop(); } - TString GetAuthInfo() const override { + std::string GetAuthInfo() const override { return Impl->GetTicket(); } diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h index 0ebb9e8d90dc..e9e4aa3ecfc6 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h @@ -1,9 +1,9 @@ #pragma once #include -#include +#include -#include +#include #include namespace NYql { diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h index 54593072adbc..c041e5ca3279 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h @@ -1,9 +1,9 @@ #pragma once #include -#include +#include -#include +#include #include namespace NYql { diff --git a/ydb/library/yql/providers/common/token_accessor/client/ya.make b/ydb/library/yql/providers/common/token_accessor/client/ya.make index d2b6a5c5d5bb..9abbe7a6a687 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/ya.make +++ b/ydb/library/yql/providers/common/token_accessor/client/ya.make @@ -8,12 +8,12 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/threading/atomic library/cpp/threading/future yql/essentials/providers/common/structured_token ydb/library/yql/providers/common/token_accessor/grpc - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) END() diff --git a/ydb/library/yql/providers/dq/actors/yt/ya.make b/ydb/library/yql/providers/dq/actors/yt/ya.make index 2fad9953b8f8..67435dcee526 100644 --- a/ydb/library/yql/providers/dq/actors/yt/ya.make +++ b/ydb/library/yql/providers/dq/actors/yt/ya.make @@ -2,7 +2,7 @@ LIBRARY() PEERDIR( ydb/library/actors/core - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client yt/cpp/mapreduce/interface ydb/library/yql/providers/dq/config yql/essentials/core/issue diff --git a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp index 42185465f123..f84c1bf1e7a1 100644 --- a/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp +++ b/ydb/library/yql/providers/dq/global_worker_manager/global_worker_manager.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp index 1cd555355e59..9cd0acf57092 100644 --- a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.cpp @@ -79,7 +79,7 @@ class TAbstractNodeResolver: public IServiceNodeResolver { conn.reset(ClientLow.CreateGRpcServiceConnection(holder).release()); }); - return MakeFuture(TConnectionResult(std::move(conn), std::move(context), nodeInfo->NodeId, nodeInfo->ClientConfig.Locator)); + return MakeFuture(TConnectionResult(std::move(conn), std::move(context), nodeInfo->NodeId, TString{nodeInfo->ClientConfig.Locator})); } void Stop() override { diff --git a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h index cc78d0bb04d3..4aee9077f42c 100644 --- a/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h +++ b/ydb/library/yql/providers/dq/global_worker_manager/service_node_resolver.h @@ -7,7 +7,7 @@ #include #include -#include +#include namespace NYql { diff --git a/ydb/library/yql/providers/dq/provider/exec/ya.make b/ydb/library/yql/providers/dq/provider/exec/ya.make index 32ec84b7165d..b790e06aaa42 100644 --- a/ydb/library/yql/providers/dq/provider/exec/ya.make +++ b/ydb/library/yql/providers/dq/provider/exec/ya.make @@ -11,7 +11,7 @@ PEERDIR( library/cpp/digest/md5 library/cpp/threading/future ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver yql/essentials/core yql/essentials/core/dq_integration ydb/library/yql/dq/runtime diff --git a/ydb/library/yql/providers/dq/provider/ya.make b/ydb/library/yql/providers/dq/provider/ya.make index 24e0610bd54f..36bfd1f57707 100644 --- a/ydb/library/yql/providers/dq/provider/ya.make +++ b/ydb/library/yql/providers/dq/provider/ya.make @@ -27,14 +27,14 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/threading/task_scheduler library/cpp/threading/future library/cpp/svnversion library/cpp/yson/node library/cpp/yson ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver yql/essentials/ast yql/essentials/core yql/essentials/core/issue diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp index 86a5a3fb516c..2e980abccb9e 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp @@ -7,7 +7,7 @@ #include -#include +#include #include diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp index b8710aac78ea..f36544857153 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp @@ -12,7 +12,7 @@ #include -#include +#include #include #include @@ -630,7 +630,7 @@ class TDqGatewayImpl: public std::enable_shared_from_this { } else { YQL_CLOG(ERROR, ProviderDq) << "OpenSession error: " << status.Msg; this_->DropSession(sessionId); - promise.SetException(status.Msg); + promise.SetException(TString{status.Msg}); } }; diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h index 5b7c27cfe1ff..1d6217f1fb66 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h @@ -14,7 +14,7 @@ #include #include -#include +#include namespace NYql { diff --git a/ydb/library/yql/providers/generic/actors/ya.make b/ydb/library/yql/providers/generic/actors/ya.make index cf1451740469..59c6acadd09d 100644 --- a/ydb/library/yql/providers/generic/actors/ya.make +++ b/ydb/library/yql/providers/generic/actors/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/library/yql/providers/generic/proto yql/essentials/public/types ydb/library/yql/providers/generic/connector/libcpp - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 28f59913cb2d..1a2a5a45d57a 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include namespace NYql::NDq { diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp index 753bb4c94027..5602d046a74f 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp @@ -44,7 +44,7 @@ namespace NYql::NDq { // 3. Otherwise use credentials provider to get token Y_ENSURE(CredentialsProvider_, "CredentialsProvider is not initialized"); - TString iamToken; + std::string iamToken; try { iamToken = CredentialsProvider_->GetAuthInfo(); } catch (const std::exception& e) { @@ -52,7 +52,7 @@ namespace NYql::NDq { return TString(e.what()); } - Y_ENSURE(iamToken, "CredentialsProvider returned empty IAM token"); + Y_ENSURE(!iamToken.empty(), "CredentialsProvider returned empty IAM token"); *dsi.mutable_credentials()->mutable_token()->mutable_value() = std::move(iamToken); return {}; diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client.h b/ydb/library/yql/providers/generic/connector/libcpp/client.h index a06c18c1e907..6f40344d4c9c 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/client.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/client.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/ydb/library/yql/providers/generic/connector/libcpp/error.cpp b/ydb/library/yql/providers/generic/connector/libcpp/error.cpp index d9acb6eeff0a..7646562cf600 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/error.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/error.cpp @@ -66,7 +66,7 @@ namespace NYql::NConnector { } else { // FIXME: more appropriate error code for network error result.set_status(Ydb::StatusIds_StatusCode::StatusIds_StatusCode_INTERNAL_ERROR); - result.set_message(status.Msg); + result.set_message(TString{status.Msg}); } return result; diff --git a/ydb/library/yql/providers/generic/connector/libcpp/error.h b/ydb/library/yql/providers/generic/connector/libcpp/error.h index c94ec6b2ae9d..94218d24e93d 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/error.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/error.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace NYql::NConnector { NApi::TError NewSuccess(); diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index cee03841f9ca..7e151179df2c 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -826,10 +826,10 @@ namespace NYql::NConnector::NTest { static TString StatusToDebugString(const NYdbGrpc::TGrpcStatus& status) { TStringBuilder s; s << "GRpcStatusCode: " << status.GRpcStatusCode << '\n'; - if (status.Msg) { + if (!status.Msg.empty()) { s << status.Msg; } - if (status.Details) { + if (!status.Details.empty()) { s << " (" << status.Details << ')'; } if (status.InternalError) { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/test_creds.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/test_creds.h index cefd4a43c98d..6a001caf000b 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/test_creds.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/test_creds.h @@ -1,12 +1,12 @@ #pragma once -#include +#include #include namespace NYql::NTestCreds { class TCredentialsProvider: public NYdb::ICredentialsProvider { public: - NYdb::TStringType GetAuthInfo() const override { + std::string GetAuthInfo() const override { return "TEST_TOKEN"; } bool IsValid() const override { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make index d31e47403a2e..1a8e45512026 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make @@ -10,7 +10,7 @@ PEERDIR( contrib/libs/apache/arrow contrib/libs/grpc ydb/core/formats/arrow/serializer - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client yql/essentials/ast ydb/library/yql/dq/actors/protos yql/essentials/providers/common/proto diff --git a/ydb/library/yql/providers/generic/provider/ya.make b/ydb/library/yql/providers/generic/provider/ya.make index 5ee4ed066a72..5f28b0a2558d 100644 --- a/ydb/library/yql/providers/generic/provider/ya.make +++ b/ydb/library/yql/providers/generic/provider/ya.make @@ -58,7 +58,7 @@ PEERDIR( ydb/library/yql/providers/generic/connector/libcpp yql/essentials/providers/result/expr_nodes ydb/library/yql/utils/plan - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) END() diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_state.h b/ydb/library/yql/providers/generic/provider/yql_generic_state.h index 9a4a003e048c..c1bab527a497 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_state.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_state.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace NKikimr::NMiniKQL { class IFunctionRegistry; diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.cpp index 5811892fcf6c..0d333df8c379 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.cpp @@ -27,7 +27,7 @@ namespace { "_yql_sys_partition_id", [](const NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage& message){ using TDataType = NYql::NUdf::TDataType; return std::make_pair( - NYql::NUdf::TUnboxedValuePod(message.GetPartitionSession()->GetPartitionId()), + NYql::NUdf::TUnboxedValuePod(static_cast(message.GetPartitionSession()->GetPartitionId())), NYql::NUdf::GetDataTypeInfo(TDataType::Slot).FixedSize ); } @@ -36,7 +36,7 @@ namespace { "_yql_sys_offset", [](const NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage& message){ using TDataType = NYql::NUdf::TDataType; return std::make_pair( - NYql::NUdf::TUnboxedValuePod(message.GetOffset()), + NYql::NUdf::TUnboxedValuePod(static_cast(message.GetOffset())), NYql::NUdf::GetDataTypeInfo(TDataType::Slot).FixedSize); } }, @@ -53,7 +53,7 @@ namespace { "_yql_sys_seq_no", [](const NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage& message){ using TDataType = NYql::NUdf::TDataType; return std::make_pair( - NYql::NUdf::TUnboxedValuePod(message.GetSeqNo()), + NYql::NUdf::TUnboxedValuePod(static_cast(message.GetSeqNo())), NYql::NUdf::GetDataTypeInfo(TDataType::Slot).FixedSize ); } diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.h index a910190f79e1..76e1b32f51f1 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_meta_extractor.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.cpp index 6f0a1b9f6286..cddf54cb5f8f 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.cpp @@ -25,8 +25,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.h index 6e09d4637eba..9c88a597699c 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_rd_read_actor.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp index 307b767ad593..5396f136c3c9 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp @@ -19,8 +19,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -32,6 +32,8 @@ #include +#include + #include #include #include @@ -212,7 +214,7 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: } TString GetSessionId() const override { - return ReadSession ? ReadSession->GetSessionId() : TString{"empty"}; + return ReadSession ? TString{ReadSession->GetSessionId()} : TString{"empty"}; } private: @@ -300,7 +302,7 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: bool recheckBatch = false; if (freeSpace > 0) { - auto events = GetReadSession().GetEvents(false, TMaybe(), static_cast(freeSpace)); + auto events = GetReadSession().GetEvents(false, std::nullopt, static_cast(freeSpace)); recheckBatch = !events.empty(); ui32 batchItemsEstimatedCount = 0; @@ -470,9 +472,9 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: const auto partitionKey = MakePartitionKey(event.GetPartitionSession()); const auto partitionKeyStr = ToString(partitionKey); for (const auto& message : event.GetMessages()) { - const TString& data = message.GetData(); + const std::string& data = message.GetData(); Self.IngressStats.Bytes += data.size(); - LWPROBE(PqReadDataReceived, TString(TStringBuilder() << Self.TxId), Self.SourceParams.GetTopicPath(), data); + LWPROBE(PqReadDataReceived, TString(TStringBuilder() << Self.TxId), Self.SourceParams.GetTopicPath(), TString{data}); SRC_LOG_T("SessionId: " << Self.GetSessionId() << " Key: " << partitionKeyStr << " Data received: " << message.DebugString(true)); if (message.GetWriteTime() < Self.StartingMessageTimestamp) { @@ -501,7 +503,7 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: SRC_LOG_E("SessionId: " << Self.GetSessionId() << " " << message << ": " << ev.DebugString()); TIssue issue(message); for (const auto& subIssue : ev.GetIssues()) { - TIssuePtr newIssue(new TIssue(subIssue)); + TIssuePtr newIssue(new TIssue(NYdb::NAdapters::ToYqlIssue(subIssue))); issue.AddSubIssue(newIssue); } Self.Send(Self.ComputeActorId, new TEvAsyncInputError(Self.InputIndex, TIssues({issue}), NYql::NDqProto::StatusIds::BAD_REQUEST)); @@ -515,7 +517,7 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: SRC_LOG_D("SessionId: " << Self.GetSessionId() << " Key: " << partitionKeyStr << " StartPartitionSessionEvent received"); - TMaybe readOffset; + std::optional readOffset; const auto offsetIt = Self.PartitionToOffset.find(partitionKey); if (offsetIt != Self.PartitionToOffset.end()) { readOffset = offsetIt->second; @@ -571,7 +573,7 @@ class TDqPqReadActor : public NActors::TActor, public NYql::NDq: } std::pair CreateItem(const NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage& message) { - const TString& data = message.GetData(); + const std::string& data = message.GetData(); i64 usedSpace = 0; NUdf::TUnboxedValuePod item; diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h index e4f9e260af49..a6905a90a2e4 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp index 460ef737513a..85b02502ccd2 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp @@ -13,8 +13,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h index 39f388a79dcc..c2c1e942d53a 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/async_io/ya.make b/ydb/library/yql/providers/pq/async_io/ya.make index da853e47bf4c..99b505adce4e 100644 --- a/ydb/library/yql/providers/pq/async_io/ya.make +++ b/ydb/library/yql/providers/pq/async_io/ya.make @@ -22,9 +22,10 @@ PEERDIR( ydb/library/yql/providers/pq/proto yql/essentials/public/types yql/essentials/utils/log - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types/credentials ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/pq/cm_client/client.h b/ydb/library/yql/providers/pq/cm_client/client.h index f1541b3d4985..fe5b0c321be5 100644 --- a/ydb/library/yql/providers/pq/cm_client/client.h +++ b/ydb/library/yql/providers/pq/cm_client/client.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/cm_client/ya.make b/ydb/library/yql/providers/pq/cm_client/ya.make index 8167e2601cbd..3e8f5449f675 100644 --- a/ydb/library/yql/providers/pq/cm_client/ya.make +++ b/ydb/library/yql/providers/pq/cm_client/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( library/cpp/threading/future yql/essentials/public/issue - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) GENERATE_ENUM_SERIALIZATION(client.h) diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_blocking_queue.h b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_blocking_queue.h index 7a8db40dc554..9f7e400b0275 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_blocking_queue.h +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_blocking_queue.h @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -27,7 +27,7 @@ class TBlockingEQueue { } } - TMaybe Pop(bool block) { + std::optional Pop(bool block) { with_lock(Mutex_) { if (block) { CanPop_.WaitI(Mutex_, [this] () {return CanPopPredicate();}); diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_file_topic_client.cpp b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_file_topic_client.cpp index 36914e27f455..ed79bb2f3cc3 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_file_topic_client.cpp +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_file_topic_client.cpp @@ -34,29 +34,29 @@ constexpr static auto FILE_POLL_PERIOD = TDuration::MilliSeconds(5); }, Pool_); } - TVector GetEvents(bool block, TMaybe maxEventsCount, size_t maxByteSize) override { + std::vector GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) override { // TODO Y_UNUSED(maxByteSize); - TVector res; - for (auto event = EventsQ_.Pop(block); !event.Empty() && res.size() < maxEventsCount.GetOrElse(std::numeric_limits::max()); event = EventsQ_.Pop(/*block=*/ false)) { + std::vector res; + for (auto event = EventsQ_.Pop(block); event.has_value() && res.size() < maxEventsCount.value_or(std::numeric_limits::max()); event = EventsQ_.Pop(/*block=*/ false)) { res.push_back(std::move(*event)); } return res; } - TVector GetEvents(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { + std::vector GetEvents(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { return GetEvents(settings.Block_, settings.MaxEventsCount_, settings.MaxByteSize_); } - TMaybe GetEvent(bool block, size_t maxByteSize) override { + std::optional GetEvent(bool block, size_t maxByteSize) override { // TODO Y_UNUSED(maxByteSize); return EventsQ_.Pop(block); } - TMaybe GetEvent(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { + std::optional GetEvent(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { return GetEvent(settings.Block_, settings.MaxByteSize_); } @@ -77,7 +77,7 @@ constexpr static auto FILE_POLL_PERIOD = TDuration::MilliSeconds(5); return Counters_; } - TString GetSessionId() const override { + std::string GetSessionId() const override { return ToString(Session_->GetPartitionSessionId()); } @@ -170,19 +170,19 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: }, Pool_); } - TMaybe GetEvent(bool block) override { + std::optional GetEvent(bool block) override { return EventsQ_.Pop(block); } - TVector GetEvents(bool block, TMaybe maxEventsCount) override { - TVector res; - for (auto event = EventsQ_.Pop(block); !event.Empty() && res.size() < maxEventsCount.GetOrElse(std::numeric_limits::max()); event = EventsQ_.Pop(/*block=*/ false)) { + std::vector GetEvents(bool block, std::optional maxEventsCount) override { + std::vector res; + for (auto event = EventsQ_.Pop(block); event.has_value() && res.size() < maxEventsCount.value_or(std::numeric_limits::max()); event = EventsQ_.Pop(/*block=*/ false)) { res.push_back(std::move(*event)); } return res; } - NThreading::TFuture GetInitSeqNo() override { + NThreading::TFuture GetInitSeqNo() override { return NThreading::MakeFuture(SeqNo_); } @@ -194,13 +194,13 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: EventsMsgQ_.Push(TOwningWriteMessage(std::move(message)), size); } - void Write(NYdb::NTopic::TContinuationToken&& token, TStringBuf data, TMaybe seqNo, - TMaybe createTimestamp) override { + void Write(NYdb::NTopic::TContinuationToken&& token, std::string_view data, std::optional seqNo, + std::optional createTimestamp) override { NYdb::NTopic::TWriteMessage message(data); - if (seqNo.Defined()) { + if (seqNo.has_value()) { message.SeqNo(*seqNo); } - if (createTimestamp.Defined()) { + if (createTimestamp.has_value()) { message.CreateTimestamp(*createTimestamp); } @@ -214,7 +214,7 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: NYdb::NTopic::TWriteMessage message(params.Data); - if (params.CreateTimestamp_.Defined()) { + if (params.CreateTimestamp_.has_value()) { message.CreateTimestamp(*params.CreateTimestamp_); } if (params.SeqNo_) { @@ -226,16 +226,16 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: } // Ignores codec in message and always writes raw for debugging purposes - void WriteEncoded(NYdb::NTopic::TContinuationToken&& token, TStringBuf data, NYdb::NTopic::ECodec codec, ui32 originalSize, - TMaybe seqNo, TMaybe createTimestamp) override { + void WriteEncoded(NYdb::NTopic::TContinuationToken&& token, std::string_view data, NYdb::NTopic::ECodec codec, uint32_t originalSize, + std::optional seqNo, std::optional createTimestamp) override { Y_UNUSED(codec); Y_UNUSED(originalSize); NYdb::NTopic::TWriteMessage message(data); - if (seqNo.Defined()) { + if (seqNo.has_value()) { message.SeqNo(*seqNo); } - if (createTimestamp.Defined()) { + if (createTimestamp.has_value()) { message.CreateTimestamp(*createTimestamp); } @@ -279,11 +279,11 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: do { auto& [content, msg] = *maybeMsg; NYdb::NTopic::TWriteSessionEvent::TWriteAck ack; - if (msg.SeqNo_.Defined()) { // FIXME should be auto generated otherwise + if (msg.SeqNo_.has_value()) { // FIXME should be auto generated otherwise ack.SeqNo = *msg.SeqNo_; } ack.State = NYdb::NTopic::TWriteSessionEvent::TWriteAck::EES_WRITTEN; - ack.Details.ConstructInPlace(offset, partitionId); + ack.Details.emplace(offset, partitionId); acks.Acks.emplace_back(std::move(ack)); offset += content.size() + 1; fo.Write(content); @@ -320,7 +320,7 @@ class TFileTopicWriteSession : public NYdb::NTopic::IWriteSession, private NYdb: TThreadPool Pool_; NYdb::NTopic::TWriterCounters::TPtr Counters_; - ui64 SeqNo_ = 0; + uint64_t SeqNo_ = 0; }; struct TDummyPartitionSession: public NYdb::NTopic::TPartitionSession { @@ -337,7 +337,7 @@ struct TDummyPartitionSession: public NYdb::NTopic::TPartitionSession { std::shared_ptr TFileTopicClient::CreateReadSession(const NYdb::NTopic::TReadSessionSettings& settings) { Y_ENSURE(!settings.Topics_.empty()); - TString topicPath = settings.Topics_.front().Path_; + auto topicPath = settings.Topics_.front().Path_; auto topicsIt = Topics_.find(make_pair("pq", topicPath)); Y_ENSURE(topicsIt != Topics_.end()); @@ -350,7 +350,7 @@ std::shared_ptr TFileTopicClient::CreateReadSession( return std::make_shared( TFile(*filePath, EOpenMode::TEnum::RdOnly), - MakeIntrusive(sessionId, topicPath, partitionId) + MakeIntrusive(sessionId, TString{topicPath}, partitionId) ); } @@ -408,7 +408,7 @@ std::shared_ptr TFileTopicClient::Cre } std::shared_ptr TFileTopicClient::CreateWriteSession(const NYdb::NTopic::TWriteSessionSettings& settings) { - TString topicPath = settings.Path_; + auto topicPath = TString{settings.Path_}; auto topicsIt = Topics_.find(make_pair("pq", topicPath)); Y_ENSURE(topicsIt != Topics_.end()); auto filePath = topicsIt->second.FilePath; diff --git a/ydb/library/yql/providers/pq/gateway/native/ya.make b/ydb/library/yql/providers/pq/gateway/native/ya.make index 19cabc364b58..8bdb78b0681a 100644 --- a/ydb/library/yql/providers/pq/gateway/native/ya.make +++ b/ydb/library/yql/providers/pq/gateway/native/ya.make @@ -12,9 +12,9 @@ PEERDIR( ydb/library/yql/providers/pq/cm_client ydb/library/yql/providers/pq/provider yql/essentials/utils - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic ) END() diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp index 8c94f126ecda..f2852d0b2f39 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp @@ -3,7 +3,7 @@ #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h index 1294abfe6a34..81da9450da89 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h @@ -6,7 +6,7 @@ #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h index f29192e39a20..04adf51d93b2 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/ydb/library/yql/providers/pq/provider/ut/ya.make b/ydb/library/yql/providers/pq/provider/ut/ya.make index 9e287abe4f7b..2589ffefc9b9 100644 --- a/ydb/library/yql/providers/pq/provider/ut/ya.make +++ b/ydb/library/yql/providers/pq/provider/ut/ya.make @@ -25,8 +25,8 @@ PEERDIR( ydb/library/yql/providers/solomon/provider yql/essentials/public/udf/service/exception_policy yql/essentials/sql/pg_dummy - ydb/public/sdk/cpp/client/ydb_params - ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/persqueue_public/codecs ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/pq/provider/ut/yql_pq_ut.cpp b/ydb/library/yql/providers/pq/provider/ut/yql_pq_ut.cpp index 3cd2c1538d4d..3df0929b3241 100644 --- a/ydb/library/yql/providers/pq/provider/ut/yql_pq_ut.cpp +++ b/ydb/library/yql/providers/pq/provider/ut/yql_pq_ut.cpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include @@ -118,7 +118,7 @@ bool RunPqProgram( NYql::CreateCommonDqTaskTransformFactory() }); - const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")); + const auto driverConfig = NYdb::TDriverConfig().SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(driverConfig); auto dqGateway = CreateLocalDqGateway(functionRegistry.Get(), dqCompFactory, dqTaskTransformFactory, {}, false/*spilling*/, CreateAsyncIoFactory(driver, pqGateway)); diff --git a/ydb/library/yql/providers/pq/provider/ya.make b/ydb/library/yql/providers/pq/provider/ya.make index 67ee5d14b7f7..6030bebb1d55 100644 --- a/ydb/library/yql/providers/pq/provider/ya.make +++ b/ydb/library/yql/providers/pq/provider/ya.make @@ -48,7 +48,7 @@ PEERDIR( ydb/library/yql/providers/pq/proto yql/essentials/providers/result/expr_nodes yql/essentials/public/udf - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h index e9dd09de6009..52c64663d4bd 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_topic_client.h b/ydb/library/yql/providers/pq/provider/yql_pq_topic_client.h index 2c463b9598af..d4fe9a2788b5 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_topic_client.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_topic_client.h @@ -1,5 +1,5 @@ #pragma once -#include +#include namespace NYql { class ITopicClient : public TThrRefBase { diff --git a/ydb/library/yql/providers/s3/credentials/credentials.h b/ydb/library/yql/providers/s3/credentials/credentials.h index 4c299c9d015f..8dff6fef9000 100644 --- a/ydb/library/yql/providers/s3/credentials/credentials.h +++ b/ydb/library/yql/providers/s3/credentials/credentials.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace NYql { diff --git a/ydb/library/yql/providers/s3/credentials/ya.make b/ydb/library/yql/providers/s3/credentials/ya.make index 70040404c00e..00847033f801 100644 --- a/ydb/library/yql/providers/s3/credentials/ya.make +++ b/ydb/library/yql/providers/s3/credentials/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/providers/s3/proto - ydb/public/sdk/cpp/client/ydb_types/credentials + ydb/public/sdk/cpp/src/client/types/credentials ) END() diff --git a/ydb/library/yql/providers/ydb/actors/ya.make b/ydb/library/yql/providers/ydb/actors/ya.make index cb6374259f90..87176303b3b9 100644 --- a/ydb/library/yql/providers/ydb/actors/ya.make +++ b/ydb/library/yql/providers/ydb/actors/ya.make @@ -12,7 +12,8 @@ PEERDIR( yql/essentials/public/types yql/essentials/utils/log ydb/public/lib/experimental - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/driver ydb/library/yql/dq/actors/compute ydb/library/yql/providers/ydb/proto ) diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp index bce98956dce0..965f5350756b 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -201,7 +202,7 @@ class TYdbReadActor : public TActorBootstrapped, public IDqComput RequestsDone = true; while(!Blocks.empty()) Blocks.pop(); - Send(ComputeActorId, new TEvAsyncInputError(InputIndex, res.GetIssues(), NYql::NDqProto::StatusIds::EXTERNAL_ERROR)); + Send(ComputeActorId, new TEvAsyncInputError(InputIndex, ::NYdb::NAdapters::ToYqlIssues(res.GetIssues()), NYql::NDqProto::StatusIds::EXTERNAL_ERROR)); } else { WakeUpTime = TMonotonic::Now() + Min(TDuration::Seconds(3), TDuration::MilliSeconds(0x30U * (1U << ++Retried))); ActorSystem->Schedule(WakeUpTime, new IEventHandle(SelfId(), TActorId(), new TEvPrivate::TEvRetryTime)); diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h index c2219ec350bc..e1d30c284cc9 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h @@ -5,7 +5,7 @@ #include #include -#include +#include namespace NYql::NDq { diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h index b5a44cf293fe..18632641f2ad 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/ydb/library/yql/providers/ydb/comp_nodes/ya.make b/ydb/library/yql/providers/ydb/comp_nodes/ya.make index 40563b04088b..a91a21c141bc 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/ya.make +++ b/ydb/library/yql/providers/ydb/comp_nodes/ya.make @@ -8,7 +8,8 @@ PEERDIR( yql/essentials/providers/common/structured_token ydb/library/yql/providers/ydb/proto ydb/public/lib/experimental - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/driver ) SRCS( diff --git a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp index fb7074c243cb..c5d9983f7cd5 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp +++ b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -167,7 +168,7 @@ using TBaseComputation = TMutableComputationNode>; void ProcessError(const NYdb::NClickhouseInternal::TScanResult& res) { const std::unique_lock lock(Sync); RequestSent = false; - Issues = res.GetIssues(); + Issues = NYdb::NAdapters::ToYqlIssues(res.GetIssues()); while (!Blocks.empty()) Blocks.pop(); } diff --git a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.h b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.h index 21149c590989..9e167fe57523 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.h +++ b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace NYql::NDqs { diff --git a/ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_factory.h b/ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_factory.h index 84a31d007333..68d537d5d2ac 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_factory.h +++ b/ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_factory.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace NYql { diff --git a/ydb/library/yql/providers/ydb/provider/ya.make b/ydb/library/yql/providers/ydb/provider/ya.make index 652f89a65fc3..19e385c0c7d6 100644 --- a/ydb/library/yql/providers/ydb/provider/ya.make +++ b/ydb/library/yql/providers/ydb/provider/ya.make @@ -45,8 +45,9 @@ PEERDIR( ydb/library/yql/providers/ydb/expr_nodes ydb/library/yql/providers/ydb/proto ydb/public/lib/experimental - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/adapters/issue + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp index a7d1235d617b..6d83bdb1b535 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace NYql { @@ -218,7 +219,7 @@ using TTableKey2DescribeTableResult = std::unordered_mapConfiguration->Clusters[TString(client.first)]; ctx.AddError(TIssue({}, TStringBuilder() << "Failed to take snapshot for: `" << client.first << "`, endpoint: " << config.Endpoint << ", status: " << snapshot.GetStatus())); for (const auto& issue : snapshot.GetIssues()) - ctx.AddError(issue); + ctx.AddError(NYdb::NAdapters::ToYqlIssue(issue)); } } @@ -255,7 +256,7 @@ using TTableKey2DescribeTableResult = std::unordered_map -#include +#include #include #include diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h index b393ec82d54f..be0ade72fd7b 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include diff --git a/ydb/library/yql/tools/dq/dq_cli/main.cpp b/ydb/library/yql/tools/dq/dq_cli/main.cpp index 36164577883b..22fb6b79632f 100644 --- a/ydb/library/yql/tools/dq/dq_cli/main.cpp +++ b/ydb/library/yql/tools/dq/dq_cli/main.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include @@ -299,7 +299,7 @@ void OpenSession(TServiceConnection& service, const TString& sessionI if (status.Ok()) { promise.SetValue(); } else { - promise.SetException(status.Msg); + promise.SetException(TString{status.Msg}); } }; @@ -333,7 +333,7 @@ Yql::DqsProto::RoutesResponse Routes(TServiceConnection& service, con if (status.Ok()) { promise.SetValue(resp); } else { - promise.SetException(status.Msg); + promise.SetException(TString{status.Msg}); } }; @@ -361,7 +361,7 @@ Yql::DqsProto::BenchmarkResponse Bench(TServiceConnection& service, c if (status.Ok()) { promise.SetValue(resp); } else { - promise.SetException(status.Msg); + promise.SetException(TString{status.Msg}); } }; diff --git a/ydb/library/yql/tools/dq/dq_cli/ya.make b/ydb/library/yql/tools/dq/dq_cli/ya.make index b6e654fbb058..542872ebd129 100644 --- a/ydb/library/yql/tools/dq/dq_cli/ya.make +++ b/ydb/library/yql/tools/dq/dq_cli/ya.make @@ -2,7 +2,7 @@ PROGRAM() PEERDIR( library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/protobuf/util library/cpp/threading/future yql/essentials/utils diff --git a/ydb/library/yql/tools/dq/worker_node/main.cpp b/ydb/library/yql/tools/dq/worker_node/main.cpp index 9a349b0f493c..d69388a6e9f5 100644 --- a/ydb/library/yql/tools/dq/worker_node/main.cpp +++ b/ydb/library/yql/tools/dq/worker_node/main.cpp @@ -134,7 +134,7 @@ NDq::IDqAsyncIoFactory::TPtr CreateAsyncIoFactory(const NYdb::TDriver& driver, I int main(int argc, char** argv) { - const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")); + const auto driverConfig = NYdb::TDriverConfig().SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(driverConfig); Y_DEFER { diff --git a/ydb/library/yql/tools/dq/worker_node/ya.make b/ydb/library/yql/tools/dq/worker_node/ya.make index dfbe133ee4cb..c4e71607301d 100644 --- a/ydb/library/yql/tools/dq/worker_node/ya.make +++ b/ydb/library/yql/tools/dq/worker_node/ya.make @@ -2,7 +2,7 @@ IF (NOT OS_WINDOWS) PROGRAM() PEERDIR( - ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs + ydb/public/sdk/cpp/src/client/persqueue_public/codecs library/cpp/getopt yt/cpp/mapreduce/client ydb/library/yql/dq/actors/compute diff --git a/ydb/library/yql/tools/dqrun/dqrun.cpp b/ydb/library/yql/tools/dqrun/dqrun.cpp index 0b109a3af273..8038713e4cb3 100644 --- a/ydb/library/yql/tools/dqrun/dqrun.cpp +++ b/ydb/library/yql/tools/dqrun/dqrun.cpp @@ -940,7 +940,7 @@ int RunMain(int argc, const char* argv[]) TVector dataProvidersInit; dataProvidersInit.push_back(GetPgDataProviderInitializer()); - const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")); + const auto driverConfig = NYdb::TDriverConfig().SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(driverConfig); Y_DEFER { diff --git a/ydb/library/yql/tools/dqrun/ya.make b/ydb/library/yql/tools/dqrun/ya.make index 9f3383946aa8..be850268c9fc 100644 --- a/ydb/library/yql/tools/dqrun/ya.make +++ b/ydb/library/yql/tools/dqrun/ya.make @@ -23,7 +23,7 @@ ENDIF() PEERDIR( contrib/libs/protobuf - ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs + ydb/public/sdk/cpp/src/client/persqueue_public/codecs ydb/library/actors/http library/cpp/getopt library/cpp/lfalloc/alloc_profiler diff --git a/ydb/library/yql/tools/mrrun/mrrun.cpp b/ydb/library/yql/tools/mrrun/mrrun.cpp index 37fdd07c5802..c501e60adef3 100644 --- a/ydb/library/yql/tools/mrrun/mrrun.cpp +++ b/ydb/library/yql/tools/mrrun/mrrun.cpp @@ -1,6 +1,6 @@ #include "mrrun.h" -#include +#include #include #include @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include @@ -668,7 +668,7 @@ int RunMain(int argc, const char* argv[]) dataProvidersInit.push_back(GetClickHouseDataProviderInitializer(httpGateway)); } - const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")); + const auto driverConfig = NYdb::TDriverConfig().SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(driverConfig); Y_DEFER { diff --git a/ydb/library/yql/tools/mrrun/ya.make b/ydb/library/yql/tools/mrrun/ya.make index 955ef407df67..023f7d57b325 100644 --- a/ydb/library/yql/tools/mrrun/ya.make +++ b/ydb/library/yql/tools/mrrun/ya.make @@ -15,7 +15,7 @@ ENDIF() PEERDIR( contrib/libs/protobuf - ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs + ydb/public/sdk/cpp/src/client/persqueue_public/codecs library/cpp/digest/md5 library/cpp/getopt library/cpp/logger @@ -61,7 +61,7 @@ PEERDIR( yql/essentials/utils/log yql/essentials/utils/backtrace yql/essentials/utils/failure_injector - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/driver yql/essentials/core/url_preprocessing yt/yql/providers/yt/comp_nodes/dq/llvm14 yt/yql/providers/yt/comp_nodes/llvm14 diff --git a/ydb/mvp/core/core_ydb.h b/ydb/mvp/core/core_ydb.h index 276551860cc2..5c014029445d 100644 --- a/ydb/mvp/core/core_ydb.h +++ b/ydb/mvp/core/core_ydb.h @@ -2,18 +2,18 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/core/core_ydb_impl.h b/ydb/mvp/core/core_ydb_impl.h index 43e56703f27e..03b974422897 100644 --- a/ydb/mvp/core/core_ydb_impl.h +++ b/ydb/mvp/core/core_ydb_impl.h @@ -7,22 +7,22 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include +#include #include + #include "appdata.h" #include "merger.h" #include "core_ydb.h" @@ -564,7 +564,7 @@ struct THandlerActorYdb { } static void WriteSchemeEntryPermissions(NJson::TJsonValue& value, - const TVector& permissions, + const std::vector& permissions, const TString& filterSubject = TString()) { value.SetType(NJson::JSON_ARRAY); THashMap> actualPermissions; @@ -573,7 +573,7 @@ struct THandlerActorYdb { continue; } auto& subjectPermissions = actualPermissions[permission.Subject]; - for (const TString& perm : permission.PermissionNames) { + for (const auto& perm : permission.PermissionNames) { subjectPermissions.emplace(perm); } } @@ -630,7 +630,7 @@ struct THandlerActorYdb { } static void WriteColumns(NJson::TJsonValue& columns, - const TVector& columnsMeta, + const std::vector& columnsMeta, const TVector& columnsKeysMeta = TVector(), const std::function& columnTypeFormatter = ColumnTypeToString) { for (const NYdb::TColumn& columnMeta : columnsMeta) { @@ -697,12 +697,12 @@ struct THandlerActorYdb { index["sizeBytes"] = indexMeta.GetSizeBytes(); NJson::TJsonValue& indexColumns = index["indexColumns"]; indexColumns.SetType(NJson::JSON_ARRAY); - for (const TString& column : indexMeta.GetIndexColumns()) { + for (const auto& column : indexMeta.GetIndexColumns()) { indexColumns.AppendValue(column); } NJson::TJsonValue& dataColumns = index["dataColumns"]; dataColumns.SetType(NJson::JSON_ARRAY); - for (const TString& column : indexMeta.GetDataColumns()) { + for (const auto& column : indexMeta.GetDataColumns()) { dataColumns.AppendValue(column); } } @@ -711,14 +711,14 @@ struct THandlerActorYdb { static void WriteShards(NJson::TJsonValue& shards, const NYdb::NTable::TTableDescription& tableDescription) { shards.SetType(NJson::JSON_ARRAY); - const TVector& ranges = tableDescription.GetKeyRanges(); - const TVector& stats = tableDescription.GetPartitionStats(); + const std::vector& ranges = tableDescription.GetKeyRanges(); + const std::vector& stats = tableDescription.GetPartitionStats(); for (ui64 nPart = 0; nPart < tableDescription.GetPartitionsCount(); ++nPart) { NJson::TJsonValue& shard = shards.AppendValue(NJson::TJsonValue()); shard.SetType(NJson::JSON_MAP); if (nPart > 0 && nPart < ranges.size()) { const NYdb::NTable::TKeyRange& range = ranges[nPart]; - NYdb::TValueParser parser(range.From().GetRef().GetValue()); + NYdb::TValueParser parser(range.From().value().GetValue()); shard["boundary"] = ColumnValueToJsonValue(parser); } if (nPart < stats.size()) { @@ -922,7 +922,7 @@ struct THandlerActorYdb { static NHttp::THttpOutgoingResponsePtr CreateStatusResponse(NHttp::THttpIncomingRequestPtr request, const NYdb::TStatus& status, const TJsonSettings& jsonSettings = TJsonSettings()) { Ydb::Operations::Operation operation; operation.set_status(static_cast(status.GetStatus())); - IssuesToMessage(status.GetIssues(), operation.mutable_issues()); + NYdb::NIssue::IssuesToMessage(status.GetIssues(), operation.mutable_issues()); return CreateStatusResponse(request, operation, jsonSettings); } diff --git a/ydb/mvp/core/grpc_log.h b/ydb/mvp/core/grpc_log.h index f08817122cc4..b0137ff25928 100644 --- a/ydb/mvp/core/grpc_log.h +++ b/ydb/mvp/core/grpc_log.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include "mvp_log.h" #include "appdata.h" diff --git a/ydb/mvp/core/mvp_tokens.h b/ydb/mvp/core/mvp_tokens.h index bfd217c54512..0da1a200335d 100644 --- a/ydb/mvp/core/mvp_tokens.h +++ b/ydb/mvp/core/mvp_tokens.h @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/core/ya.make b/ydb/mvp/core/ya.make index 83ab4e43b861..92cf86abe4ec 100644 --- a/ydb/mvp/core/ya.make +++ b/ydb/mvp/core/ya.make @@ -58,15 +58,15 @@ PEERDIR( library/cpp/lfalloc/alloc_profiler ydb/core/viewer/json ydb/core/protos - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_persqueue_core + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/library/operation_id ydb/public/api/protos ydb/public/api/grpc - ydb/public/lib/operation_id ) END() diff --git a/ydb/mvp/meta/meta.cpp b/ydb/mvp/meta/meta.cpp index 00e45146c106..b7ef01e12492 100644 --- a/ydb/mvp/meta/meta.cpp +++ b/ydb/mvp/meta/meta.cpp @@ -99,8 +99,8 @@ bool GetCacheOwnership(const TString& id, NMeta::TGetCacheOwnershipCallback cb) auto resultSet = res.GetResultSet(0); NYdb::TResultSetParser rsParser(resultSet); if (rsParser.TryNextRow()) { - TString forward = (rsParser.ColumnParser(0).GetOptionalUtf8()).GetRef(); - TInstant deadline = (rsParser.ColumnParser(1).GetOptionalTimestamp()).GetRef(); + TString forward = rsParser.ColumnParser(0).GetOptionalUtf8().value(); + TInstant deadline = rsParser.ColumnParser(1).GetOptionalTimestamp().value(); if (forward == LocalEndpoint) { MLOG_D("GetCacheOwnership(" << id << ") - got data (forward to myself until " << deadline << ")"); cb({.Deadline = deadline}); diff --git a/ydb/mvp/meta/meta_cloud.h b/ydb/mvp/meta/meta_cloud.h index 06b67ed1193a..a6ef2dba6417 100644 --- a/ydb/mvp/meta/meta_cloud.h +++ b/ydb/mvp/meta/meta_cloud.h @@ -8,12 +8,12 @@ #include #include #include -#include -#include +#include +#include #include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/meta/meta_cluster.h b/ydb/mvp/meta/meta_cluster.h index f29a8445512e..3b714bd10424 100644 --- a/ydb/mvp/meta/meta_cluster.h +++ b/ydb/mvp/meta/meta_cluster.h @@ -8,11 +8,11 @@ #include #include #include -#include -#include +#include +#include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/meta/meta_clusters.h b/ydb/mvp/meta/meta_clusters.h index 7155d15dc4e1..9f534600a617 100644 --- a/ydb/mvp/meta/meta_clusters.h +++ b/ydb/mvp/meta/meta_clusters.h @@ -8,9 +8,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include @@ -18,9 +18,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/ydb/mvp/meta/meta_cp_databases.h b/ydb/mvp/meta/meta_cp_databases.h index fe5c3e8c3028..b0b5f24fee2b 100644 --- a/ydb/mvp/meta/meta_cp_databases.h +++ b/ydb/mvp/meta/meta_cp_databases.h @@ -9,11 +9,11 @@ #include #include #include -#include -#include +#include +#include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/meta/meta_db_clusters.h b/ydb/mvp/meta/meta_db_clusters.h index e9260694d259..1fcbac88d47f 100644 --- a/ydb/mvp/meta/meta_db_clusters.h +++ b/ydb/mvp/meta/meta_db_clusters.h @@ -8,11 +8,11 @@ #include #include #include -#include -#include +#include +#include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/meta/meta_versions.cpp b/ydb/mvp/meta/meta_versions.cpp index 035798a63160..c1e1a4adb4a6 100644 --- a/ydb/mvp/meta/meta_versions.cpp +++ b/ydb/mvp/meta/meta_versions.cpp @@ -81,10 +81,10 @@ class TVersionListLoadActor : THandlerActorYdbMeta, THandlerActorYdb, public NAc while (parser.TryNextRow()) { // Almost no type checking: assume here strict conformance to the schema. // Records with empty color_class are skipped as invalid. - TMaybe color_class = parser.ColumnParser("color_class").GetOptionalUint32(); - if (!color_class.Empty()) { + auto color_class = parser.ColumnParser("color_class").GetOptionalUint32(); + if (color_class.has_value()) { versionInfoCache->emplace_back( - parser.ColumnParser("version_str").GetUtf8(), + TString{parser.ColumnParser("version_str").GetUtf8()}, *color_class ); } diff --git a/ydb/mvp/meta/ya.make b/ydb/mvp/meta/ya.make index f75d2b1792be..b712ff7a4209 100644 --- a/ydb/mvp/meta/ya.make +++ b/ydb/mvp/meta/ya.make @@ -30,6 +30,7 @@ PEERDIR( yql/essentials/core/expr_nodes ydb/library/aclib/protos library/cpp/protobuf/json + library/cpp/getopt ) YQL_LAST_ABI_VERSION() diff --git a/ydb/mvp/oidc_proxy/oidc_session_create_yandex.cpp b/ydb/mvp/oidc_proxy/oidc_session_create_yandex.cpp index 6a3c45420ddf..bc50482f4f4c 100644 --- a/ydb/mvp/oidc_proxy/oidc_session_create_yandex.cpp +++ b/ydb/mvp/oidc_proxy/oidc_session_create_yandex.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/ydb/mvp/oidc_proxy/openid_connect.h b/ydb/mvp/oidc_proxy/openid_connect.h index 76b639115098..2490520b1b2d 100644 --- a/ydb/mvp/oidc_proxy/openid_connect.h +++ b/ydb/mvp/oidc_proxy/openid_connect.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/mvp/oidc_proxy/ya.make b/ydb/mvp/oidc_proxy/ya.make index d812fa27a5b1..5aaf1b108294 100644 --- a/ydb/mvp/oidc_proxy/ya.make +++ b/ydb/mvp/oidc_proxy/ya.make @@ -26,6 +26,7 @@ SRCS( PEERDIR( ydb/mvp/core ydb/public/api/client/yc_private/oauth + library/cpp/getopt ) YQL_LAST_ABI_VERSION() diff --git a/ydb/public/lib/deprecated/client/grpc_client.h b/ydb/public/lib/deprecated/client/grpc_client.h index 0fbffcf62d4f..d878c0c9b26f 100644 --- a/ydb/public/lib/deprecated/client/grpc_client.h +++ b/ydb/public/lib/deprecated/client/grpc_client.h @@ -6,7 +6,7 @@ #include #include -#include +#include namespace NKikimr { namespace NGRpcProxy { diff --git a/ydb/public/lib/deprecated/client/ya.make b/ydb/public/lib/deprecated/client/ya.make index 59e934d70dc0..93733a98526d 100644 --- a/ydb/public/lib/deprecated/client/ya.make +++ b/ydb/public/lib/deprecated/client/ya.make @@ -8,7 +8,7 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/messagebus ydb/public/lib/base ) diff --git a/ydb/public/lib/deprecated/json_value/ut/ya.make b/ydb/public/lib/deprecated/json_value/ut/ya.make new file mode 100644 index 000000000000..1bbca3ebb03b --- /dev/null +++ b/ydb/public/lib/deprecated/json_value/ut/ya.make @@ -0,0 +1,18 @@ +UNITTEST_FOR(ydb/public/lib/json_value) + +SIZE(MEDIUM) + +FORK_SUBTESTS() + +SRCS( + ydb_json_value_ut.cpp +) + +PEERDIR( + library/cpp/json + library/cpp/testing/unittest + ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/client/ydb_params +) + +END() diff --git a/ydb/public/lib/deprecated/json_value/ya.make b/ydb/public/lib/deprecated/json_value/ya.make new file mode 100644 index 000000000000..258e12f5be01 --- /dev/null +++ b/ydb/public/lib/deprecated/json_value/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +SRCS( + ydb_json_value.cpp + ydb_json_value_ut.cpp +) + +PEERDIR( + library/cpp/json/writer + library/cpp/string_utils/base64 + ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/client/ydb_value + yql/essentials/types/uuid +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/lib/deprecated/json_value/ydb_json_value.cpp b/ydb/public/lib/deprecated/json_value/ydb_json_value.cpp new file mode 100644 index 000000000000..3a00ca0fa391 --- /dev/null +++ b/ydb/public/lib/deprecated/json_value/ydb_json_value.cpp @@ -0,0 +1,1105 @@ +#include "ydb_json_value.h" + +#include +#include +#include +#include +#include +#include + +namespace NYdb::NDeprecated { + + namespace { + + class TUtf8Transcoder + { + public: + explicit TUtf8Transcoder() + { + }; + + TStringBuf Encode(TStringBuf str) { + Buffer.clear(); + IsAscii = true; + for (CurPos = 0; CurPos < str.size(); ++CurPos) { + ui8 c = str[CurPos]; + if (c == '"' || c == '\\') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back(c); + } else if (c == '\b') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('b'); + } else if (c == '\t') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('t'); + } else if (c == '\f') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('f'); + } else if (c == '\r') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('r'); + } else if (c == '\n') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('n'); + } else if (c < '\x20' || c > '\x7E') { + SwitchToNonAscii(str); + TString tmp = Sprintf("\\u%04X", c); + for (unsigned char c : tmp) { + Buffer.push_back(c); + } + } else { + if (!IsAscii) { + Buffer.push_back(c); + } + } + } + if (IsAscii) { + return str; + } else { + return TStringBuf(Buffer.data(), Buffer.size()); + } + } + + TStringBuf Decode(TStringBuf str) { + Buffer.clear(); + IsAscii = true; + for (size_t i = 0; i < str.size(); ++i) { + char c = str[i]; + if (ui8(c) < 128) { + if (!IsAscii) { + Buffer.push_back(c); + } + } else if ((c & '\xFC') == '\xC0') { + if (IsAscii) { + Buffer.resize(i); + std::copy(str.data(), str.data() + i, Buffer.data()); + IsAscii = false; + } + Buffer.push_back(((c & '\x03') << 6) | (str[i + 1] & '\x3F')); + i += 1; + } else { + ThrowFatalError("Unicode symbols with codes greater than 255 are not supported."); + } + } + if (IsAscii) { + return str; + } else { + return TStringBuf(Buffer.data(), Buffer.size()); + } + } + + private: + void SwitchToNonAscii(TStringBuf& str) { + if (IsAscii) { + Buffer.resize(CurPos); + std::copy(str.data(), str.data() + CurPos, Buffer.data()); + IsAscii = false; + } + } + + private: + bool IsAscii; + std::vector Buffer; + size_t CurPos; + }; + + ui32 ParseNumber(ui32& pos, const::std::string_view& buf, ui32& value, i8 dig_cnt) { + ui32 count = 0U; + for (value = 0U; dig_cnt && pos < buf.size(); --dig_cnt, ++pos) { + if (const auto c = buf[pos]; c >= '0' && c <= '9') { + value = value * 10U + (c - '0'); + ++count; + continue; + } + break; + } + + return count; + } + + std::chrono::year_month_day ParseDate(ui32& pos, const std::string_view& buf) { + bool beforeChrist = false; + if (pos < buf.size()) { + switch (buf.data()[pos]) { + case '-': + beforeChrist = true; + [[fallthrough]]; + case '+': + ++pos; + [[fallthrough]]; + default: + break; + } + } + + ui32 year, month, day; + if (!ParseNumber(pos, buf, year, 6U) || pos == buf.size() || buf[pos] != '-' || + !ParseNumber(++pos, buf, month, 2U) || pos == buf.size() || buf[pos] != '-' || + !ParseNumber(++pos, buf, day, 2U)) { + return {}; + } + + const i32 iyear = beforeChrist ? -year : year; + return std::chrono::year_month_day{std::chrono::year{iyear}, std::chrono::month{month}, std::chrono::day{day}}; + } + + std::optional ParseTime(ui32& pos, const std::string_view& buf) { + ui32 hour, minute, second; + if (pos == buf.size() || buf[pos] != 'T' || + !ParseNumber(++pos, buf, hour, 2U) || pos == buf.size() || buf[pos] != ':' || + !ParseNumber(++pos, buf, minute, 2U) || pos == buf.size() || buf[pos] != ':' || + !ParseNumber(++pos, buf, second, 2U) || pos == buf.size() || + hour >= 24U || minute >= 60U || second >= 60U) { + return std::nullopt; + } + + return hour * 3600U + minute * 60U + second; + } + + constexpr i64 SecondsInDay = 86400LL; + constexpr i64 MicroMiltiplier = 1000000LL; + + std::optional ParseDateTime(ui32& pos, const std::string_view& buf) { + const auto date = ParseDate(pos, buf); + if (!date.ok()) + return std::nullopt; + + const auto time = ParseTime(pos, buf); + if (!time || buf[pos] != 'Z') + return std::nullopt; + + return i64(std::chrono::sys_days(date).time_since_epoch().count()) * SecondsInDay + i64(*time); + } + + std::optional ParseMicroseconds(ui32& pos, const std::string_view& buf) { + if (buf[pos] == '.') { + ui32 ms = 0U; + auto prevPos = ++pos; + if (!ParseNumber(pos, buf, ms, 6U)) { + return std::nullopt; + } + + for (prevPos = pos - prevPos; prevPos < 6U; ++prevPos) { + ms *= 10U; + } + + // Skip unused digits + while (pos < buf.size() && '0' <= buf[pos] && buf[pos] <= '9') { + ++pos; + } + return ms; + } + return 0U; + } + + std::optional ParseTimestamp(const std::string_view& buf) { + ui32 pos = 0U; + const auto date = ParseDate(pos, buf); + if (!date.ok()) + return std::nullopt; + + const auto time = ParseTime(pos, buf); + if (!time) + return std::nullopt; + + const auto microseconds = ParseMicroseconds(pos, buf); + if (!microseconds || buf[pos] != 'Z') + return std::nullopt; + + return (i64(std::chrono::sys_days(date).time_since_epoch().count()) * SecondsInDay + i64(*time)) * MicroMiltiplier + *microseconds; + } + + void WriteDate(TStringBuilder& out, const i32 days) { + const std::chrono::year_month_day ymd{std::chrono::sys_days(std::chrono::days(days))}; + if (!ymd.ok()) { + ThrowFatalError(TStringBuilder() << "Invalid value for date: " << days); + } + out << int(ymd.year()) << '-' << LeftPad(unsigned(ymd.month()), 2U, '0') << '-' << LeftPad(unsigned(ymd.day()), 2U, '0'); + } + + void WriteTime(TStringBuilder& out, const ui32 time) { + out << LeftPad(time / 3600, 2U, '0') << ':' << LeftPad(time % 3600 / 60, 2U, '0') << ':' << LeftPad(time % 60, 2U, '0'); + } + + void WriteDatetime(TStringBuilder& out, i64 datetime) { + auto date = datetime / SecondsInDay; + datetime -= date * SecondsInDay; + if (datetime < 0) { + --date; + datetime += SecondsInDay; + } + + WriteDate(out, i32(date)); + out << 'T'; + WriteTime(out, ui32(datetime)); + } + + TString FormatDate(i32 days) { + TStringBuilder str; + WriteDate(str, days); + return str; + } + + TString FormatDatetime(i64 datetime) { + TStringBuilder str; + WriteDatetime(str, datetime); + str << 'Z'; + return str; + } + + TString FormatTimestamp(i64 timestamp) { + auto datetime = timestamp / MicroMiltiplier; + timestamp -= datetime * MicroMiltiplier; + if (timestamp < 0) { + --datetime; + timestamp += MicroMiltiplier; + } + TStringBuilder str; + WriteDatetime(str, datetime); + if (timestamp) { + str << '.' << LeftPad(timestamp, 6U, '0'); + } + str << 'Z'; + return str; + } + + class TYdbToJsonConverter { + public: + TYdbToJsonConverter(TValueParser& parser, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) + : Parser(parser) + , Writer(writer) + , Encoding(encoding) + { + } + + void Convert() { + ParseValue(); + } + + private: + void ParsePrimitiveValue(EPrimitiveType type) { + switch (type) { + case EPrimitiveType::Bool: + Writer.WriteBool(Parser.GetBool()); + break; + case EPrimitiveType::Int8: + Writer.WriteInt(Parser.GetInt8()); + break; + case EPrimitiveType::Uint8: + Writer.WriteInt(Parser.GetUint8()); + break; + case EPrimitiveType::Int16: + Writer.WriteInt(Parser.GetInt16()); + break; + case EPrimitiveType::Uint16: + Writer.WriteInt(Parser.GetUint16()); + break; + case EPrimitiveType::Int32: + Writer.WriteInt(Parser.GetInt32()); + break; + case EPrimitiveType::Uint32: + Writer.WriteULongLong(Parser.GetUint32()); + break; + case EPrimitiveType::Int64: + Writer.WriteLongLong(Parser.GetInt64()); + break; + case EPrimitiveType::Uint64: + Writer.WriteULongLong(Parser.GetUint64()); + break; + case EPrimitiveType::Float: + Writer.WriteFloat(Parser.GetFloat(), PREC_AUTO); + break; + case EPrimitiveType::Double: + Writer.WriteDouble(Parser.GetDouble(), PREC_AUTO); + break; + case EPrimitiveType::Date: + Writer.WriteString(Parser.GetDate().FormatGmTime("%Y-%m-%d")); + break; + case EPrimitiveType::Datetime: + Writer.WriteString(Parser.GetDatetime().ToStringUpToSeconds()); + break; + case EPrimitiveType::Timestamp: + Writer.WriteString(Parser.GetTimestamp().ToString()); + break; + case EPrimitiveType::Interval: + Writer.WriteLongLong(Parser.GetInterval()); + break; + case EPrimitiveType::Date32: + Writer.WriteString(FormatDate(Parser.GetDate32())); + break; + case EPrimitiveType::Datetime64: + Writer.WriteString(FormatDatetime(Parser.GetDatetime64())); + break; + case EPrimitiveType::Timestamp64: + Writer.WriteString(FormatTimestamp(Parser.GetTimestamp64())); + break; + case EPrimitiveType::Interval64: + Writer.WriteLongLong(Parser.GetInterval64()); + break; + case EPrimitiveType::TzDate: + Writer.WriteString(Parser.GetTzDate()); + break; + case EPrimitiveType::TzDatetime: + Writer.WriteString(Parser.GetTzDatetime()); + break; + case EPrimitiveType::TzTimestamp: + Writer.WriteString(Parser.GetTzTimestamp()); + break; + case EPrimitiveType::String: + Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetString())); + break; + case EPrimitiveType::Utf8: + Writer.WriteString(Parser.GetUtf8()); + break; + case EPrimitiveType::Yson: + Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetYson())); + break; + case EPrimitiveType::Json: + Writer.WriteString(Parser.GetJson()); + break; + case EPrimitiveType::Uuid: + Writer.WriteString(Parser.GetUuid().ToString()); + break; + case EPrimitiveType::JsonDocument: + Writer.WriteString(Parser.GetJsonDocument()); + break; + case EPrimitiveType::DyNumber: + Writer.WriteString(Parser.GetDyNumber()); + break; + default: + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + } + } + + void ParseValue() { + switch (Parser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + ParsePrimitiveValue(Parser.GetPrimitiveType()); + break; + + case TTypeParser::ETypeKind::Decimal: + Writer.WriteString(Parser.GetDecimal().ToString()); + break; + + case TTypeParser::ETypeKind::Pg: + if (Parser.GetPg().IsNull()) { + Writer.WriteNull(); + } else if (Parser.GetPg().IsText()) { + Writer.WriteString(Parser.GetPg().Content_); + } else { + Writer.BeginList(); + Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetPg().Content_)); + Writer.EndList(); + } + break; + + case TTypeParser::ETypeKind::Optional: + Parser.OpenOptional(); + if (Parser.IsNull()) { + Writer.WriteNull(); + } else { + ParseValue(); + } + Parser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::Tagged: + Parser.OpenTagged(); + ParseValue(); + Parser.CloseTagged(); + break; + + case TTypeParser::ETypeKind::EmptyList: + { + Writer.BeginList(); + Writer.EndList(); + break; + } + + case TTypeParser::ETypeKind::List: + Parser.OpenList(); + Writer.BeginList(); + + while (Parser.TryNextListItem()) { + ParseValue(); + } + + Parser.CloseList(); + Writer.EndList(); + break; + + case TTypeParser::ETypeKind::Struct: + Parser.OpenStruct(); + Writer.BeginObject(); + + while (Parser.TryNextMember()) { + Writer.WriteKey(Parser.GetMemberName()); + ParseValue(); + } + + Parser.CloseStruct(); + Writer.EndObject(); + break; + + case TTypeParser::ETypeKind::Tuple: + Parser.OpenTuple(); + Writer.BeginList(); + + while (Parser.TryNextElement()) { + ParseValue(); + } + + Parser.CloseTuple(); + Writer.EndList(); + break; + + case TTypeParser::ETypeKind::EmptyDict: + { + Writer.BeginList(); + Writer.EndList(); + break; + } + + case TTypeParser::ETypeKind::Dict: + Parser.OpenDict(); + Writer.BeginList(); + + while (Parser.TryNextDictItem()) { + Writer.BeginList(); + Parser.DictKey(); + ParseValue(); + Parser.DictPayload(); + ParseValue(); + Writer.EndList(); + } + Parser.CloseDict(); + Writer.EndList(); + break; + case TTypeParser::ETypeKind::Null: + Writer.WriteNull(); + break; + default: + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << Parser.GetKind()); + } + } + + TString BinaryStringToJsonString(const TString& s) { + TStringStream str; + str << "\""; + switch (Encoding) { + case EBinaryStringEncoding::Unicode: + str << Utf8Transcoder.Encode(s); + break; + case EBinaryStringEncoding::Base64: + str << Base64Encode(s); + break; + default: + ThrowFatalError(TStringBuilder() << "Unknown binary string encode mode: " + << static_cast(Encoding)); + } + str << "\""; + return str.Str(); + } + + private: + TValueParser& Parser; + NJsonWriter::TBuf& Writer; + EBinaryStringEncoding Encoding; + TUtf8Transcoder Utf8Transcoder; + }; +} + +void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, + EBinaryStringEncoding encoding) +{ + TValueParser typeParser(value); + TYdbToJsonConverter converter(typeParser, writer, encoding); + converter.Convert(); +} + +TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding) +{ + TStringStream out; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); + + FormatValueJson(value, writer, encoding); + + return out.Str(); +} + +void FormatResultRowJson(TResultSetParser& parser, const TVector& columns, NJsonWriter::TBuf& writer, + EBinaryStringEncoding encoding) +{ + writer.BeginObject(); + for (ui32 i = 0; i < columns.size(); ++i) { + writer.WriteKey(columns[i].Name); + TYdbToJsonConverter converter(parser.ColumnParser(i), writer, encoding); + converter.Convert(); + } + writer.EndObject(); +} + +TString FormatResultRowJson(TResultSetParser& parser, const TVector& columns, + EBinaryStringEncoding encoding) +{ + TStringStream out; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); + + FormatResultRowJson(parser, columns, writer, encoding); + + return out.Str(); +} + +void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding) +{ + auto columns = result.GetColumnsMeta(); + + TResultSetParser parser(result); + + while (parser.TryNextRow()) { + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, out); + FormatResultRowJson(parser, columns, writer, encoding); + *out << Endl; + } +} + +TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding) +{ + TStringStream out; + + FormatResultSetJson(result, &out, encoding); + + return out.Str(); +} + +namespace { + class TJsonToYdbConverter { + public: + TJsonToYdbConverter(TValueBuilder& valueBuilder, const NJson::TJsonValue& jsonValue, TTypeParser& typeParser, + EBinaryStringEncoding encoding) + : ValueBuilder(valueBuilder) + , JsonValue(jsonValue) + , TypeParser(typeParser) + , Encoding(encoding) + { + } + + void Convert() { + ParseValue(JsonValue); + } + + private: + void ParsePrimitiveValue(const NJson::TJsonValue& jsonValue, EPrimitiveType type) { + switch (type) { + case EPrimitiveType::Bool: + EnsureType(jsonValue, NJson::JSON_BOOLEAN); + ValueBuilder.Bool(jsonValue.GetBoolean()); + break; + case EPrimitiveType::Int8: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits::max() || intValue < std::numeric_limits::min()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int8 type"); + } + ValueBuilder.Int8(intValue); + break; + } + case EPrimitiveType::Uint8: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits::max()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt8 type"); + } + ValueBuilder.Uint8(intValue); + break; + } + case EPrimitiveType::Int16: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits::max() || intValue < std::numeric_limits::min()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int16 type"); + } + ValueBuilder.Int16(intValue); + break; + } + case EPrimitiveType::Uint16: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits::max()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt16 type"); + } + ValueBuilder.Uint16(intValue); + break; + } + case EPrimitiveType::Int32: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits::max() || intValue < std::numeric_limits::min()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int32 type"); + } + ValueBuilder.Int32(intValue); + break; + } + case EPrimitiveType::Uint32: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits::max()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt32 type"); + } + ValueBuilder.Uint32(intValue); + break; + } + case EPrimitiveType::Int64: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits::max() || intValue < std::numeric_limits::min()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int64 type"); + } + ValueBuilder.Int64(intValue); + break; + } + case EPrimitiveType::Uint64: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits::max()) { + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt64 type"); + } + ValueBuilder.Uint64(intValue); + break; + } + case EPrimitiveType::Float: + EnsureType(jsonValue, NJson::JSON_DOUBLE); + ValueBuilder.Float(jsonValue.GetDouble()); + break; + case EPrimitiveType::Double: + EnsureType(jsonValue, NJson::JSON_DOUBLE); + ValueBuilder.Double(jsonValue.GetDouble()); + break; + case EPrimitiveType::Date: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant date; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), date)) { + ThrowFatalError(TStringBuilder() << "Can't parse date from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Date(date); + break; + } + case EPrimitiveType::Datetime: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant dateTime; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), dateTime)) { + ThrowFatalError(TStringBuilder() << "Can't parse dateTime from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Datetime(dateTime); + break; + } + case EPrimitiveType::Timestamp: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant timestamp; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), timestamp)) { + ThrowFatalError(TStringBuilder() << "Can't parse timestamp from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Timestamp(timestamp); + break; + } + case EPrimitiveType::Interval: + EnsureType(jsonValue, NJson::JSON_INTEGER); + ValueBuilder.Interval(jsonValue.GetInteger()); + break; + case EPrimitiveType::Date32: + { + EnsureType(jsonValue, NJson::JSON_STRING); + ui32 pos = 0U; + const auto date = ParseDate(pos, jsonValue.GetString()); + if (!date.ok()) { + ThrowFatalError(TStringBuilder() << "Can't parse date from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Date32(std::chrono::sys_days(date).time_since_epoch().count()); + break; + } + case EPrimitiveType::Datetime64: + { + EnsureType(jsonValue, NJson::JSON_STRING); + ui32 pos = 0U; + const auto datetime = ParseDateTime(pos, jsonValue.GetString()); + if (!datetime) { + ThrowFatalError(TStringBuilder() << "Can't parse time point from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Datetime64(*datetime); + break; + } + case EPrimitiveType::Timestamp64: + { + EnsureType(jsonValue, NJson::JSON_STRING); + const auto timestamp = ParseTimestamp(jsonValue.GetString()); + if (!timestamp) { + ThrowFatalError(TStringBuilder() << "Can't parse timestamp from string \"" << jsonValue.GetString() << "\""); + } + ValueBuilder.Timestamp64(*timestamp); + break; + } + case EPrimitiveType::Interval64: + EnsureType(jsonValue, NJson::JSON_INTEGER); + ValueBuilder.Interval64(jsonValue.GetInteger()); + break; + case EPrimitiveType::TzDate: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzDate(jsonValue.GetString()); + break; + case EPrimitiveType::TzDatetime: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzDatetime(jsonValue.GetString()); + break; + case EPrimitiveType::TzTimestamp: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzTimestamp(jsonValue.GetString()); + break; + case EPrimitiveType::String: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.String(JsonStringToBinaryString(jsonValue.GetString())); + break; + case EPrimitiveType::Utf8: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Utf8(jsonValue.GetString()); + break; + case EPrimitiveType::Yson: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Yson(JsonStringToBinaryString(jsonValue.GetString())); + break; + case EPrimitiveType::Json: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Json(jsonValue.GetString()); + break; + case EPrimitiveType::Uuid: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Uuid(jsonValue.GetString()); + break; + case EPrimitiveType::JsonDocument: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.JsonDocument(jsonValue.GetString()); + break; + case EPrimitiveType::DyNumber: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.DyNumber(jsonValue.GetString()); + break; + default: + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + } + } + + void BuildTypeInner(TTypeBuilder& typeBuilder) { + switch (TypeParser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + typeBuilder.Primitive(TypeParser.GetPrimitive()); + break; + case TTypeParser::ETypeKind::Decimal: + typeBuilder.Decimal(TypeParser.GetDecimal()); + break; + case TTypeParser::ETypeKind::Pg: + typeBuilder.Pg(TypeParser.GetPg()); + break; + case TTypeParser::ETypeKind::Optional: + TypeParser.OpenOptional(); + typeBuilder.BeginOptional(); + BuildTypeInner(typeBuilder); + typeBuilder.EndOptional(); + TypeParser.CloseOptional(); + break; + case TTypeParser::ETypeKind::List: + TypeParser.OpenList(); + typeBuilder.BeginList(); + BuildTypeInner(typeBuilder); + TypeParser.CloseList(); + typeBuilder.EndList(); + break; + case TTypeParser::ETypeKind::Struct: + TypeParser.OpenStruct(); + typeBuilder.BeginStruct(); + while (TypeParser.TryNextMember()) { + typeBuilder.AddMember(TypeParser.GetMemberName()); + BuildTypeInner(typeBuilder); + } + TypeParser.CloseStruct(); + typeBuilder.EndStruct(); + break; + case TTypeParser::ETypeKind::Tuple: + TypeParser.OpenTuple(); + typeBuilder.BeginTuple(); + while (TypeParser.TryNextMember()) { + typeBuilder.AddElement(); + BuildTypeInner(typeBuilder); + } + TypeParser.CloseTuple(); + typeBuilder.EndTuple(); + break; + case TTypeParser::ETypeKind::Dict: + TypeParser.OpenDict(); + typeBuilder.BeginDict(); + TypeParser.DictKey(); + typeBuilder.DictKey(); + BuildTypeInner(typeBuilder); + TypeParser.DictPayload(); + typeBuilder.DictPayload(); + BuildTypeInner(typeBuilder); + TypeParser.CloseDict(); + typeBuilder.EndDict(); + break; + default: + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); + } + } + + TType GetType() { + TTypeBuilder typeBuilder; + BuildTypeInner(typeBuilder); + return typeBuilder.Build(); + } + + void ParseValue(const NJson::TJsonValue& jsonValue) { + switch (TypeParser.GetKind()) { + case TTypeParser::ETypeKind::Null: + EnsureType(jsonValue, NJson::JSON_NULL); + break; + + case TTypeParser::ETypeKind::Primitive: + ParsePrimitiveValue(jsonValue, TypeParser.GetPrimitive()); + break; + + case TTypeParser::ETypeKind::Decimal: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Decimal(TDecimalValue(jsonValue.GetString(), TypeParser.GetDecimal().Precision, TypeParser.GetDecimal().Scale)); + break; + + case TTypeParser::ETypeKind::Pg: + if (jsonValue.GetType() == NJson::JSON_STRING) { + ValueBuilder.Pg(TPgValue(TPgValue::VK_TEXT, jsonValue.GetString(), TypeParser.GetPg())); + } else if (jsonValue.GetType() == NJson::JSON_NULL) { + ValueBuilder.Pg(TPgValue(TPgValue::VK_NULL, {}, TypeParser.GetPg())); + } else { + EnsureType(jsonValue, NJson::JSON_ARRAY); + if (jsonValue.GetArray().size() != 1) { + ThrowFatalError(TStringBuilder() << "Pg type should be encoded as array with size 1, but not " << jsonValue.GetArray().size()); + } + auto& innerJsonValue = jsonValue.GetArray().at(0); + EnsureType(innerJsonValue, NJson::JSON_STRING); + auto binary = JsonStringToBinaryString(innerJsonValue.GetString()); + ValueBuilder.Pg(TPgValue(TPgValue::VK_BINARY, binary, TypeParser.GetPg())); + } + break; + + case TTypeParser::ETypeKind::Optional: + TypeParser.OpenOptional(); + if (jsonValue.IsNull() && TypeParser.GetKind() != TTypeParser::ETypeKind::Optional) { + ValueBuilder.EmptyOptional(GetType()); + } else { + ValueBuilder.BeginOptional(); + ParseValue(jsonValue); + ValueBuilder.EndOptional(); + } + TypeParser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::Tagged: + TypeParser.OpenTagged(); + ValueBuilder.BeginTagged(TypeParser.GetTag()); + ParseValue(jsonValue); + ValueBuilder.EndTagged(); + TypeParser.CloseTagged(); + break; + + case TTypeParser::ETypeKind::EmptyList: + EnsureType(jsonValue, NJson::JSON_ARRAY); + break; + + case TTypeParser::ETypeKind::List: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenList(); + if (jsonValue.GetArray().empty()) { + ValueBuilder.EmptyList(GetType()); + } else { + ValueBuilder.BeginList(); + for (const auto& element : jsonValue.GetArray()) { + ValueBuilder.AddListItem(); + ParseValue(element); + } + ValueBuilder.EndList(); + } + TypeParser.CloseList(); + break; + + case TTypeParser::ETypeKind::Struct: + { + EnsureType(jsonValue, NJson::JSON_MAP); + TypeParser.OpenStruct(); + ValueBuilder.BeginStruct(); + + const auto& jsonMap = jsonValue.GetMap(); + while (TypeParser.TryNextMember()) { + const TString& memberName = TypeParser.GetMemberName(); + const auto it = jsonMap.find(memberName); + if (it == jsonMap.end()) { + ThrowFatalError(TStringBuilder() << "No member \"" << memberName + << "\" in the map in json string for YDB struct type"); + } + ValueBuilder.AddMember(memberName); + ParseValue(it->second); + } + + ValueBuilder.EndStruct(); + TypeParser.CloseStruct(); + break; + } + + case TTypeParser::ETypeKind::Tuple: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenTuple(); + ValueBuilder.BeginTuple(); + + for (const auto& element : jsonValue.GetArray()) { + if (!TypeParser.TryNextElement()) { + ThrowFatalError("Tuple in json string should contain less elements than provided"); + } + ValueBuilder.AddElement(); + ParseValue(element); + } + if (TypeParser.TryNextElement()) { + ThrowFatalError("Tuple in json string should contain more elements than provided"); + } + + ValueBuilder.EndTuple(); + TypeParser.CloseTuple(); + break; + + case TTypeParser::ETypeKind::EmptyDict: + EnsureType(jsonValue, NJson::JSON_ARRAY); + break; + + case TTypeParser::ETypeKind::Dict: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenDict(); + + if (jsonValue.GetArray().size()) { + ValueBuilder.BeginDict(); + for (const auto& keyValueElement : jsonValue.GetArray()) { + EnsureType(keyValueElement, NJson::JSON_ARRAY); + const auto& keyValueArray = keyValueElement.GetArray(); + if (keyValueArray.size() != 2) { + ThrowFatalError("Each element of a dict type in YDB must be represented with " + "exactly 2 elements in array in json string"); + } + auto it = keyValueArray.begin(); + + ValueBuilder.AddDictItem(); + + TypeParser.DictKey(); + ValueBuilder.DictKey(); + ParseValue(*it); + + TypeParser.DictPayload(); + ValueBuilder.DictPayload(); + ParseValue(*(++it)); + } + ValueBuilder.EndDict(); + } else { + TypeParser.DictKey(); + TType keyType = GetType(); + TypeParser.DictPayload(); + TType payloadType = GetType(); + ValueBuilder.EmptyDict(keyType, payloadType); + } + + TypeParser.CloseDict(); + break; + + default: + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); + } + + } + + TString JsonStringToBinaryString(const TString& s) { + TStringStream str; + switch (Encoding) { + case EBinaryStringEncoding::Unicode: + str << Utf8Transcoder.Decode(s); + break; + case EBinaryStringEncoding::Base64: + str << Base64Decode(s); + break; + default: + ThrowFatalError("Unknown binary string encode mode"); + break; + } + return str.Str(); + } + + void EnsureType(const NJson::TJsonValue& value, NJson::EJsonValueType type) { + if (value.GetType() != type) { + if (value.GetType() == NJson::EJsonValueType::JSON_INTEGER && type == NJson::EJsonValueType::JSON_UINTEGER + || value.GetType() == NJson::EJsonValueType::JSON_UINTEGER && type == NJson::EJsonValueType::JSON_INTEGER) { + return; + } + if ((value.GetType() == NJson::EJsonValueType::JSON_INTEGER || value.GetType() == NJson::EJsonValueType::JSON_UINTEGER) + && type == NJson::EJsonValueType::JSON_DOUBLE) { + return; + } + TStringStream str; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &str); + writer.WriteJsonValue(&value); + ThrowFatalError(TStringBuilder() << "Wrong type for json value \"" << str.Str() + << "\". Expected type: " << type << ", received type: " << value.GetType() << ". "); + } + } + + private: + TValueBuilder& ValueBuilder; + const NJson::TJsonValue& JsonValue; + TTypeParser& TypeParser; + EBinaryStringEncoding Encoding; + TUtf8Transcoder Utf8Transcoder; + }; +} + +TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding) { + NJson::TJsonValue jsonValue; + + try { + if (!NJson::ReadJsonTree(jsonString, &jsonValue, true)) { + ThrowFatalError(TStringBuilder() << "Can't parse string \"" << jsonString << "\" as json."); + } + } + catch (std::exception& e) { + ThrowFatalError( + TStringBuilder() << "Exception while parsing string \"" << jsonString << "\" as json: " << e.what()); + } + return JsonToYdbValue(jsonValue, type, encoding); +} + +TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding) { + TValueBuilder builder; + TTypeParser typeParser(type); + + TJsonToYdbConverter converter(builder, jsonValue, typeParser, encoding); + converter.Convert(); + + return builder.Build(); +} + +} // namespace NYdb diff --git a/ydb/public/lib/deprecated/json_value/ydb_json_value.h b/ydb/public/lib/deprecated/json_value/ydb_json_value.h new file mode 100644 index 000000000000..7093e10f6653 --- /dev/null +++ b/ydb/public/lib/deprecated/json_value/ydb_json_value.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include + +namespace NYdb::NDeprecated { + +enum class EBinaryStringEncoding { + /* For binary strings, encode every byte that is not a printable ascii symbol (codes 32-126) as utf-8. + * Exceptions: \\ \" \b \t \f \r \n + * Example: "Hello\x01" -> "Hello\\u0001" + */ + Unicode = 1, + + /* Encode binary strings to base64 + */ + Base64 +}; + +// ====== YDB to json ====== +void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); + +TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding); + +void FormatResultRowJson(TResultSetParser& parser, const TVector& columns, NJsonWriter::TBuf& writer, + EBinaryStringEncoding encoding); + +TString FormatResultRowJson(TResultSetParser& parser, const TVector& columns, + EBinaryStringEncoding encoding); + +void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding); + +TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding); + +// ====== json to YDB ====== +TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding); +TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding); + +} // namespace NYdb diff --git a/ydb/public/lib/deprecated/json_value/ydb_json_value_ut.cpp b/ydb/public/lib/deprecated/json_value/ydb_json_value_ut.cpp new file mode 100644 index 000000000000..6fb271b22308 --- /dev/null +++ b/ydb/public/lib/deprecated/json_value/ydb_json_value_ut.cpp @@ -0,0 +1,697 @@ +#include "ydb_json_value.h" + +#include +#include +#include +#include +#include + +namespace NYdb::NDeprecated { + +Y_UNIT_TEST_SUITE(JsonValueTest) { + Y_UNIT_TEST(PrimitiveValueBool) { + TValue value = TValueBuilder() + .Bool(true) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "true"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt8) { + TValue value = TValueBuilder() + .Int8(-128) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-128"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint8) { + TValue value = TValueBuilder() + .Uint8(255) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "255"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt16) { + TValue value = TValueBuilder() + .Int16(-32768) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-32768"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint16) { + TValue value = TValueBuilder() + .Uint16(65535) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "65535"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt32) { + TValue value = TValueBuilder() + .Int32(-2147483648) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-2147483648"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint32) { + TValue value = TValueBuilder() + .Uint32(4294967295) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "4294967295"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt64) { + TValue value = TValueBuilder() + .Int64(-9223372036854775807 - 1) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-9223372036854775808"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint64) { + TValue value = TValueBuilder() + .Uint64(18446744073709551615ull) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "18446744073709551615"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueFloat) { + TValue value = TValueBuilder() + .Float(0.1234567890123456789) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "0.12345679"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDouble) { + TValue value = TValueBuilder() + .Double(0.1234567890123456789) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "0.12345678901234568"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDate) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02"); + TValue value = TValueBuilder() + .Date(TInstant::Days(timestamp.Days())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDatetime) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05Z"); + TValue value = TValueBuilder() + .Datetime(TInstant::Seconds(timestamp.Seconds())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueTimestamp) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05.678901Z"); + TValue value = TValueBuilder() + .Timestamp(TInstant::MicroSeconds(timestamp.MicroSeconds())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05.678901Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDate32) { + TValue value = TValueBuilder() + .Date32(-10) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("1969-12-22")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDatetime64) { + TValue value = TValueBuilder() + .Datetime64(-10) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("1969-12-31T23:59:50Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueTimestamp64) { + TValue value = TValueBuilder() + .Timestamp64(-10) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("1969-12-31T23:59:59.999990Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueSimpleString) { + TValue value = TValueBuilder() + .String("Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ") + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + jsonString, + "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + namespace { + TString GenerateBinaryString() { + TStringStream str; + for (ui8 i = 0; i < 255; ++i) { + str << static_cast(i); + } + str << static_cast(0xff); + return str.Str(); + } + } + + Y_UNIT_TEST(BinaryStringAsciiFollowedByNonAscii) { + TString binaryString = TStringBuilder() << "abc" << static_cast(0xff) + << static_cast(0xfe); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + jsonString, + R"("abc\u00FF\u00FE")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(BinaryStringUnicode) { + TString binaryString = GenerateBinaryString(); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + TStringBuilder from0To255Utf8; + from0To255Utf8 << "\""; + from0To255Utf8 << + R"(\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F)" + R"(\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F)" + " !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + R"(\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F)" + R"(\u0090\u0091\u0092\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F)" + R"(\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF)" + R"(\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF)" + R"(\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF)" + R"(\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE\u00DF)" + R"(\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF)" + R"(\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF)"; + from0To255Utf8 << "\""; + UNIT_ASSERT_NO_DIFF( + jsonString, + from0To255Utf8 + ); + + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(BinaryStringBase64) { + TString binaryString = GenerateBinaryString(); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + jsonString, + R"("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHS)" + R"(ElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJG)" + R"(Sk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna2)" + R"(9zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(EmptyBinaryStringUnicode) { + TString binaryString; + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(EmptyBinaryStringBase64) { + TString binaryString; + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(EmptyList) { + TValue value = TValueBuilder() + .EmptyList(TTypeBuilder().Primitive(EPrimitiveType::Int64).Build()) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + namespace { + TString InvalidJsonToBinaryStringBase(const TString& jsonString) { + TString errorMessage; + try { + TType stringType = TTypeBuilder().Primitive(EPrimitiveType::String).Build(); + TValue resultValue = JsonToYdbValue(jsonString, stringType, EBinaryStringEncoding::Unicode); + UNIT_FAIL("Exception should have been thrown, but it hasn't"); + } + catch (TContractViolation& e) { + errorMessage = e.what(); + } + catch (std::exception& e) { + UNIT_FAIL(TStringBuilder() << "Uncaught exception: " << e.what()); + } + return errorMessage; + } + } + + Y_UNIT_TEST(InvalidJsonToBinaryString1) { + TString jsonString = R"(some string")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Invalid value" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString2) { + TString jsonString = R"("some string)"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Missing a closing quotation mark in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString3) { + TString jsonString = "\"some string \\\""; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Missing a closing quotation mark in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString4) { + TString jsonString = R"("some \ string")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Invalid escape character in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString5) { + TString jsonString = R"("some string \u001")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Incorrect hex digit after \\u escape in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString6) { + TString jsonString = R"("some string \u0140")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Unicode symbols with codes greater than 255 are not supported" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString7) { + TString jsonString = R"("some string \u00AG")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Incorrect hex digit after \\u escape in string" + ); + } + + Y_UNIT_TEST(PrimitiveValueUtf8String1) { + TString utf8Str = "Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() "; + TValue value = TValueBuilder() + .Utf8(utf8Str) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + + UNIT_ASSERT_NO_DIFF( + jsonString, + "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUtf8String2) { + TString utf8Str = "\xD0\xB0\xD0\xB1\xD0\xB2\xD0\xB3\xD0\xB4\xD0\xB5\xD1\x91\xD0\xB6\xD0\xB7\xD0\xB8\xD0\xBA\xD0\xBB" + "\xD0\xBC\xD0\xBD\xD0\xBE\xD0\xBF\xD1\x80\xD1\x81\xD1\x82\xD1\x83\xD1\x84\xD1\x85\xD1\x86\xD1\x87\xD1\x88" + "\xD1\x89\xD1\x8A\xD1\x8B\xD1\x8C\xD1\x8D\xD1\x8E\xD1\x8F\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95" + "\xD0\x81\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2" + "\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF"; + TValue value = TValueBuilder() + .Utf8(utf8Str) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + + UNIT_ASSERT_NO_DIFF( + jsonString, + TStringBuilder() << '"' << utf8Str << '"' + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(TaggedValue) { + TValue value = TValueBuilder() + .BeginTagged("my_tag") + .BeginTagged("my_inner_tag") + .Uint32(1) + .EndTagged() + .EndTagged() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"(1)"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueEmptyList) { + TValue value = TValueBuilder() + .EmptyList(TTypeBuilder().Primitive(EPrimitiveType::Uint32).Build()) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueIntList) { + TValue value = TValueBuilder() + .BeginList() + .AddListItem() + .Int32(1) + .AddListItem() + .Int32(10) + .AddListItem() + .Int32(100) + .EndList() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[1,10,100]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueStruct) { + TValue value = TValueBuilder() + .BeginStruct() + .AddMember("Id") + .Uint32(1) + .AddMember("Name") + .String("Anna") + .AddMember("Value") + .Int32(-100) + .AddMember("Description") + .EmptyOptional(EPrimitiveType::Utf8) + .EndStruct() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"({"Id":1,"Name":"Anna","Value":-100,"Description":null})"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(NewDatetimeValuesStruct) { + TValue value = TValueBuilder() + .BeginStruct() + .AddMember("Interval64") + .Interval64(1) + .AddMember("Date32") + .Date32(-1024) + .AddMember("Datetime64") + .Datetime64(-123456789) + .AddMember("Timestamp64") + .Timestamp64(-123456789000100LL) + .EndStruct() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"({"Interval64":1,"Date32":"1967-03-14","Datetime64":"1966-02-02T02:26:51Z","Timestamp64":"1966-02-02T02:26:50.999900Z"})"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueTuple) { + TValue value = TValueBuilder() + .BeginTuple() + .AddElement() + .BeginOptional() + .BeginOptional() + .Int32(10) + .EndOptional() + .EndOptional() + .AddElement() + .BeginOptional() + .BeginOptional() + .BeginOptional() + .Int64(-1) + .EndOptional() + .EndOptional() + .EndOptional() + .AddElement() + .BeginOptional() + .EmptyOptional(TTypeBuilder().Primitive(EPrimitiveType::String).Build()) + .EndOptional() + .AddElement() + .BeginOptional() + .BeginOptional() + .EmptyOptional(EPrimitiveType::Utf8) + .EndOptional() + .EndOptional() + .EndTuple() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[10,-1,null,null]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueDict) { + TValue value = TValueBuilder() + .BeginDict() + .AddDictItem() + .DictKey() + .Int64(1) + .DictPayload() + .String("Value1") + .AddDictItem() + .DictKey() + .Int64(2) + .DictPayload() + .String("Value2") + .EndDict() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"([[1,"Value1"],[2,"Value2"]])"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PgValue) { + TPgType pgType(""); + TPgValue v1(TPgValue::VK_TEXT, "text_value", pgType); + TPgValue v2(TPgValue::VK_BINARY, "binary_value", pgType); + TPgValue v3(TPgValue::VK_TEXT, "", pgType); + TPgValue v4(TPgValue::VK_BINARY, "", pgType); + TPgValue v5(TPgValue::VK_NULL, "", pgType); + + TValue value = TValueBuilder() + .BeginList() + .AddListItem() + .Pg(v1) + .AddListItem() + .Pg(v2) + .AddListItem() + .Pg(v3) + .AddListItem() + .Pg(v4) + .AddListItem() + .Pg(v5) + .EndList() + .Build(); + + // unicode + const TString jsonString1 = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString1, R"(["text_value",["binary_value"],"",[""],null])"); + + TValue resultValue1 = JsonToYdbValue(jsonString1, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue1).DebugString() + ); + + // base64 + const TString jsonString2 = FormatValueJson(value, EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF(jsonString2, R"(["text_value",["YmluYXJ5X3ZhbHVl"],"",[""],null])"); + + TValue resultValue2 = JsonToYdbValue(jsonString2, value.GetType(), EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue2).DebugString() + ); + } +} + +} // namespace NYdb diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp index da09beaa98c1..daf35e59137e 100644 --- a/ydb/public/lib/deprecated/kicli/kikimr.cpp +++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp @@ -281,7 +281,7 @@ class TKikimr::TGRpcImpl : public TKikimr::TImpl { virtual TString GetCurrentLocation() const override { TString host; ui32 port; - NMsgBusProxy::TMsgBusClientConfig::CrackAddress(GRpcClient->GetConfig().Locator, host, port); + NMsgBusProxy::TMsgBusClientConfig::CrackAddress(TString{GRpcClient->GetConfig().Locator}, host, port); return host; } @@ -301,7 +301,7 @@ class TKikimr::TGRpcImpl : public TKikimr::TImpl { NGRpcProxy::TGRpcClientConfig config(GRpcClient->GetConfig()); TString hostname; ui32 port; - NMsgBusProxy::TMsgBusClientConfig::CrackAddress(config.Locator, hostname, port); + NMsgBusProxy::TMsgBusClientConfig::CrackAddress(TString{config.Locator}, hostname, port); TString newLocation = TStringBuilder() << EscapeIPv6(location) << ':' << port; if (newLocation == config.Locator) { return TCleanupCallback(); diff --git a/ydb/public/lib/deprecated/kicli/ya.make b/ydb/public/lib/deprecated/kicli/ya.make index c14ef26e6fe1..080193aa256e 100644 --- a/ydb/public/lib/deprecated/kicli/ya.make +++ b/ydb/public/lib/deprecated/kicli/ya.make @@ -20,6 +20,7 @@ PEERDIR( library/cpp/threading/future ydb/core/protos ydb/library/aclib + ydb/library/yql/public/ydb_issue ydb/public/api/grpc ydb/public/api/grpc/draft ydb/public/api/protos diff --git a/ydb/public/lib/deprecated/ya.make b/ydb/public/lib/deprecated/ya.make index 84dab1af07b5..700b51343cf2 100644 --- a/ydb/public/lib/deprecated/ya.make +++ b/ydb/public/lib/deprecated/ya.make @@ -1,4 +1,6 @@ RECURSE( client + json_value kicli + yson_value ) diff --git a/ydb/public/lib/deprecated/yson_value/ya.make b/ydb/public/lib/deprecated/yson_value/ya.make new file mode 100644 index 000000000000..6e7149092179 --- /dev/null +++ b/ydb/public/lib/deprecated/yson_value/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( + ydb_yson_value.cpp +) + +PEERDIR( + library/cpp/yson + library/cpp/yson/node + ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/client/ydb_value + yql/essentials/types/uuid +) + +END() diff --git a/ydb/public/lib/deprecated/yson_value/ydb_yson_value.cpp b/ydb/public/lib/deprecated/yson_value/ydb_yson_value.cpp new file mode 100644 index 000000000000..edc334a2471a --- /dev/null +++ b/ydb/public/lib/deprecated/yson_value/ydb_yson_value.cpp @@ -0,0 +1,277 @@ +#include "ydb_yson_value.h" + +#include +#include + +#include + +namespace NYdb::NDeprecated { + +static void PrimitiveValueToYson(EPrimitiveType type, TValueParser& parser, NYson::TYsonWriter& writer) +{ + switch (type) { + case EPrimitiveType::Bool: + writer.OnBooleanScalar(parser.GetBool()); + break; + case EPrimitiveType::Int8: + writer.OnInt64Scalar(parser.GetInt8()); + break; + case EPrimitiveType::Uint8: + writer.OnUint64Scalar(parser.GetUint8()); + break; + case EPrimitiveType::Int16: + writer.OnInt64Scalar(parser.GetInt16()); + break; + case EPrimitiveType::Uint16: + writer.OnUint64Scalar(parser.GetUint16()); + break; + case EPrimitiveType::Int32: + writer.OnInt64Scalar(parser.GetInt32()); + break; + case EPrimitiveType::Uint32: + writer.OnUint64Scalar(parser.GetUint32()); + break; + case EPrimitiveType::Int64: + writer.OnInt64Scalar(parser.GetInt64()); + break; + case EPrimitiveType::Uint64: + writer.OnUint64Scalar(parser.GetUint64()); + break; + case EPrimitiveType::Float: + writer.OnDoubleScalar(parser.GetFloat()); + break; + case EPrimitiveType::Double: + writer.OnDoubleScalar(parser.GetDouble()); + break; + case EPrimitiveType::Date: + writer.OnUint64Scalar(parser.GetDate().Days()); + break; + case EPrimitiveType::Datetime: + writer.OnUint64Scalar(parser.GetDatetime().Seconds()); + break; + case EPrimitiveType::Timestamp: + writer.OnUint64Scalar(parser.GetTimestamp().MicroSeconds()); + break; + case EPrimitiveType::Interval: + writer.OnInt64Scalar(parser.GetInterval()); + break; + case EPrimitiveType::Date32: + writer.OnInt64Scalar(parser.GetDate32()); + break; + case EPrimitiveType::Datetime64: + writer.OnInt64Scalar(parser.GetDatetime64()); + break; + case EPrimitiveType::Timestamp64: + writer.OnInt64Scalar(parser.GetTimestamp64()); + break; + case EPrimitiveType::Interval64: + writer.OnInt64Scalar(parser.GetInterval64()); + break; + case EPrimitiveType::TzDate: + writer.OnStringScalar(parser.GetTzDate()); + break; + case EPrimitiveType::TzDatetime: + writer.OnStringScalar(parser.GetTzDatetime()); + break; + case EPrimitiveType::TzTimestamp: + writer.OnStringScalar(parser.GetTzTimestamp()); + break; + case EPrimitiveType::String: + writer.OnStringScalar(parser.GetString()); + break; + case EPrimitiveType::Utf8: + writer.OnStringScalar(parser.GetUtf8()); + break; + case EPrimitiveType::Yson: + writer.OnStringScalar(parser.GetYson()); + break; + case EPrimitiveType::Json: + writer.OnStringScalar(parser.GetJson()); + break; + case EPrimitiveType::JsonDocument: + writer.OnStringScalar(parser.GetJsonDocument()); + break; + case EPrimitiveType::Uuid: + writer.OnStringScalar(parser.GetUuid().ToString()); + break; + case EPrimitiveType::DyNumber: + writer.OnStringScalar(parser.GetDyNumber()); + break; + default: + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + } +} + +static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& writer) +{ + switch (parser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + PrimitiveValueToYson(parser.GetPrimitiveType(), parser, writer); + break; + + case TTypeParser::ETypeKind::Decimal: + writer.OnStringScalar(parser.GetDecimal().ToString()); + break; + + case TTypeParser::ETypeKind::Pg: + if (parser.GetPg().IsNull()) { + writer.OnEntity(); + } else if (parser.GetPg().IsText()) { + writer.OnStringScalar(parser.GetPg().Content_); + } else { + writer.OnBeginList(); + writer.OnListItem(); + writer.OnStringScalar(parser.GetPg().Content_); + writer.OnEndList(); + } + break; + + case TTypeParser::ETypeKind::Optional: + parser.OpenOptional(); + if (parser.IsNull()) { + writer.OnEntity(); + } else { + writer.OnBeginList(); + writer.OnListItem(); + FormatValueYsonInternal(parser, writer); + writer.OnEndList(); + } + parser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::Tagged: + parser.OpenTagged(); + FormatValueYsonInternal(parser, writer); + parser.CloseTagged(); + break; + + case TTypeParser::ETypeKind::EmptyList: + writer.OnBeginList(); + writer.OnEndList(); + break; + + case TTypeParser::ETypeKind::List: + parser.OpenList(); + writer.OnBeginList(); + + while (parser.TryNextListItem()) { + writer.OnListItem(); + FormatValueYsonInternal(parser, writer); + } + + writer.OnEndList(); + parser.CloseList(); + break; + + case TTypeParser::ETypeKind::Struct: + parser.OpenStruct(); + writer.OnBeginList(); + + while (parser.TryNextMember()) { + writer.OnListItem(); + FormatValueYsonInternal(parser, writer); + } + + writer.OnEndList(); + parser.CloseStruct(); + break; + + case TTypeParser::ETypeKind::Tuple: + parser.OpenTuple(); + writer.OnBeginList(); + + while (parser.TryNextElement()) { + writer.OnListItem(); + FormatValueYsonInternal(parser, writer); + } + + writer.OnEndList(); + parser.CloseTuple(); + break; + + case TTypeParser::ETypeKind::EmptyDict: + writer.OnBeginList(); + writer.OnEndList(); + break; + + case TTypeParser::ETypeKind::Dict: + parser.OpenDict(); + writer.OnBeginList(); + while (parser.TryNextDictItem()) { + writer.OnListItem(); + writer.OnBeginList(); + + writer.OnListItem(); + parser.DictKey(); + FormatValueYsonInternal(parser, writer); + + writer.OnListItem(); + parser.DictPayload(); + FormatValueYsonInternal(parser, writer); + + writer.OnEndList(); + } + writer.OnEndList(); + parser.CloseDict(); + break; + + case TTypeParser::ETypeKind::Void: + writer.OnStringScalar("Void"); + break; + + case TTypeParser::ETypeKind::Null: + writer.OnEntity(); + break; + + default: + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << parser.GetKind()); + } +} + +void FormatValueYson(const TValue& value, NYson::TYsonWriter& writer) +{ + TValueParser parser(value); + FormatValueYsonInternal(parser, writer); +} + +TString FormatValueYson(const TValue& value, NYson::EYsonFormat ysonFormat) +{ + TStringStream out; + NYson::TYsonWriter writer(&out, ysonFormat, ::NYson::EYsonType::Node, true); + + FormatValueYson(value, writer); + + return out.Str(); +} + +void FormatResultSetYson(const TResultSet& result, NYson::TYsonWriter& writer) +{ + auto columns = result.GetColumnsMeta(); + + TResultSetParser parser(result); + writer.OnBeginList(); + + while (parser.TryNextRow()) { + writer.OnListItem(); + writer.OnBeginList(); + for (ui32 i = 0; i < columns.size(); ++i) { + writer.OnListItem(); + FormatValueYsonInternal(parser.ColumnParser(i), writer); + } + writer.OnEndList(); + } + + writer.OnEndList(); +} + +TString FormatResultSetYson(const TResultSet& result, NYson::EYsonFormat ysonFormat) +{ + TStringStream out; + NYson::TYsonWriter writer(&out, ysonFormat, ::NYson::EYsonType::Node, true); + + FormatResultSetYson(result, writer); + + return out.Str(); +} + +} // namespace NYdb diff --git a/ydb/public/lib/deprecated/yson_value/ydb_yson_value.h b/ydb/public/lib/deprecated/yson_value/ydb_yson_value.h new file mode 100644 index 000000000000..6e2bf3224642 --- /dev/null +++ b/ydb/public/lib/deprecated/yson_value/ydb_yson_value.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +namespace NYdb::NDeprecated { + +void FormatValueYson(const TValue& value, NYson::TYsonWriter& writer); + +TString FormatValueYson(const TValue& value, NYson::EYsonFormat ysonFormat = NYson::EYsonFormat::Text); + +void FormatResultSetYson(const TResultSet& result, NYson::TYsonWriter& writer); + +TString FormatResultSetYson(const TResultSet& result, NYson::EYsonFormat ysonFormat = NYson::EYsonFormat::Text); + +} // namespace NYdb diff --git a/ydb/public/lib/experimental/ya.make b/ydb/public/lib/experimental/ya.make index c70bfc809573..4391868f6ecb 100644 --- a/ydb/public/lib/experimental/ya.make +++ b/ydb/public/lib/experimental/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( ydb_clickhouse_internal.cpp ydb_logstore.cpp @@ -9,8 +13,8 @@ SRCS( PEERDIR( ydb/core/scheme ydb/public/api/grpc/draft - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp index eb0300489775..91205357b643 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp @@ -1,12 +1,12 @@ #include "ydb_clickhouse_internal.h" #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include -#include -#include +#include +#include // TODO: Bad dependency??? #include diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.h b/ydb/public/lib/experimental/ydb_clickhouse_internal.h index 21b1db98b6e6..4dfbfb6e4216 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.h +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include // TODO: Bad dependency??? diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp index 510f50bb3f70..0bceecfa955f 100644 --- a/ydb/public/lib/experimental/ydb_logstore.cpp +++ b/ydb/public/lib/experimental/ydb_logstore.cpp @@ -1,13 +1,13 @@ #include "ydb_logstore.h" #define INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #undef INCLUDE_YDB_INTERNAL_H #include -#include -#include +#include +#include #include #include @@ -85,7 +85,7 @@ TSchema::TSchema(const Ydb::LogStore::Schema& schema) void TSchema::SerializeTo(Ydb::LogStore::Schema& schema) const { for (const auto& c : Columns) { auto& col = *schema.add_columns(); - col.set_name(c.Name); + col.set_name(TString(c.Name)); col.mutable_type()->CopyFrom(TProtoAccessor::GetProto(c.Type)); } for (const auto& pkc : PrimaryKeyColumns) { diff --git a/ydb/public/lib/experimental/ydb_logstore.h b/ydb/public/lib/experimental/ydb_logstore.h index be8148b023f3..2e7b408fbd30 100644 --- a/ydb/public/lib/experimental/ydb_logstore.h +++ b/ydb/public/lib/experimental/ydb_logstore.h @@ -1,10 +1,12 @@ #pragma once -#include -#include -#include +#include +#include +#include #include +#include + namespace Ydb::LogStore { class Schema; diff --git a/ydb/public/lib/experimental/ydb_object_storage.cpp b/ydb/public/lib/experimental/ydb_object_storage.cpp index 51f262402947..489b237432c4 100644 --- a/ydb/public/lib/experimental/ydb_object_storage.cpp +++ b/ydb/public/lib/experimental/ydb_object_storage.cpp @@ -1,7 +1,7 @@ #include "ydb_object_storage.h" #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include namespace NYdb { namespace NObjectStorage { @@ -82,8 +82,8 @@ class TObjectStorageClient::TImpl : public TClientImplCommonstatus(); - NYql::TIssues issues; - NYql::IssuesFromMessage(response->issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(response->issues(), issues); status = TPlainStatus(static_cast(msgStatus), std::move(issues)); for (auto commonPrefix : response->Getcommon_prefixes()) { diff --git a/ydb/public/lib/experimental/ydb_object_storage.h b/ydb/public/lib/experimental/ydb_object_storage.h index 89a32af437d4..e16308d33a47 100644 --- a/ydb/public/lib/experimental/ydb_object_storage.h +++ b/ydb/public/lib/experimental/ydb_object_storage.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include namespace NYdb { namespace NObjectStorage { diff --git a/ydb/public/lib/fq/fq.cpp b/ydb/public/lib/fq/fq.cpp index f942f2af764a..64c82548ae65 100644 --- a/ydb/public/lib/fq/fq.cpp +++ b/ydb/public/lib/fq/fq.cpp @@ -1,10 +1,10 @@ #include "fq.h" #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H -#include +#include namespace NYdb::NFq { diff --git a/ydb/public/lib/fq/fq.h b/ydb/public/lib/fq/fq.h index ec00b79371e3..8d30f15d92eb 100644 --- a/ydb/public/lib/fq/fq.h +++ b/ydb/public/lib/fq/fq.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/ydb/public/lib/fq/ya.make b/ydb/public/lib/fq/ya.make index b357a169b3e7..65b27b99d7dc 100644 --- a/ydb/public/lib/fq/ya.make +++ b/ydb/public/lib/fq/ya.make @@ -1,5 +1,9 @@ LIBRARY() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( fq.cpp scope.cpp @@ -8,7 +12,7 @@ SRCS( PEERDIR( library/cpp/json ydb/public/api/grpc/draft - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/lib/idx_test/idx_test.h b/ydb/public/lib/idx_test/idx_test.h index 4bee3797587f..f37ad6c94bd6 100644 --- a/ydb/public/lib/idx_test/idx_test.h +++ b/ydb/public/lib/idx_test/idx_test.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/ydb/public/lib/idx_test/idx_test_checker.cpp b/ydb/public/lib/idx_test/idx_test_checker.cpp index 74ca8b9fc18a..5c15613b7713 100644 --- a/ydb/public/lib/idx_test/idx_test_checker.cpp +++ b/ydb/public/lib/idx_test/idx_test_checker.cpp @@ -23,7 +23,7 @@ class TChecker : public IChecker { if (!TableDescription_) { throw yexception() << "Unable to load table description"; } - if (!TableDescription_.GetRef().GetIndexDescriptions()) { + if (TableDescription_.GetRef().GetIndexDescriptions().empty()) { throw yexception() << "Index metadata was not found"; } CheckIndexes(tableName); @@ -32,8 +32,8 @@ class TChecker : public IChecker { private: void CheckIndexData(const TString& tableName, const TIndexDescription& indexDesc, - const THashMap& indexedColumnsMap, - const THashMap>& mainTable) { + const THashMap& indexedColumnsMap, + const THashMap>& mainTable) { TMaybe tableIterator; auto settings = TReadTableSettings(); @@ -86,7 +86,7 @@ class TChecker : public IChecker { if (ProgressTracker_) { ProgressTracker_->Start("rows read", "Reading indexTable table " + indexTableName + "..."); } - ThrowOnError(Client_.RetryOperationSync([indexTableName, settings, &tableIterator](TSession session) { + NStatusHelpers::ThrowOnError(Client_.RetryOperationSync([indexTableName, settings, &tableIterator](TSession session) { auto result = session.ReadTable(indexTableName, settings).GetValueSync(); if (result.IsSuccess()) { @@ -104,7 +104,7 @@ class TChecker : public IChecker { break; } - ThrowOnError(tablePart); + NStatusHelpers::ThrowOnError(tablePart); } auto rsParser = TResultSetParser(tablePart.ExtractPart()); @@ -150,9 +150,9 @@ class TChecker : public IChecker { void ReadMainTable( const TString& tableName, - const TVector& columns, - const THashMap& columnMap, - THashMap>& buf) + const std::vector& columns, + const THashMap& columnMap, + THashMap>& buf) { TMaybe tableIterator; @@ -182,7 +182,7 @@ class TChecker : public IChecker { } } - ThrowOnError(Client_.RetryOperationSync([tableName, settings, &tableIterator](TSession session) { + NStatusHelpers::ThrowOnError(Client_.RetryOperationSync([tableName, settings, &tableIterator](TSession session) { auto result = session.ReadTable(tableName, settings).GetValueSync(); if (result.IsSuccess()) { @@ -200,7 +200,7 @@ class TChecker : public IChecker { break; } - ThrowOnError(tablePart); + NStatusHelpers::ThrowOnError(tablePart); } auto rsParser = TResultSetParser(tablePart.ExtractPart()); @@ -234,9 +234,9 @@ class TChecker : public IChecker { // or pk and indexes // Columns need to check index - TVector indexedColumns; + std::vector indexedColumns; // Map column name -> position in result row in data table - THashMap indexedColumnsMap; + THashMap indexedColumnsMap; size_t id = 0; for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { @@ -258,7 +258,7 @@ class TChecker : public IChecker { } // Data table - THashMap> mainTable; + THashMap> mainTable; ReadMainTable(tableName, indexedColumns, indexedColumnsMap, mainTable); for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { diff --git a/ydb/public/lib/idx_test/idx_test_common.cpp b/ydb/public/lib/idx_test/idx_test_common.cpp index 404fa8dfafc5..53e814b84b31 100644 --- a/ydb/public/lib/idx_test/idx_test_common.cpp +++ b/ydb/public/lib/idx_test/idx_test_common.cpp @@ -1,9 +1,9 @@ #include "idx_test_common.h" -#include +#include using namespace NYdb; -using namespace NYdb::NTable; +using namespace NYdb::V3::NTable; namespace NIdxTest { @@ -22,7 +22,7 @@ TMaybe DescribeTable(const TString& tableName, TTableClient c return describeResult; }; auto status = client.RetryOperationSync(describeFn); - ThrowOnError(status); + NStatusHelpers::ThrowOnError(status); return res; } diff --git a/ydb/public/lib/idx_test/idx_test_common.h b/ydb/public/lib/idx_test/idx_test_common.h index 469844a28982..d4dfcde24a5d 100644 --- a/ydb/public/lib/idx_test/idx_test_common.h +++ b/ydb/public/lib/idx_test/idx_test_common.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NYdb { namespace NTable { @@ -11,20 +11,6 @@ namespace NTable { namespace NIdxTest { -class TYdbErrorException : public yexception { -public: - TYdbErrorException(const NYdb::TStatus& status) - : Status(status) {} - - NYdb::TStatus Status; -}; - -inline void ThrowOnError(const NYdb::TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } -} - TMaybe DescribeTable(const TString& tableName, NYdb::NTable::TTableClient client); } diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.cpp b/ydb/public/lib/idx_test/idx_test_data_provider.cpp index bd0319d89c89..3d663e359750 100644 --- a/ydb/public/lib/idx_test/idx_test_data_provider.cpp +++ b/ydb/public/lib/idx_test/idx_test_data_provider.cpp @@ -130,7 +130,7 @@ NYdb::TValue CreateValue(const TColumn& column, const TRandomValueProvider& rvp) return value.Build(); } -NYdb::TValue CreateRow(const TVector& columns, const TRandomValueProvider& rvp) { +NYdb::TValue CreateRow(const std::vector& columns, const TRandomValueProvider& rvp) { NYdb::TValueBuilder value; value.BeginStruct(); for (const NYdb::TColumn& col : columns) { diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.h b/ydb/public/lib/idx_test/idx_test_data_provider.h index bf0b420ba2cd..af262e457396 100644 --- a/ydb/public/lib/idx_test/idx_test_data_provider.h +++ b/ydb/public/lib/idx_test/idx_test_data_provider.h @@ -1,6 +1,6 @@ #pragma once -#include -#include +#include +#include #include #include @@ -94,7 +94,7 @@ class TRandomValueProvider { NYdb::TValue CreateOptionalValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); NYdb::TValue CreateValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); -NYdb::TValue CreateRow(const TVector& columns, const TRandomValueProvider& rvp); +NYdb::TValue CreateRow(const std::vector& columns, const TRandomValueProvider& rvp); NYdb::TParams CreateParamsAsItems(const TVector& values, const TVector& paramNames); NYdb::TParams CreateParamsAsList(const TVector& batch, const TString& paramName); void AddParamsAsList(NYdb::TParamsBuilder& paramsBuilder, const TVector& batch, const TString& paramName); diff --git a/ydb/public/lib/idx_test/idx_test_loader.cpp b/ydb/public/lib/idx_test/idx_test_loader.cpp index 55bf290131b0..b0d9771311c4 100644 --- a/ydb/public/lib/idx_test/idx_test_loader.cpp +++ b/ydb/public/lib/idx_test/idx_test_loader.cpp @@ -3,7 +3,7 @@ #include "idx_test_data_provider.h" #include -#include +#include #include #include @@ -115,7 +115,7 @@ static bool IsIndexedType(const TType& type) { } template -static bool HasColumn(const T& container, const TString& col) { +static bool HasColumn(const T& container, const std::string& col) { return Find(container.begin(), container.end(), col) != container.end(); } @@ -139,7 +139,7 @@ class TAlterTableAddIndexTask TAsyncStatus Run() override { if (!Description_) { // Nothing to add, skip - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } ChooseColumnsForIndex(); @@ -154,12 +154,12 @@ class TAlterTableAddIndexTask // status BAD_REQUEST - index already exists (realy?), treat it as success // TODO: improve this check if (status.GetStatus() == EStatus::BAD_REQUEST) { - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } return future; }); }); - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } IWorkLoader::ELoadCommand GetTaskId() const override { @@ -167,7 +167,7 @@ class TAlterTableAddIndexTask } private: void ChooseColumnsForIndex() { - TVector columns; + std::vector columns; bool hasIndexedType = false; @@ -187,7 +187,7 @@ class TAlterTableAddIndexTask return; } - TVector dataColumn; + std::vector dataColumn; if (WithDataColumn_) { for (const TTableColumn& col : TableDescription_.GetTableColumns()) { if (HasColumn(TableDescription_.GetPrimaryKeyColumns(), col.Name)) { @@ -236,7 +236,7 @@ class TSelectAndUpsertIfUniqTask TAsyncStatus Run() override { return Client_.RetryOperation([this](TSession session) mutable { - return GetCheckIndexUniq()(1, session, TMaybe(), THashMap()) + return GetCheckIndexUniq()(1, session, std::optional(), THashMap()) .Apply([this](NThreading::TFuture future) { auto result = future.ExtractValue(); const auto& status = result.Status; @@ -268,7 +268,7 @@ class TSelectAndUpsertIfUniqTask const auto params = ::NIdxTest::CreateParamsAsList(batch, ParamName_); return result.Tx->GetSession().ExecuteDataQuery( - programText, TTxControl::Tx(result.Tx.GetRef()), std::move(params), + programText, TTxControl::Tx(result.Tx.value()), std::move(params), TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) .Apply([](TAsyncDataQueryResult future){ auto result = future.ExtractValue(); @@ -294,25 +294,25 @@ class TSelectAndUpsertIfUniqTask struct IndexValues { TStatus Status; THashMap Values; - TMaybe Tx; + std::optional Tx; }; using TCheckIndexCb = std::function( size_t i, TSession session, - TMaybe, + std::optional, THashMap)>; TCheckIndexCb GetCheckIndexUniq() { return [this] ( size_t i, TSession session, - TMaybe tx, + std::optional tx, THashMap&& checked) mutable { if (i == Programs_.size()) { return NThreading::MakeFuture( { - TStatus(EStatus::SUCCESS, NYql::TIssues()), + TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()), checked, tx } @@ -326,14 +326,14 @@ class TSelectAndUpsertIfUniqTask for (const auto& col : p.second) { const auto val = ::NIdxTest::CreateValue(col.first, *this); - checked.insert({col.first.Name, val}); + checked.insert({TString{col.first.Name}, val}); values.push_back(val); paramNames.push_back(col.second); } const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); - TTxControl txctrl = tx ? TTxControl::Tx(tx.GetRef()) : TTxControl::BeginTx(TTxSettings::SerializableRW()); + TTxControl txctrl = tx ? TTxControl::Tx(tx.value()) : TTxControl::BeginTx(TTxSettings::SerializableRW()); return session.ExecuteDataQuery( p.first, @@ -348,7 +348,7 @@ class TSelectAndUpsertIfUniqTask { result, THashMap(), - TMaybe() + std::optional() } ); } @@ -359,7 +359,7 @@ class TSelectAndUpsertIfUniqTask } else { return NThreading::MakeFuture( { - TStatus(EStatus::PRECONDITION_FAILED, NYql::TIssues()), + TStatus(EStatus::PRECONDITION_FAILED, NYdb::NIssue::TIssues()), THashMap(), result.GetTransaction() } @@ -382,7 +382,7 @@ class TSelectAndUpsertIfUniqTask for (const auto& col : columns) { auto pkType = NIdxTest::CreateValue(col, *this); auto typeString = NYdb::FormatType(pkType.GetType()); - colHash.insert({col.Name, {col, typeString}}); + colHash.insert({TString{col.Name}, {col, TString{typeString}}}); upsertInput.push_back({col, ""}); } @@ -442,7 +442,7 @@ class TSelectAndUpsertIfUniqTask TString TableName_; TTableClient Client_; - TVector>>> Programs_; + TVector>>> Programs_; mutable TString Err_; }; @@ -505,7 +505,7 @@ class TSelectAndCompareTask } const auto& mainResultSet = NYdb::FormatResultSetYson(result.GetResultSet(0));; - return GetCheckIndexOp()(1, session, result, mainResultSet, ""); + return GetCheckIndexOp()(1, session, result, TString{mainResultSet}, ""); }); }).Apply([this](TAsyncStatus future) { TString err; @@ -568,7 +568,7 @@ class TSelectAndCompareTask vp.OpenOptional(); if (vp.IsNull()) { //Cerr << "Null value found, skip check.." << Endl; - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } parNames.push_back(col.second); } @@ -621,7 +621,7 @@ class TSelectAndCompareTask for (const auto& col : columns) { auto pkType = NIdxTest::CreateOptionalValue(col, *this); auto typeString = NYdb::FormatType(pkType.GetType()); - colHash.insert({col.Name, {col, typeString}}); + colHash.insert({TString{col.Name}, {col, TString{typeString}}}); allColumns += col.Name; if (++id != columns.size()) allColumns += ", "; @@ -631,7 +631,7 @@ class TSelectAndCompareTask TString pkPredicate; TString select1; TVector> pkPredicates; - for (const TString& str : pkColNames) { + for (const auto& str : pkColNames) { const TString paramName = Sprintf("$items_%zu", id); select1 += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), colHash.find(str)->second.second.c_str()); pkPredicates.push_back({colHash.find(str)->second.first, paramName}); @@ -662,7 +662,7 @@ class TSelectAndCompareTask id++; } // Add key column to request to handle non uniq index - for (const TString& str : pkColNames) { + for (const auto& str : pkColNames) { const TString paramName = Sprintf("$items_%zu", id); declare += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), colHash.find(str)->second.second.c_str()); @@ -686,7 +686,7 @@ class TSelectAndCompareTask TString TableName_; TTableClient Client_; - TVector>>> Programs_; + TVector>>> Programs_; mutable TString Err_; TMutex Mtx_; @@ -735,7 +735,7 @@ class TUpdateViaParamItemsTask TVector pkCol; THashSet pkColHash; for (const auto& pk : pkColNames) { - pkColHash.insert(pk); + pkColHash.insert(TString{pk}); } for (const auto& col : columns) { if (pkColHash.contains(col.Name)) { @@ -748,7 +748,7 @@ class TUpdateViaParamItemsTask for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { if (indexName.empty() || indexDesc.GetIndexName() == indexName) { for (const auto& col : indexDesc.GetIndexColumns()) { - indexColumns.insert(col); + indexColumns.insert(TString{col}); } } } @@ -857,7 +857,7 @@ class TUpdateViaParamItemsTask // Columns with given index (only if more then 1 index) if (TableDescription_.GetIndexDescriptions().size() > 1) { for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - Programs_.emplace_back(CreateProgram(indexDesc.GetIndexName())); + Programs_.emplace_back(CreateProgram(TString{indexDesc.GetIndexName()})); } } } @@ -923,14 +923,14 @@ class TUpdateViaParamListTask THashSet indexColumns; for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { for (const auto& col : indexDesc.GetIndexColumns()) { - indexColumns.insert(col); + indexColumns.insert(TString{col}); } } THashSet pks; for (const auto& pk : TableDescription_.GetPrimaryKeyColumns()) { - pks.insert(pk); - indexColumns.insert(pk); + pks.insert(TString{pk}); + indexColumns.insert(TString{pk}); } for (const auto& col : columns) { @@ -999,7 +999,7 @@ class TUpdateViaParamListTask // Columns with given index (only if more then 1 index) if (TableDescription_.GetIndexDescriptions().size() > 1) { for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - Programs_.emplace_back(CreateUpsertProgram(indexDesc.GetIndexName())); + Programs_.emplace_back(CreateUpsertProgram(TString{indexDesc.GetIndexName()})); } } } diff --git a/ydb/public/lib/idx_test/idx_test_upload.cpp b/ydb/public/lib/idx_test/idx_test_upload.cpp index 9a6266da6dda..e7a73285688d 100644 --- a/ydb/public/lib/idx_test/idx_test_upload.cpp +++ b/ydb/public/lib/idx_test/idx_test_upload.cpp @@ -35,7 +35,7 @@ class TUploader : public IUploader { }; void CreateTable(TTableDescription tableDesc) { - ThrowOnError(Client_.RetryOperationSync([this, tableDesc](TSession session) { + NStatusHelpers::ThrowOnError(Client_.RetryOperationSync([this, tableDesc](TSession session) { TTableDescription desc(tableDesc); return session.CreateTable( Table_, @@ -71,12 +71,12 @@ class TUploader : public IUploader { return [this, dataProvider, id](TSession session) -> TAsyncStatus { if (FinishedByError) - return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues())); auto maybeParamsList = CreateParams(dataProvider, id); if (!maybeParamsList) { UploadStatuses_[id] = EUploadStatus::Done; - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } auto paramsList = maybeParamsList.GetRef(); @@ -133,7 +133,7 @@ class TUploader : public IUploader { NThreading::WaitExceptionOrAll(resultFutures).Wait(); if (errStatus) - ThrowOnError(errStatus.GetRef()); + NStatusHelpers::ThrowOnError(errStatus.GetRef()); } TTableClient Client_; diff --git a/ydb/public/lib/idx_test/ya.make b/ydb/public/lib/idx_test/ya.make index 79aad5fce33d..616eea9ed5a3 100644 --- a/ydb/public/lib/idx_test/ya.make +++ b/ydb/public/lib/idx_test/ya.make @@ -11,7 +11,7 @@ SRCS( PEERDIR( library/cpp/string_utils/base64 - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) GENERATE_ENUM_SERIALIZATION(idx_test.h) diff --git a/ydb/public/lib/json_value/ut/ya.make b/ydb/public/lib/json_value/ut/ya.make index 1bbca3ebb03b..11f10a796245 100644 --- a/ydb/public/lib/json_value/ut/ya.make +++ b/ydb/public/lib/json_value/ut/ya.make @@ -11,8 +11,8 @@ SRCS( PEERDIR( library/cpp/json library/cpp/testing/unittest - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_params + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/params ) END() diff --git a/ydb/public/lib/json_value/ya.make b/ydb/public/lib/json_value/ya.make index 258e12f5be01..2b9439805c9f 100644 --- a/ydb/public/lib/json_value/ya.make +++ b/ydb/public/lib/json_value/ya.make @@ -8,8 +8,8 @@ SRCS( PEERDIR( library/cpp/json/writer library/cpp/string_utils/base64 - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/value yql/essentials/types/uuid ) diff --git a/ydb/public/lib/json_value/ydb_json_value.cpp b/ydb/public/lib/json_value/ydb_json_value.cpp index 5e8793aacb33..3ddaa0912304 100644 --- a/ydb/public/lib/json_value/ydb_json_value.cpp +++ b/ydb/public/lib/json_value/ydb_json_value.cpp @@ -489,7 +489,7 @@ namespace NYdb { } } - TString BinaryStringToJsonString(const TString& s) { + TString BinaryStringToJsonString(const std::string& s) { TStringStream str; str << "\""; switch (Encoding) { @@ -533,7 +533,7 @@ TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding) return out.Str(); } -void FormatResultRowJson(TResultSetParser& parser, const TVector& columns, NJsonWriter::TBuf& writer, +void FormatResultRowJson(TResultSetParser& parser, const std::vector& columns, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) { writer.BeginObject(); @@ -545,7 +545,7 @@ void FormatResultRowJson(TResultSetParser& parser, const TVector& colum writer.EndObject(); } -TString FormatResultRowJson(TResultSetParser& parser, const TVector& columns, +TString FormatResultRowJson(TResultSetParser& parser, const std::vector& columns, EBinaryStringEncoding encoding) { TStringStream out; @@ -789,7 +789,7 @@ namespace { break; case EPrimitiveType::Uuid: EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.Uuid(jsonValue.GetString()); + ValueBuilder.Uuid(TUuidValue{jsonValue.GetString()}); break; case EPrimitiveType::JsonDocument: EnsureType(jsonValue, NJson::JSON_STRING); @@ -952,7 +952,7 @@ namespace { const auto& jsonMap = jsonValue.GetMap(); while (TypeParser.TryNextMember()) { - const TString& memberName = TypeParser.GetMemberName(); + const auto& memberName = TypeParser.GetMemberName(); const auto it = jsonMap.find(memberName); if (it == jsonMap.end()) { ThrowFatalError(TStringBuilder() << "No member \"" << memberName diff --git a/ydb/public/lib/json_value/ydb_json_value.h b/ydb/public/lib/json_value/ydb_json_value.h index 24fa92d02e46..f4e614db481e 100644 --- a/ydb/public/lib/json_value/ydb_json_value.h +++ b/ydb/public/lib/json_value/ydb_json_value.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include @@ -25,10 +25,10 @@ void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStri TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding); -void FormatResultRowJson(TResultSetParser& parser, const TVector& columns, NJsonWriter::TBuf& writer, +void FormatResultRowJson(TResultSetParser& parser, const std::vector& columns, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); -TString FormatResultRowJson(TResultSetParser& parser, const TVector& columns, +TString FormatResultRowJson(TResultSetParser& parser, const std::vector& columns, EBinaryStringEncoding encoding); void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding); diff --git a/ydb/public/lib/json_value/ydb_json_value_ut.cpp b/ydb/public/lib/json_value/ydb_json_value_ut.cpp index 635da0ecea5f..5d6f7869be3c 100644 --- a/ydb/public/lib/json_value/ydb_json_value_ut.cpp +++ b/ydb/public/lib/json_value/ydb_json_value_ut.cpp @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include namespace NYdb { diff --git a/ydb/public/lib/jwt/jwt.h b/ydb/public/lib/jwt/jwt.h index 6d60e43cfec9..411e3edb7e17 100644 --- a/ydb/public/lib/jwt/jwt.h +++ b/ydb/public/lib/jwt/jwt.h @@ -1,21 +1 @@ -#pragma once - -#include -#include - -namespace NYdb { - -struct TJwtParams { - TStringType PrivKey; - TStringType PubKey; - TStringType AccountId; - TStringType KeyId; -}; - -TJwtParams ParseJwtParams(const TStringType& jsonParamsStr); -TStringType MakeSignedJwt( - const TJwtParams& params, - const TDuration& lifetime = TDuration::Hours(1) -); - -} // namespace NYdb +#include diff --git a/ydb/public/lib/jwt/ya.make b/ydb/public/lib/jwt/ya.make index c7bb97706411..0e7192e2a8b9 100644 --- a/ydb/public/lib/jwt/ya.make +++ b/ydb/public/lib/jwt/ya.make @@ -1,14 +1,11 @@ LIBRARY() SRCS( - jwt.cpp jwt.h ) PEERDIR( - contrib/libs/jwt-cpp - library/cpp/json - ydb/public/sdk/cpp/client/impl/ydb_internal/common + ydb/public/sdk/cpp/src/library/jwt ) END() diff --git a/ydb/public/lib/operation_id/operation_id.cpp b/ydb/public/lib/operation_id/operation_id.cpp deleted file mode 100644 index d417881ecf9f..000000000000 --- a/ydb/public/lib/operation_id/operation_id.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "operation_id.h" - -#include - -#include -#include - -#include -#include - -namespace NKikimr { -namespace NOperationId { - -using namespace NUri; - -static const TString QueryIdPrefix = "ydb://preparedqueryid/4?id="; - -TString FormatPreparedQueryIdCompat(const TString& in) { - return QueryIdPrefix + in; -} - -bool DecodePreparedQueryIdCompat(const TString& in, TString& out) { - if (in.size() <= QueryIdPrefix.size()) { - ythrow yexception() << "Unable to parse input string"; - } - if (in.compare(0, QueryIdPrefix.size(), QueryIdPrefix) == 0) { - out = in.substr(QueryIdPrefix.size()); - return true; - } - return false; -} - -TString ProtoToString(const Ydb::TOperationId& proto) { - using namespace ::google::protobuf; - const Reflection& reflection = *proto.GetReflection(); - std::vector fields; - reflection.ListFields(proto, &fields); - TStringStream res; - switch (proto.GetKind()) { - case Ydb::TOperationId::OPERATION_DDL: - case Ydb::TOperationId::OPERATION_DML: - res << "ydb://operation"; - break; - case Ydb::TOperationId::SESSION_YQL: - res << "ydb://session"; - break; - case Ydb::TOperationId::PREPARED_QUERY_ID: - res << "ydb://preparedqueryid"; - break; - case Ydb::TOperationId::CMS_REQUEST: - res << "ydb://cmsrequest"; - break; - case Ydb::TOperationId::EXPORT: - res << "ydb://export"; - break; - case Ydb::TOperationId::IMPORT: - res << "ydb://import"; - break; - case Ydb::TOperationId::BUILD_INDEX: - res << "ydb://buildindex"; - break; - case Ydb::TOperationId::SCRIPT_EXECUTION: - res << "ydb://scriptexec"; - break; - default: - Y_ABORT_UNLESS(false, "unexpected kind"); - } - // According to protobuf documentation: - // Fields (both normal fields and extension fields) will be listed ordered by field number, - // so we can rely on it to build url string - for (const FieldDescriptor* field : fields) { - Y_ASSERT(field != nullptr); - if (field) { - if (field->is_repeated()) { - int size = reflection.FieldSize(proto, field); - if (size) { - res << "?"; - } - for (int i = 0; i < size; i++) { - const auto& message = reflection.GetRepeatedMessage(proto, field, i); - const auto& data = dynamic_cast(message); - TUri::ReEncode(res, data.GetKey()); - res << "="; - TUri::ReEncode(res, data.GetValue()); - if (i < size - 1) { - res << "&"; - } - } - } else { - res << "/"; - const FieldDescriptor::CppType type = field->cpp_type(); - switch (type) { - case FieldDescriptor::CPPTYPE_ENUM: - res << reflection.GetEnumValue(proto, field); - break; - default: - Y_ABORT_UNLESS(false, "unexpected protobuf field type"); - break; - } - } - } - } - return res.Str(); -} - -TOperationId::TOperationId() { - SetKind(Ydb::TOperationId::UNUSED); -} - -TOperationId::TOperationId(const TString &string, bool allowEmpty) { - if (allowEmpty && string.empty()) { - SetKind(Ydb::TOperationId::UNUSED); - return; - } - - TUri uri; - TState::EParsed er = uri.Parse(string, TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible); - if (er != TState::ParsedOK) { - ythrow yexception() << "Unable to parse input string"; - } - - const TString& path = uri.PrintS(TField::FlagPath).substr(1); // start from 1 to remove first '/' - if (path.length() < 1) { - ythrow yexception() << "Invalid path length"; - } - - int kind; - if (!TryFromString(path, kind)) { - ythrow yexception() << "Unable to cast \"kind\" field: " << path; - } - - if (!EKind_IsValid(kind)) { - ythrow yexception() << "Invalid operation kind: " << kind; - } - - SetKind(static_cast(kind)); - - const TString& query = uri.PrintS(TField::FlagQuery); - - if (query) { - TCgiParameters params(query.substr(1)); // start from 1 to remove first '?' - for (auto it : params) { - auto data = AddData(); - data->SetKey(it.first); - data->SetValue(it.second); - Index_[it.first].push_back(&data->GetValue()); - } - } -} - -const TVector& TOperationId::GetValue(const TString &key) const { - auto it = Index_.find(key); - if (it != Index_.end()) { - return it->second; - } - ythrow yexception() << "Unable to find key: " << key; -} - -TString TOperationId::GetSubKind() const { - auto it = Index_.find("kind"); - if (it == Index_.end()) { - return TString(); - } - - if (it->second.size() != 1) { - ythrow yexception() << "Unable to retreive sub-kind"; - } - - return *it->second.at(0); -} - -void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value) { - auto data = proto.AddData(); - data->SetKey(key); - data->SetValue(value); -} - -Ydb::TOperationId::EKind ParseKind(const TStringBuf value) { - if (value.StartsWith("ss/backgrounds")) { - return Ydb::TOperationId::SS_BG_TASKS; - } - - if (value.StartsWith("export")) { - return Ydb::TOperationId::EXPORT; - } - - if (value.StartsWith("import")) { - return Ydb::TOperationId::IMPORT; - } - - if (value.StartsWith("buildindex")) { - return Ydb::TOperationId::BUILD_INDEX; - } - - if (value.StartsWith("scriptexec")) { - return Ydb::TOperationId::SCRIPT_EXECUTION; - } - - return Ydb::TOperationId::UNUSED; -} - -} // namespace NOperationId -} // namespace NKikimr diff --git a/ydb/public/lib/operation_id/operation_id.h b/ydb/public/lib/operation_id/operation_id.h index 2a659bed89d3..a04367972e1f 100644 --- a/ydb/public/lib/operation_id/operation_id.h +++ b/ydb/public/lib/operation_id/operation_id.h @@ -1,32 +1 @@ -#pragma once - -#include - -#include -#include -#include - -namespace NKikimr { -namespace NOperationId { - -class TOperationId : public Ydb::TOperationId { -public: - TOperationId(); - explicit TOperationId(const TString& string, bool allowEmpty = false); - const TVector& GetValue(const TString& key) const; - TString GetSubKind() const; - -private: - THashMap> Index_; -}; - -TString ProtoToString(const Ydb::TOperationId& proto); -void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value); -void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const char* value, size_t size); -Ydb::TOperationId::EKind ParseKind(const TStringBuf value); - -TString FormatPreparedQueryIdCompat(const TString& str); -bool DecodePreparedQueryIdCompat(const TString& in, TString& out); - -} // namespace NOperationId -} // namespace NKikimr +#include diff --git a/ydb/public/lib/operation_id/ya.make b/ydb/public/lib/operation_id/ya.make index 40d77ec872cc..89829ee98349 100644 --- a/ydb/public/lib/operation_id/ya.make +++ b/ydb/public/lib/operation_id/ya.make @@ -1,18 +1,11 @@ LIBRARY() SRCS( - operation_id.cpp + operation_id.h ) PEERDIR( - contrib/libs/protobuf - library/cpp/cgiparam - library/cpp/uri - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/lib/ut_helpers/ut_helpers_query.cpp b/ydb/public/lib/ut_helpers/ut_helpers_query.cpp index 1d2193fbac25..afc45a79d0f9 100644 --- a/ydb/public/lib/ut_helpers/ut_helpers_query.cpp +++ b/ydb/public/lib/ut_helpers/ut_helpers_query.cpp @@ -1,8 +1,8 @@ #include "ut_helpers_query.h" #include -#include -#include +#include +#include #include #include diff --git a/ydb/public/lib/ut_helpers/ut_helpers_query.h b/ydb/public/lib/ut_helpers/ut_helpers_query.h index 18d2c958f9a5..27b7177aa9b2 100644 --- a/ydb/public/lib/ut_helpers/ut_helpers_query.h +++ b/ydb/public/lib/ut_helpers/ut_helpers_query.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace NTestHelpers { diff --git a/ydb/public/lib/ut_helpers/ya.make b/ydb/public/lib/ut_helpers/ya.make index 133c009c7d83..99d655226d0c 100644 --- a/ydb/public/lib/ut_helpers/ya.make +++ b/ydb/public/lib/ut_helpers/ya.make @@ -9,7 +9,7 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/api/protos ydb/public/api/protos/out - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ) END() diff --git a/ydb/public/lib/value/value.cpp b/ydb/public/lib/value/value.cpp index 1d145085a93c..96339970ab90 100644 --- a/ydb/public/lib/value/value.cpp +++ b/ydb/public/lib/value/value.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace NKikimr { namespace NClient { diff --git a/ydb/public/lib/value/ya.make b/ydb/public/lib/value/ya.make index c87b3d471067..11d101454244 100644 --- a/ydb/public/lib/value/ya.make +++ b/ydb/public/lib/value/ya.make @@ -10,7 +10,7 @@ PEERDIR( ydb/core/protos ydb/library/mkql_proto/protos ydb/public/lib/scheme_types - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/value ) END() diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp index a75c331610d2..a70d879c5697 100644 --- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp +++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include @@ -187,16 +187,16 @@ class TQueryResultScanner { if constexpr (std::is_same_v) { if (streamPart.HasQueryStats()) { ServerTiming += streamPart.GetQueryStats().GetTotalDuration(); - QueryPlan = streamPart.GetQueryStats().GetPlan().GetOrElse(""); - PlanAst = streamPart.GetQueryStats().GetAst().GetOrElse(""); + QueryPlan = streamPart.GetQueryStats().GetPlan().value_or(""); + PlanAst = streamPart.GetQueryStats().GetAst().value_or(""); } } else { const auto& stats = streamPart.GetStats(); rsIndex = streamPart.GetResultSetIndex(); if (stats) { ServerTiming += stats->GetTotalDuration(); - QueryPlan = stats->GetPlan().GetOrElse(""); - PlanAst = stats->GetAst().GetOrElse(""); + QueryPlan = stats->GetPlan().value_or(""); + PlanAst = stats->GetAst().value_or(""); } } diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h index 34d3fd72ba20..48464dc1eda1 100644 --- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h +++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include #include @@ -79,7 +79,6 @@ struct TQueryBenchmarkDeadline { }; TString FullTablePath(const TString& database, const TString& table); -void ThrowOnError(const TStatus& status); bool HasCharsInString(const TString& str); TQueryBenchmarkResult Execute(const TString & query, NTable::TTableClient & client, const TQueryBenchmarkDeadline& deadline); TQueryBenchmarkResult Execute(const TString & query, NQuery::TQueryClient & client, const TQueryBenchmarkDeadline& deadline); diff --git a/ydb/public/lib/ydb_cli/commands/command_base/ya.make b/ydb/public/lib/ydb_cli/commands/command_base/ya.make index 11ff452e9f98..c53336b7b885 100644 --- a/ydb/public/lib/ydb_cli/commands/command_base/ya.make +++ b/ydb/public/lib/ydb_cli/commands/command_base/ya.make @@ -6,8 +6,8 @@ SRCS( PEERDIR( ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/driver ) END() diff --git a/ydb/public/lib/ydb_cli/commands/interactive/interactive_cli.cpp b/ydb/public/lib/ydb_cli/commands/interactive/interactive_cli.cpp index bad39179748d..560020e135a5 100644 --- a/ydb/public/lib/ydb_cli/commands/interactive/interactive_cli.cpp +++ b/ydb/public/lib/ydb_cli/commands/interactive/interactive_cli.cpp @@ -192,7 +192,7 @@ void TInteractiveCLI::Run() { sqlCommand.SetCollectStatsMode(std::move(queryStatsMode)); sqlCommand.SetSyntax("yql"); sqlCommand.Run(Config); - } catch (TYdbErrorException &error) { + } catch (NStatusHelpers::TYdbErrorException &error) { Cerr << error; } catch (yexception & error) { Cerr << error; diff --git a/ydb/public/lib/ydb_cli/commands/query_workload.cpp b/ydb/public/lib/ydb_cli/commands/query_workload.cpp index f7dfc7c95c4f..164fe370a567 100644 --- a/ydb/public/lib/ydb_cli/commands/query_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/query_workload.cpp @@ -1,7 +1,7 @@ #include "query_workload.h" #include -#include +#include #include #include #include @@ -116,7 +116,7 @@ int TCommandQueryWorkloadRun::Run(TConfig& config) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); TDuration local_duration; @@ -128,7 +128,7 @@ int TCommandQueryWorkloadRun::Run(TConfig& config) { } if (streamPart.GetStats()) { - const auto& queryStats = streamPart.GetStats().GetRef(); + const auto& queryStats = streamPart.GetStats().value(); local_duration += queryStats.GetTotalDuration(); } } diff --git a/ydb/public/lib/ydb_cli/commands/sdk_core_access/ya.make b/ydb/public/lib/ydb_cli/commands/sdk_core_access/ya.make index bf0c37a06d74..84000fdf3b37 100644 --- a/ydb/public/lib/ydb_cli/commands/sdk_core_access/ya.make +++ b/ydb/public/lib/ydb_cli/commands/sdk_core_access/ya.make @@ -4,9 +4,13 @@ SRCS( ../ydb_sdk_core_access.cpp ) +ADDINCL( + ydb/public/sdk/cpp +) + PEERDIR( - ydb/public/sdk/cpp/client/ydb_common_client/impl - ydb/public/sdk/cpp/client/ydb_types + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/types ) END() diff --git a/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.cpp b/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.cpp index f9aff69e0eef..6c54ddc057d9 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.cpp @@ -7,7 +7,7 @@ #include #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include @@ -70,7 +70,7 @@ void TTopicOperationsScenario::InitDriver(const TConfig& config) { Driver = std::make_unique(TYdbCommand::CreateDriver(config, - MakeLogBackend(config.VerbosityLevel))); + std::unique_ptr(MakeLogBackend(config.VerbosityLevel).Release()))); } void TTopicOperationsScenario::InitStatsCollector() @@ -115,7 +115,7 @@ void TTopicOperationsScenario::DropTopic(const TString& database, TCommandWorkloadTopicDescribe::GenerateFullTopicName(database, topic); auto result = client.DropTopic(topicPath).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } void TTopicOperationsScenario::DropTable(const TString& database, const TString& table) @@ -123,7 +123,7 @@ void TTopicOperationsScenario::DropTable(const TString& database, const TString& NTable::TTableClient client(*Driver); auto session = GetSession(client); auto result = session.DropTable(database + "/" + table).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } void TTopicOperationsScenario::ExecSchemeQuery(const TString& query) @@ -131,7 +131,7 @@ void TTopicOperationsScenario::ExecSchemeQuery(const TString& query) NTable::TTableClient client(*Driver); auto session = GetSession(client); auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } void TTopicOperationsScenario::ExecDataQuery(const TString& query, @@ -142,7 +142,7 @@ void TTopicOperationsScenario::ExecDataQuery(const TString& query, auto result = session.ExecuteDataQuery(query, NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } void TTopicOperationsScenario::EnsureTopicNotExist(const TString& topic) @@ -194,13 +194,13 @@ void TTopicOperationsScenario::CreateTopic(const TString& topic, } auto result = client.CreateTopic(topic, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } NTable::TSession TTopicOperationsScenario::GetSession(NTable::TTableClient& client) { auto result = client.GetSession({}).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return result.GetSession(); } diff --git a/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.h b/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.h index eda04fd90d1d..0913cc492dc2 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.h +++ b/ydb/public/lib/ydb_cli/commands/topic_operations_scenario.h @@ -15,14 +15,14 @@ class TLogBackend; class TLog; -namespace NYdb { +namespace NYdb::inline V3 { class TDriver; class TParams; } -namespace NYdb::NTable { +namespace NYdb::inline V3::NTable { class TSession; class TTableClient; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_clean.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_clean.cpp index bb651554eec1..b90a051e4d96 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_clean.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_clean.cpp @@ -3,7 +3,7 @@ #include "topic_workload_describe.h" #include "topic_workload_defines.h" -#include +#include #include diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_describe.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_describe.h index e7eb0c1d4508..98d05e39675f 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_describe.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_describe.h @@ -3,7 +3,7 @@ #include #include -#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_init.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_init.cpp index 8ca847cdef42..a5fe9af0af93 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_init.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_init.cpp @@ -4,7 +4,7 @@ #include "topic_workload_describe.h" #include -#include +#include using namespace NYdb::NConsoleClient; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_params.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_params.cpp index f4063479c85b..89e65c99be85 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_params.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_params.cpp @@ -5,7 +5,9 @@ #include #include -#include +#include + +#include using namespace NYdb::NConsoleClient; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.cpp index c9b9109479a6..0954e0613f94 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.cpp @@ -3,7 +3,7 @@ #include "topic_workload_describe.h" -#include +#include #include using namespace NYdb::NConsoleClient; @@ -38,9 +38,9 @@ void TTopicWorkloadReader::ReaderLoop(TTopicWorkloadReaderParams& params, TInsta WRITE_LOG(params.Log, ELogPriority::TLOG_EMERG, TStringBuilder() << "Topic '" << params.TopicName << "' doesn't have a consumer '" << consumerName << "'. Run command 'workload init' with parameter '--consumers'."); exit(EXIT_FAILURE); } - settings.ConsumerName(consumerName).AppendTopics(params.TopicName); + settings.ConsumerName(consumerName).AppendTopics(std::string{params.TopicName}); } else { - NYdb::NTopic::TTopicReadSettings topic = params.TopicName; + NYdb::NTopic::TTopicReadSettings topic = std::string{params.TopicName}; auto partitions = describeTopicResult.GetPartitions(); for(auto partition: partitions) { topic.AppendPartitionIds(partition.GetPartitionId()); @@ -81,7 +81,7 @@ void TTopicWorkloadReader::ReaderLoop(TTopicWorkloadReaderParams& params, TInsta } readSession->WaitEvent().Wait(TDuration::Seconds(1)); - TVector events = GetEvents(*readSession, params, txSupport); + std::vector events = GetEvents(*readSession, params, txSupport); // we could wait for the event for almost one second, so we need to update the value of the variable now = TInstant::Now(); @@ -95,7 +95,7 @@ void TTopicWorkloadReader::ReaderLoop(TTopicWorkloadReaderParams& params, TInsta params.StatsCollector->AddReaderEvent(params.ReaderIdx, {message.GetData().size(), fullTime}); if (txSupport) { - txSupport->AppendRow(message.GetData()); + txSupport->AppendRow(TString{message.GetData()}); } WRITE_LOG(params.Log, ELogPriority::TLOG_DEBUG, TStringBuilder() << "Got message: " << message.GetMessageGroupId() @@ -147,9 +147,9 @@ void TTopicWorkloadReader::ReaderLoop(TTopicWorkloadReaderParams& params, TInsta } } -TVector TTopicWorkloadReader::GetEvents(NYdb::NTopic::IReadSession& readSession, - TTopicWorkloadReaderParams& params, - std::optional& txSupport) +std::vector TTopicWorkloadReader::GetEvents(NYdb::NTopic::IReadSession& readSession, + TTopicWorkloadReaderParams& params, + std::optional& txSupport) { TVector events; NTopic::TReadSessionGetEventSettings settings; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.h index bf3307581e3b..f783c1e885cb 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader.h @@ -3,7 +3,7 @@ #include "topic_workload_defines.h" #include "topic_workload_stats_collector.h" -#include +#include #include #include @@ -44,9 +44,9 @@ namespace NYdb { private: static void ReaderLoop(TTopicWorkloadReaderParams& params, TInstant endTime); - static TVector GetEvents(NYdb::NTopic::IReadSession& readSession, - TTopicWorkloadReaderParams& params, - std::optional& txSupport); + static std::vector GetEvents(NYdb::NTopic::IReadSession& readSession, + TTopicWorkloadReaderParams& params, + std::optional& txSupport); static void TryCommitTx(TTopicWorkloadReaderParams& params, std::optional& txSupport, diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader_transaction_support.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader_transaction_support.h index ab549b6f713b..a612a8c03eb8 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader_transaction_support.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_reader_transaction_support.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NYdb::NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer.h index c2f77d0a2464..f370df0c7a80 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer.h @@ -4,7 +4,7 @@ #include "topic_workload_stats_collector.h" #include "topic_workload_reader_transaction_support.h" -#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.cpp index 2c0899bbf8cb..391cfb2861de 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.cpp @@ -63,7 +63,7 @@ TString TTopicWorkloadWriterProducer::GetGeneratedMessage() const { bool TTopicWorkloadWriterProducer::WaitForInitSeqNo() { Y_ASSERT(WriteSession_); - NThreading::TFuture InitSeqNo = WriteSession_->GetInitSeqNo(); + NThreading::TFuture InitSeqNo = WriteSession_->GetInitSeqNo(); while (!*Params_.ErrorFlag) { if (!InitSeqNo.HasValue() && !InitSeqNo.Wait(TDuration::Seconds(1))) { WRITE_LOG(Params_.Log, ELogPriority::TLOG_WARNING, @@ -116,7 +116,7 @@ void TTopicWorkloadWriterProducer::WaitForContinuationToken(const TDuration& tim << ": foundEvent - " << foundEvent); if (foundEvent) { - auto variant = WriteSession_->GetEvent(true).GetRef(); + auto variant = WriteSession_->GetEvent(true).value(); if (std::holds_alternative(variant)) { auto event = std::get(variant); ContinuationToken_ = std::move(event.ContinuationToken); diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.h index 693a222e3c52..17e1360e88ce 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_writer_producer.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "topic_workload_writer.h" #include diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/ut/topic_workload_writer_producer_ut.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/ut/topic_workload_writer_producer_ut.cpp index c10b897753c9..d8019dc3a997 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/ut/topic_workload_writer_producer_ut.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/ut/topic_workload_writer_producer_ut.cpp @@ -17,19 +17,19 @@ namespace NTests { public: MOCK_METHOD(NThreading::TFuture, WaitEvent, (), (override)); - MOCK_METHOD(TMaybe, GetEvent, (bool block), (override)); + MOCK_METHOD(std::optional, GetEvent, (bool block), (override)); - MOCK_METHOD(TVector, GetEvents, (bool block, TMaybe maxEventsCount), (override)); + MOCK_METHOD(std::vector, GetEvents, (bool block, std::optional maxEventsCount), (override)); - MOCK_METHOD(NThreading::TFuture, GetInitSeqNo, (), (override)); + MOCK_METHOD(NThreading::TFuture, GetInitSeqNo, (), (override)); MOCK_METHOD(void, Write, (TContinuationToken&& continuationToken, TWriteMessage&& message, NTable::TTransaction* tx), (override)); - MOCK_METHOD(void, Write, (TContinuationToken&& continuationToken, TStringBuf data, TMaybe seqNo, TMaybe createTimestamp), (override)); + MOCK_METHOD(void, Write, (TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo, std::optional createTimestamp), (override)); MOCK_METHOD(void, WriteEncoded, (TContinuationToken&& continuationToken, TWriteMessage&& params, NTable::TTransaction* tx), (override)); - MOCK_METHOD(void, WriteEncoded, (TContinuationToken&& continuationToken, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe seqNo, TMaybe createTimestamp), (override)); + MOCK_METHOD(void, WriteEncoded, (TContinuationToken&& continuationToken, std::string_view data, ECodec codec, uint32_t originalSize, std::optional seqNo, std::optional createTimestamp), (override)); MOCK_METHOD(bool, Close, (TDuration closeTimeout), (override)); @@ -152,7 +152,7 @@ namespace NTests { void TFixture::InitContinuationToken(TTopicWorkloadWriterProducer& producer) { auto continuationToken = MockContinuationTokenIssuer::IssueContinuationToken(); - TMaybe event = std::variant< + std::optional event = std::variant< TWriteSessionEvent::TAcksEvent, TWriteSessionEvent::TReadyToAcceptEvent, TSessionClosedEvent @@ -188,7 +188,7 @@ namespace NTests { Y_UNIT_TEST_F(Send_ShouldCallWriteMethodOfTheWriteSession, TFixture) { // arrange auto producer = CreateProducer(); - auto initSeqNo = 0ul; + uint64_t initSeqNo = 0; ON_CALL(*WriteSession, GetInitSeqNo()).WillByDefault(testing::Return(NThreading::MakeFuture(initSeqNo))); producer.WaitForInitSeqNo(); InitContinuationToken(producer); @@ -221,7 +221,7 @@ namespace NTests { Y_UNIT_TEST_F(WaitForContinuationToken_ShouldThrowExceptionIfEventOfTheWrongType, TFixture) { auto producer = CreateProducer(); - TMaybe event = std::variant< + std::optional event = std::variant< TWriteSessionEvent::TAcksEvent, TWriteSessionEvent::TReadyToAcceptEvent, TSessionClosedEvent diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/ya.make b/ydb/public/lib/ydb_cli/commands/topic_workload/ya.make index 261088607ad5..760cbefcafff 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/ya.make +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/ya.make @@ -23,15 +23,14 @@ PEERDIR( ydb/public/api/grpc ydb/public/api/protos ydb/public/api/protos/annotations - ydb/public/lib/operation_id - ydb/public/lib/operation_id/protos - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types/operation - ydb/public/sdk/cpp/client/ydb_types/status + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types/operation + ydb/public/sdk/cpp/src/client/types/status library/cpp/unified_agent_client library/cpp/histogram/hdr ) diff --git a/ydb/public/lib/ydb_cli/commands/transfer_workload/transfer_workload_topic_to_table_run.cpp b/ydb/public/lib/ydb_cli/commands/transfer_workload/transfer_workload_topic_to_table_run.cpp index cfa92429fc63..fd739946aec7 100644 --- a/ydb/public/lib/ydb_cli/commands/transfer_workload/transfer_workload_topic_to_table_run.cpp +++ b/ydb/public/lib/ydb_cli/commands/transfer_workload/transfer_workload_topic_to_table_run.cpp @@ -11,7 +11,7 @@ #include #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include diff --git a/ydb/public/lib/ydb_cli/commands/transfer_workload/ya.make b/ydb/public/lib/ydb_cli/commands/transfer_workload/ya.make index 59a70073d140..3467460bcd24 100644 --- a/ydb/public/lib/ydb_cli/commands/transfer_workload/ya.make +++ b/ydb/public/lib/ydb_cli/commands/transfer_workload/ya.make @@ -1,5 +1,9 @@ LIBRARY(transfer_workload) +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( transfer_workload.cpp transfer_workload_topic_to_table.cpp @@ -15,15 +19,15 @@ PEERDIR( ydb/public/api/grpc ydb/public/api/protos ydb/public/api/protos/annotations - ydb/public/lib/operation_id - ydb/public/lib/operation_id/protos - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types/operation - ydb/public/sdk/cpp/client/ydb_types/status + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types/operation + ydb/public/sdk/cpp/src/client/types/status ) END() diff --git a/ydb/public/lib/ydb_cli/commands/ya.make b/ydb/public/lib/ydb_cli/commands/ya.make index cccbd4fc5e70..618524ec1e17 100644 --- a/ydb/public/lib/ydb_cli/commands/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ya.make @@ -1,5 +1,9 @@ LIBRARY(clicommands) +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( benchmark_utils.cpp topic_operations_scenario.cpp @@ -43,7 +47,8 @@ PEERDIR( ydb/library/formats/arrow/csv/table ydb/library/workload ydb/library/yaml_config/public - ydb/public/lib/operation_id + yql/essentials/public/decimal + ydb/public/sdk/cpp/src/library/operation_id ydb/public/lib/stat_visualization ydb/public/lib/ydb_cli/common ydb/public/lib/ydb_cli/commands/command_base @@ -56,20 +61,20 @@ PEERDIR( ydb/public/lib/ydb_cli/dump/files ydb/public/lib/ydb_cli/import ydb/public/lib/ydb_cli/topic - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_bsconfig - ydb/public/sdk/cpp/client/ydb_coordination - ydb/public/sdk/cpp/client/ydb_debug - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_import - ydb/public/sdk/cpp/client/ydb_monitoring - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types/credentials/login + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/bsconfig + ydb/public/sdk/cpp/src/client/coordination + ydb/public/sdk/cpp/src/client/debug + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/import + ydb/public/sdk/cpp/src/client/monitoring + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types/credentials/login ) GENERATE_ENUM_SERIALIZATION(ydb_ping.h) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index dfe1a7b302dc..78db85a9009d 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -1,6 +1,6 @@ #include "ydb_cluster.h" -#include +#include using namespace NKikimr; @@ -31,7 +31,7 @@ int TCommandClusterBootstrap::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); NYdb::NStorageConfig::TStorageConfigClient client(*driver); auto result = client.BootstrapCluster(SelfAssemblyUUID).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_command.cpp b/ydb/public/lib/ydb_cli/commands/ydb_command.cpp index 2f30407ff29d..d668c3416007 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_command.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_command.cpp @@ -26,7 +26,7 @@ TDriver TYdbCommand::CreateDriver(const TConfig& config) { return TDriver(CreateDriverConfig(config)); } -TDriver TYdbCommand::CreateDriver(const TConfig& config, THolder&& loggingBackend) { +TDriver TYdbCommand::CreateDriver(const TConfig& config, std::unique_ptr&& loggingBackend) { auto driverConfig = CreateDriverConfig(config); driverConfig.SetLog(std::move(loggingBackend)); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_command.h b/ydb/public/lib/ydb_cli/commands/ydb_command.h index 23e78fcfdcd3..1a8902cd8872 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_command.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_command.h @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include @@ -19,7 +19,7 @@ class TYdbCommand : public TClientCommand { ); static TDriver CreateDriver(const TConfig& config); - static TDriver CreateDriver(const TConfig& config, THolder&& loggingBackend); + static TDriver CreateDriver(const TConfig& config, std::unique_ptr&& loggingBackend); private: static TDriverConfig CreateDriverConfig(const TConfig& config); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_common.h b/ydb/public/lib/ydb_cli/commands/ydb_common.h index d7f7973902ac..fb99bb128eb5 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_common.h @@ -1,47 +1,25 @@ #pragma once -#include -#include +#include +#include namespace NYdb { namespace NConsoleClient { -class TYdbErrorException : public yexception { -public: - TYdbErrorException(NYdb::TStatus status) - : Status(std::move(status)) - { } - - friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { - return out << e.Status; - } - -private: - NYdb::TStatus Status; -}; - -inline void ThrowOnError(NYdb::TStatus status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } else if (status.GetIssues()) { - Cerr << status; - } -} - -inline void ThrowOnError(const NYdb::TOperation& operation) { +inline void ThrowOnError(const NYdb::V3::TOperation& operation) { if (!operation.Ready()) return; - ThrowOnError(operation.Status()); + NStatusHelpers::ThrowOnError(operation.Status()); } -inline bool ThrowOnErrorAndCheckEOS(NYdb::TStreamPartStatus status) { +inline bool ThrowOnErrorAndCheckEOS(NYdb::V3::TStreamPartStatus status) { if (!status.IsSuccess()) { if (status.EOS()) { return true; } - throw TYdbErrorException(status) << static_cast(status); + throw NStatusHelpers::TYdbErrorException(status) << static_cast(status); } else if (status.GetIssues()) { - Cerr << static_cast(status); + Cerr << static_cast(status); } return false; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_discovery/ya.make b/ydb/public/lib/ydb_cli/commands/ydb_discovery/ya.make index 6d2a0f3d60e3..7f591363adfd 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_discovery/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ydb_discovery/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( ydb/public/lib/ydb_cli/commands/command_base ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/ydb_discovery + ydb/public/sdk/cpp/src/client/discovery ) END() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 386a781fa737..d6e2995b8e6b 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -1,6 +1,6 @@ #include "ydb_dynamic_config.h" -#include +#include #include #include @@ -57,9 +57,9 @@ int TCommandConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); auto result = client.GetConfig().GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); - auto cfg = result.GetConfig(); + auto cfg = TString{result.GetConfig()}; ui64 version = 0; @@ -88,11 +88,11 @@ int TCommandConfigFetch::Run(TConfig& config) { if (All) { for (auto [id, cfg] : result.GetVolatileConfigs()) { if (StripMetadata) { - cfg = NYamlConfig::StripMetadata(cfg); + cfg = NYamlConfig::StripMetadata(TString{cfg}); } if (!OutDir) { - Cout << WrapYaml(cfg); + Cout << WrapYaml(TString{cfg}); } else { auto filename = TString("volatile_") + ToString(version) + "_" + ToString(id) + ".yaml"; auto filepath = (TFsPath(OutDir) / filename); @@ -156,7 +156,7 @@ int TCommandConfigReplace::Run(TConfig& config) { return client.ReplaceConfig(DynamicConfig, DryRun, AllowUnknownFields).GetValueSync(); }; auto status = exec(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); if (!status.GetIssues()) { Cout << status << Endl; @@ -244,14 +244,14 @@ int TCommandConfigResolve::Run(TConfig& config) { if (NodeId) { auto result = client.GetNodeLabels(NodeId).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); // TODO: maybe we should merge labels instead Labels = result.GetLabels(); } TString configStr; - TMap volatileConfigStrs; + std::map volatileConfigStrs; if (!Filename.empty()) { configStr = TFileInput(Filename).ReadAll(); @@ -276,7 +276,7 @@ int TCommandConfigResolve::Run(TConfig& config) { if (FromCluster) { auto result = client.GetConfig().GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); configStr = result.GetConfig(); volatileConfigStrs = result.GetVolatileConfigs(); @@ -286,7 +286,7 @@ int TCommandConfigResolve::Run(TConfig& config) { if (!SkipVolatile) { for (auto& [_, cfgStr]: volatileConfigStrs) { - auto volatileCfg = NFyaml::TDocument::Parse(cfgStr); + auto volatileCfg = NFyaml::TDocument::Parse(TString{cfgStr}); auto selectors = volatileCfg.Root().Map().at("selector_config"); NYamlConfig::AppendVolatileConfigs(tree, selectors); } @@ -336,7 +336,7 @@ int TCommandConfigResolve::Run(TConfig& config) { auto map = doc.Buildf("{}"); TSet namedLabels; for (auto& [name, value] : Labels) { - namedLabels.insert(NYamlConfig::TNamedLabel{name, value}); + namedLabels.insert(NYamlConfig::TNamedLabel{TString{name}, TString{value}}); auto node = doc.Buildf("%s: {type: COMMON, value: %s}", name.c_str(), value.c_str()); map.Insert(node); } @@ -351,13 +351,13 @@ int TCommandConfigResolve::Run(TConfig& config) { } else { if (All) { auto result = client.VerboseResolveConfig(configStr, volatileConfigStrs).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); TVector labels(result.GetLabels().begin(), result.GetLabels().end()); for (const auto& [labelSets, configStr] : result.GetConfigs()) { auto doc = NFyaml::TDocument::Parse("---\nlabel_sets: []\nconfig: {}\n"); - auto config = NFyaml::TDocument::Parse(configStr); + auto config = NFyaml::TDocument::Parse(TString{configStr}); auto node = config.Root().Copy(doc); doc.Root().Map().at("config").Insert(node.Ref()); @@ -390,7 +390,7 @@ int TCommandConfigResolve::Run(TConfig& config) { } } else { const auto result = client.ResolveConfig(configStr, volatileConfigStrs, Labels).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); auto doc = NFyaml::TDocument::Parse("---\nlabel_sets: []\nconfig: {}\n"); @@ -398,13 +398,13 @@ int TCommandConfigResolve::Run(TConfig& config) { auto map = doc.Buildf("{}"); TSet namedLabels; for (auto& [name, value] : Labels) { - namedLabels.insert(NYamlConfig::TNamedLabel{name, value}); + namedLabels.insert(NYamlConfig::TNamedLabel{TString{name}, TString{value}}); auto node = doc.Buildf("%s: {type: COMMON, value: %s}", name.c_str(), value.c_str()); map.Insert(node); } labelSetsSeq.Append(map); - auto config = NFyaml::TDocument::Parse(result.GetConfig()); + auto config = NFyaml::TDocument::Parse(TString{result.GetConfig()}); auto node = config.Root().Copy(doc); doc.Root().Map().at("config").Insert(node.Ref()); @@ -473,7 +473,7 @@ int TCommandConfigVolatileAdd::Run(TConfig& config) { if (!IgnoreCheck) { auto result = client.GetConfig().GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); if (result.GetConfig().empty()) { ythrow yexception() << "Config on server is empty"; @@ -481,7 +481,7 @@ int TCommandConfigVolatileAdd::Run(TConfig& config) { NYamlConfig::GetVolatileMetadata(configStr); - auto tree = NFyaml::TDocument::Parse(result.GetConfig()); + auto tree = NFyaml::TDocument::Parse(TString{result.GetConfig()}); auto volatileCfg = NFyaml::TDocument::Parse(configStr); auto selectors = volatileCfg.Root().Map().at("selector_config"); @@ -493,7 +493,7 @@ int TCommandConfigVolatileAdd::Run(TConfig& config) { } auto status = client.AddVolatileConfig(configStr).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); Cout << status << Endl; @@ -584,13 +584,13 @@ int TCommandConfigVolatileDrop::Run(TConfig& config) { } if (Force) { - return client.ForceRemoveVolatileConfig(TVector(Ids.begin(), Ids.end())).GetValueSync(); + return client.ForceRemoveVolatileConfig(std::vector(Ids.begin(), Ids.end())).GetValueSync(); } - return client.RemoveVolatileConfig(Cluster, Version, TVector(Ids.begin(), Ids.end())).GetValueSync(); + return client.RemoveVolatileConfig(Cluster, Version, std::vector(Ids.begin(), Ids.end())).GetValueSync(); }(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); Cout << status << Endl; @@ -625,7 +625,7 @@ int TCommandConfigVolatileFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); auto result = client.GetConfig().GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); if (OutDir) { TFsPath dir(OutDir); @@ -636,14 +636,14 @@ int TCommandConfigVolatileFetch::Run(TConfig& config) { for (auto [id, cfg] : result.GetVolatileConfigs()) { if (All || Ids.contains(id)) { - version = NYamlConfig::GetVolatileMetadata(cfg).Version.value(); + version = NYamlConfig::GetVolatileMetadata(TString{cfg}).Version.value(); if (StripMetadata) { - cfg = NYamlConfig::StripMetadata(cfg); + cfg = NYamlConfig::StripMetadata(TString{cfg}); } if (!OutDir) { - Cout << WrapYaml(cfg); + Cout << WrapYaml(TString{cfg}); } else { auto filename = TString("volatile_") + ToString(version) + "_" + ToString(id) + ".yaml"; auto filepath = (TFsPath(OutDir) / filename); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h index 3e43f846cbc6..81e6748fa948 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h @@ -49,7 +49,7 @@ class TCommandConfigResolve : public TYdbCommand { int Run(TConfig& config) override; private: - TMap Labels; + std::map Labels; bool All = false; TString Filename; TString Dir; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_latency.cpp b/ydb/public/lib/ydb_cli/commands/ydb_latency.cpp index 6b90f0eece69..9ca07dc6be21 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_latency.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_latency.cpp @@ -1,7 +1,7 @@ #include "ydb_latency.h" -#include -#include +#include +#include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_ping.cpp b/ydb/public/lib/ydb_cli/commands/ydb_ping.cpp index 94c0ed9ae7be..c77b79236414 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_ping.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_ping.cpp @@ -1,7 +1,7 @@ #include "ydb_ping.h" -#include -#include +#include +#include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_ping.h b/ydb/public/lib/ydb_cli/commands/ydb_ping.h index 27065f8f0dbe..ea7497f61a3d 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_ping.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_ping.h @@ -7,6 +7,7 @@ namespace NYdb { +inline namespace V3 { namespace NQuery { class TQueryClient; class TSession; @@ -14,7 +15,8 @@ namespace NQuery { namespace NDebug { class TDebugClient; -}; +} +} namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp index ff86b7832eeb..903ba2162861 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp @@ -94,8 +94,8 @@ namespace { }; - TString BlurSecret(const TString& in) { - TString out(in); + std::string BlurSecret(const std::string& in) { + std::string out(in); size_t clearSymbolsCount = Min(size_t(10), out.length() / 4); for (size_t i = clearSymbolsCount; i < out.length() - clearSymbolsCount; ++i) { out[i] = '*'; @@ -103,8 +103,8 @@ namespace { return out; } - TString ReplaceWithAsterisks(const TString& in) { - return TString(in.length(), '*'); + std::string ReplaceWithAsterisks(const std::string& in) { + return std::string(in.length(), '*'); } void SetupProfileName(TString& profileName, std::shared_ptr profileManager) { @@ -197,7 +197,7 @@ namespace { } } - TString TryBlurValue(const TString& authMethod, const TString& value) { + std::string TryBlurValue(const TString& authMethod, const std::string& value) { if (!IsStdoutInteractive() || authMethod == "sa-key-file" || authMethod == "token-file" || authMethod == "yc-token-file" || authMethod == "oauth2-key-file") { return value; } @@ -297,10 +297,10 @@ void TCommandConnectionInfo::PrintInfo(TConfig& config) { } } if (config.UseStaticCredentials) { - if (config.StaticCredentials.User) { + if (!config.StaticCredentials.User.empty()) { Cout << "user: " << config.StaticCredentials.User << Endl; } - if (config.StaticCredentials.Password) { + if (!config.StaticCredentials.Password.empty()) { Cout << "password: " << TryBlurValue("password", config.StaticCredentials.Password) << Endl; } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 3041806714ae..58ccceeac9d9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -18,9 +18,9 @@ #include "ydb_workload.h" #include -#include -#include -#include +#include +#include +#include #include #include @@ -98,7 +98,7 @@ void TClientCommandRootCommon::SetCredentialsGetter(TConfig& config) { } if (config.UseStaticCredentials) { - if (config.StaticCredentials.User) { + if (!config.StaticCredentials.User.empty()) { return CreateLoginCredentialsProviderFactory(config.StaticCredentials); } } @@ -254,7 +254,7 @@ void TClientCommandRootCommon::Config(TConfig& config) { if (config.HelpCommandVerbosiltyLevel >= 2) { TStringBuilder supportedJwtAlgorithms; - for (const TString& alg : GetSupportedOauth2TokenExchangeJwtAlgorithms()) { + for (const std::string& alg : GetSupportedOauth2TokenExchangeJwtAlgorithms()) { if (supportedJwtAlgorithms) { supportedJwtAlgorithms << ", "; } @@ -802,7 +802,7 @@ bool TClientCommandRootCommon::GetCredentialsFromProfile(std::shared_ptr(); - if (!config.StaticCredentials.Password) { + if (config.StaticCredentials.Password.empty()) { DoNotAskForPassword = true; } } @@ -820,7 +820,7 @@ bool TClientCommandRootCommon::GetCredentialsFromProfile(std::shared_ptr +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_sdk_core_access.h b/ydb/public/lib/ydb_cli/commands/ydb_sdk_core_access.h index 5f2f4b72a999..8821a7e09eb0 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_sdk_core_access.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_sdk_core_access.h @@ -1,9 +1,9 @@ #pragma once -#include +#include #include -namespace NYdb { +namespace NYdb::inline V3 { class TDriver; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_auth.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_auth.cpp index 4df91337edf0..e582d4f533f9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_auth.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_auth.cpp @@ -40,7 +40,7 @@ int TCommandGetToken::Run(TConfig& config) { TDummyClient client(driver); auto authInfo = credentialsProviderFactory->CreateProvider(client.GetCoreFacility())->GetAuthInfo(); - if (authInfo) { + if (!authInfo.empty()) { Cout << authInfo << Endl; return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp index 18aa3a65e5a1..523cd611e66c 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp @@ -24,13 +24,13 @@ int TCommandListEndpoints::Run(TConfig& config) { NDiscovery::TListEndpointsResult result = client.ListEndpoints( FillSettings(NDiscovery::TListEndpointsSettings()) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); PrintResponse(result); return EXIT_SUCCESS; } void TCommandListEndpoints::PrintResponse(NDiscovery::TListEndpointsResult& result) { - const TVector& endpoints = result.GetEndpointsInfo(); + const std::vector& endpoints = result.GetEndpointsInfo(); if (endpoints.size()) { for (auto& endpoint : endpoints) { if (endpoint.Ssl) { @@ -39,7 +39,7 @@ void TCommandListEndpoints::PrintResponse(NDiscovery::TListEndpointsResult& resu Cout << "grpc://"; } Cout << endpoint.Address << ":" << endpoint.Port; - if (endpoint.Location) { + if (!endpoint.Location.empty()) { Cout << " [" << endpoint.Location << "]"; } for (const auto& service : endpoint.Services) { @@ -68,21 +68,21 @@ int TCommandWhoAmI::Run(TConfig& config) { NDiscovery::TWhoAmIResult result = client.WhoAmI( FillSettings(NDiscovery::TWhoAmISettings().WithGroups(WithGroups)) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); PrintResponse(result); driver.Stop(true); return EXIT_SUCCESS; } void TCommandWhoAmI::PrintResponse(NDiscovery::TWhoAmIResult& result) { - const TString& userName = result.GetUserName(); - if (userName) { + const std::string& userName = result.GetUserName(); + if (!userName.empty()) { Cout << "User SID: " << userName << Endl; if (WithGroups) { - const TVector& groups = result.GetGroups(); + const std::vector& groups = result.GetGroups(); if (groups.size() > 0) { Cout << Endl << "Group SIDs:" << Endl; - for (const TString& group : groups) { + for (const std::string& group : groups) { Cout << group << Endl; } } else { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h index 153718bfa023..f0e25dc3f376 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h @@ -3,7 +3,7 @@ #include #include -#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp index 3e1dd10911ab..bd9269a418a9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp @@ -1,6 +1,6 @@ #include "ydb_service_export.h" -#include +#include #include #include #include @@ -28,7 +28,7 @@ namespace { dstPath.ChopSuffix(slash); const auto ret = RecursiveList(client, TString{srcPath}, TRecursiveListSettings().Filter(&FilterTables)); - ThrowOnError(ret.Status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(ret.Status); if (ret.Entries.size() == 1 && srcPath == ret.Entries[0].Name) { return {{TString{srcPath}, TString{dstPath}}}; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h index 95371ff24565..3bfab34a567a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h @@ -3,7 +3,7 @@ #include "ydb_command.h" #include "ydb_common.h" -#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp index 6834e0977e66..7421b4951ede 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp @@ -313,7 +313,7 @@ int TCommandImportFromCsv::Run(TConfig& config) { } TImportFileClient client(CreateDriver(config), config, settings); - ThrowOnError(client.Import(FilePaths, Path)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Import(FilePaths, Path)); return EXIT_SUCCESS; } @@ -343,7 +343,7 @@ int TCommandImportFromJson::Run(TConfig& config) { settings.Threads(Threads); TImportFileClient client(CreateDriver(config), config, settings); - ThrowOnError(client.Import(FilePaths, Path)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Import(FilePaths, Path)); return EXIT_SUCCESS; } @@ -362,7 +362,7 @@ int TCommandImportFromParquet::Run(TConfig& config) { settings.Threads(Threads); TImportFileClient client(CreateDriver(config), config, settings); - ThrowOnError(client.Import(FilePaths, Path)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Import(FilePaths, Path)); return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h index 5702f4bc9705..27da29f8584d 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h @@ -2,8 +2,8 @@ #include "ydb_command.h" -#include -#include +#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.cpp index d28fc16eddc5..363abd66bd94 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.cpp @@ -1,7 +1,7 @@ #include "ydb_service_monitoring.h" #include -#include +#include namespace NYdb { namespace NConsoleClient { @@ -42,7 +42,7 @@ int TCommandSelfCheck::Run(TConfig& config) { NMonitoring::TSelfCheckResult result = client.SelfCheck( FillSettings(settings) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return PrintResponse(result); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.h b/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.h index d4f558c272b4..81c804fc15c6 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_monitoring.h @@ -4,7 +4,7 @@ #include "ydb_common.h" #include -#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp index 99e32abbfd20..ec002497824a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp @@ -1,9 +1,9 @@ #include "ydb_service_operation.h" -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -34,7 +34,7 @@ namespace { template void ListOperations(NOperation::TOperationClient& client, ui64 pageSize, const TString& pageToken, EDataFormat format) { NOperation::TOperationsList operations = client.List(pageSize, pageToken).GetValueSync(); - ThrowOnError(operations); + NStatusHelpers::ThrowOnErrorOrPrintIssues(operations); PrintOperationsList(operations, format); } @@ -87,21 +87,21 @@ int TCommandGetOperation::Run(TConfig& config) { NOperation::TOperationClient client(CreateDriver(config)); switch (OperationId.GetKind()) { - case Ydb::TOperationId::EXPORT: + case TOperationId::EXPORT: if (OperationId.GetSubKind() == "s3") { return GetOperation(client, OperationId, OutputFormat); } else { // fallback to "yt" return GetOperation(client, OperationId, OutputFormat); } - case Ydb::TOperationId::IMPORT: + case TOperationId::IMPORT: if (OperationId.GetSubKind() == "s3") { return GetOperation(client, OperationId, OutputFormat); } else { throw TMisuseException() << "Invalid operation ID (unexpected sub-kind of operation)"; } - case Ydb::TOperationId::BUILD_INDEX: + case TOperationId::BUILD_INDEX: return GetOperation(client, OperationId, OutputFormat); - case Ydb::TOperationId::SCRIPT_EXECUTION: + case TOperationId::SCRIPT_EXECUTION: return GetOperation(client, OperationId, OutputFormat); default: throw TMisuseException() << "Invalid operation ID (unexpected kind of operation)"; @@ -117,7 +117,7 @@ TCommandCancelOperation::TCommandCancelOperation() int TCommandCancelOperation::Run(TConfig& config) { NOperation::TOperationClient client(CreateDriver(config)); - ThrowOnError(client.Cancel(OperationId).GetValueSync()); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Cancel(OperationId).GetValueSync()); return EXIT_SUCCESS; } @@ -128,7 +128,7 @@ TCommandForgetOperation::TCommandForgetOperation() int TCommandForgetOperation::Run(TConfig& config) { NOperation::TOperationClient client(CreateDriver(config)); - ThrowOnError(client.Forget(OperationId).GetValueSync()); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Forget(OperationId).GetValueSync()); return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h index f0b79e2b3786..8ae54c072e0a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h @@ -3,8 +3,8 @@ #include "ydb_command.h" #include "ydb_common.h" -#include -#include +#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index 39f472423725..75627fbe119a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -3,8 +3,10 @@ #include #include #include -#include -#include +#include +#include + +#include #include @@ -46,7 +48,7 @@ void TCommandMakeDirectory::Parse(TConfig& config) { int TCommandMakeDirectory::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.MakeDirectory( Path, FillSettings(NScheme::TMakeDirectorySettings()) @@ -89,25 +91,25 @@ int TCommandRemoveDirectory::Run(TConfig& config) { NTopic::TTopicClient topicClient(driver); NQuery::TQueryClient queryClient(driver); const auto prompt = Prompt.GetOrElse(ERecursiveRemovePrompt::Once); - ThrowOnError(RemoveDirectoryRecursive(schemeClient, tableClient, &topicClient, &queryClient, Path, prompt, settings)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(RemoveDirectoryRecursive(schemeClient, tableClient, &topicClient, &queryClient, Path, prompt, settings)); } else { if (Prompt) { if (!NConsoleClient::Prompt(*Prompt, Path, NScheme::ESchemeEntryType::Directory)) { return EXIT_SUCCESS; } } - ThrowOnError(schemeClient.RemoveDirectory(Path, settings).GetValueSync()); + NStatusHelpers::ThrowOnErrorOrPrintIssues(schemeClient.RemoveDirectory(Path, settings).GetValueSync()); } return EXIT_SUCCESS; } namespace { - void PrintPermissions(const TVector& permissions) { + void PrintPermissions(const std::vector& permissions) { if (permissions.size()) { for (const NScheme::TPermissions& permission : permissions) { Cout << permission.Subject << ":"; - for (const TString& name : permission.PermissionNames) { + for (const std::string& name : permission.PermissionNames) { if (name != *permission.PermissionNames.begin()) { Cout << ","; } @@ -122,9 +124,9 @@ namespace { } void PrintAllPermissions( - const TString& owner, - const TVector& permissions, - const TVector& effectivePermissions + const std::string& owner, + const std::vector& permissions, + const std::vector& effectivePermissions ) { Cout << "Owner: " << owner << Endl << Endl << "Permissions: " << Endl; PrintPermissions(permissions); @@ -256,7 +258,7 @@ int TCommandDescribe::Run(TConfig& config) { if (!result.IsSuccess()) { return TryTopicConsumerDescribeOrFail(driver, result); } - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return PrintPathResponse(driver, result); } @@ -293,11 +295,11 @@ int TCommandDescribe::DescribeEntryDefault(NScheme::TSchemeEntry entry) { } namespace { - TString FormatCodecs(const TVector& codecs) { + TString FormatCodecs(const std::vector& codecs) { return JoinSeq(", ", codecs); } - void PrintTopicConsumers(const TVector& consumers) { + void PrintTopicConsumers(const std::vector& consumers) { if (consumers.empty()) { return; } @@ -332,7 +334,7 @@ namespace { void PrintMain(const NTopic::TTopicDescription& topicDescription) { Cout << Endl << "Main:"; Cout << Endl << "RetentionPeriod: " << topicDescription.GetRetentionPeriod().Hours() << " hours"; - if (topicDescription.GetRetentionStorageMb().Defined()) { + if (topicDescription.GetRetentionStorageMb().has_value()) { Cout << Endl << "StorageRetention: " << *topicDescription.GetRetentionStorageMb() << " MB"; } Cout << Endl << "PartitionsCount: " << topicDescription.GetTotalPartitionsCount(); @@ -419,7 +421,7 @@ int TCommandDescribe::DescribeTopic(TDriver& driver) { settings.IncludeStats(ShowStats || ShowPartitionStats); auto result = topicClient.DescribeTopic(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); const auto& desc = result.GetTopicDescription(); return PrintDescription(this, OutputFormat, desc, &TCommandDescribe::PrintTopicResponsePretty); @@ -428,7 +430,7 @@ int TCommandDescribe::DescribeTopic(TDriver& driver) { int TCommandDescribe::DescribeTable(TDriver& driver) { NTable::TTableClient client(driver); NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(sessionResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(sessionResult); NTable::TDescribeTableResult result = sessionResult.GetSession().DescribeTable( Path, FillSettings( @@ -438,7 +440,7 @@ int TCommandDescribe::DescribeTable(TDriver& driver) { .WithPartitionStatistics(ShowPartitionStats) ) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); auto desc = result.GetTableDescription(); return PrintDescription(this, OutputFormat, desc, &TCommandDescribe::PrintTableResponsePretty); @@ -447,7 +449,7 @@ int TCommandDescribe::DescribeTable(TDriver& driver) { int TCommandDescribe::DescribeColumnTable(TDriver& driver) { NTable::TTableClient client(driver); NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(sessionResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(sessionResult); NTable::TDescribeTableResult result = sessionResult.GetSession().DescribeTable( Path, FillSettings( @@ -455,7 +457,7 @@ int TCommandDescribe::DescribeColumnTable(TDriver& driver) { .WithTableStatistics(ShowStats) ) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); auto desc = result.GetTableDescription(); return PrintDescription(this, OutputFormat, desc, &TCommandDescribe::PrintTableResponsePretty); @@ -464,11 +466,11 @@ int TCommandDescribe::DescribeColumnTable(TDriver& driver) { int TCommandDescribe::PrintCoordinationNodeResponsePretty(const NYdb::NCoordination::TNodeDescription& result) const { Cout << Endl << "AttachConsistencyMode: " << result.GetAttachConsistencyMode() << Endl; Cout << "ReadConsistencyMode: " << result.GetReadConsistencyMode() << Endl; - if (result.GetSessionGracePeriod().Defined()) { - Cout << "SessionGracePeriod: " << result.GetSessionGracePeriod() << Endl; + if (result.GetSessionGracePeriod().has_value()) { + Cout << "SessionGracePeriod: " << result.GetSessionGracePeriod().value() << Endl; } - if (result.GetSelfCheckPeriod().Defined()) { - Cout << "SelfCheckPeriod: " << result.GetSelfCheckPeriod() << Endl; + if (result.GetSelfCheckPeriod().has_value()) { + Cout << "SelfCheckPeriod: " << result.GetSelfCheckPeriod().value() << Endl; } Cout << "RatelimiterCountersMode: " << result.GetRateLimiterCountersMode() << Endl; return EXIT_SUCCESS; @@ -558,7 +560,7 @@ int TCommandDescribe::PrintReplicationResponsePretty(const NYdb::NReplication::T break; } - if (const auto& items = desc.GetItems()) { + if (const auto& items = desc.GetItems(); !items.empty()) { TVector columnNames = { "#", "Source", "Destination", "Changefeed" }; if (ShowStats) { columnNames.push_back("Lag"); @@ -591,7 +593,7 @@ int TCommandDescribe::DescribeReplication(const TDriver& driver) { .IncludeStats(ShowStats); auto result = client.DescribeReplication(Path, settings).ExtractValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return PrintDescription(this, OutputFormat, result, &TCommandDescribe::PrintReplicationResponsePretty); } @@ -604,7 +606,7 @@ int TCommandDescribe::PrintViewResponsePretty(const NYdb::NView::TDescribeViewRe int TCommandDescribe::DescribeView(const TDriver& driver) { NView::TViewClient client(driver); auto result = client.DescribeView(Path, {}).ExtractValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return PrintDescription(this, OutputFormat, result, &TCommandDescribe::PrintViewResponsePretty); } @@ -617,7 +619,7 @@ namespace { Cerr << Endl; TPrettyTable table({ "Name", "Type", "Family", "Key" }, TPrettyTableConfig().WithoutRowDelimiters()); - const TVector& keyColumns = tableDescription.GetPrimaryKeyColumns(); + const std::vector& keyColumns = tableDescription.GetPrimaryKeyColumns(); for (const NTable::TTableColumn& column : tableDescription.GetTableColumns()) { TString key = ""; auto itKey = std::find(keyColumns.begin(), keyColumns.end(), column.Name); @@ -641,7 +643,7 @@ namespace { } void PrintIndexes(const NTable::TTableDescription& tableDescription) { - const TVector& indexes = tableDescription.GetIndexDescriptions(); + const std::vector& indexes = tableDescription.GetIndexDescriptions(); if (!indexes.size()) { return; } @@ -693,37 +695,37 @@ namespace { const auto commitLog1 = settings.GetTabletCommitLog1(); const auto external = settings.GetExternal(); const auto storeExternalBlobs = settings.GetStoreExternalBlobs(); - if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.Defined()) { + if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.has_value()) { return; } Cout << Endl << "Storage settings: " << Endl; if (commitLog0) { - Cout << "Internal channel 0 commit log storage pool: " << commitLog0.GetRef() << Endl; + Cout << "Internal channel 0 commit log storage pool: " << commitLog0.value() << Endl; } if (commitLog1) { - Cout << "Internal channel 1 commit log storage pool: " << commitLog1.GetRef() << Endl; + Cout << "Internal channel 1 commit log storage pool: " << commitLog1.value() << Endl; } if (external) { - Cout << "External blobs storage pool: " << external.GetRef() << Endl; + Cout << "External blobs storage pool: " << external.value() << Endl; } if (storeExternalBlobs) { Cout << "Store large values in \"external blobs\": " - << (storeExternalBlobs.GetRef() ? "true" : "false") << Endl; + << (storeExternalBlobs.value() ? "true" : "false") << Endl; } } void PrintColumnFamilies(const NTable::TTableDescription& tableDescription) { - if (!tableDescription.GetColumnFamilies()) { + if (tableDescription.GetColumnFamilies().empty()) { return; } TPrettyTable table({ "Name", "Data", "Compression", "Keep in memory" }, TPrettyTableConfig().WithoutRowDelimiters()); for (const NTable::TColumnFamilyDescription& family : tableDescription.GetColumnFamilies()) { - TMaybe data = family.GetData(); + std::optional data = family.GetData(); TString compression; if (family.GetCompression()) { - switch (family.GetCompression().GetRef()) { + switch (family.GetCompression().value()) { case NTable::EColumnFamilyCompression::None: compression = "None"; break; @@ -732,16 +734,16 @@ namespace { break; default: compression = TStringBuilder() << "unknown(" - << static_cast(family.GetCompression().GetRef()) << ")"; + << static_cast(family.GetCompression().value()) << ")"; } } TStringBuilder keepInMemory; - if (family.GetKeepInMemory().Defined()) { - keepInMemory << keepInMemory << family.GetKeepInMemory().GetRef(); + if (family.GetKeepInMemory().has_value()) { + keepInMemory << keepInMemory << family.GetKeepInMemory().value(); } table.AddRow() .Column(0, family.GetName()) - .Column(1, data ? data.GetRef() : "") + .Column(1, data ? data.value() : "") .Column(2, compression) .Column(3, keepInMemory); } @@ -750,7 +752,7 @@ namespace { } void PrintAttributes(const NTable::TTableDescription& tableDescription) { - if (!tableDescription.GetAttributes().size()) { + if (tableDescription.GetAttributes().empty()) { return; } TPrettyTable table({ "Name", "Value" }, TPrettyTableConfig().WithoutRowDelimiters()); @@ -805,16 +807,16 @@ namespace { const auto& settings = tableDescription.GetPartitioningSettings(); const auto partBySize = settings.GetPartitioningBySize(); const auto partByLoad = settings.GetPartitioningByLoad(); - if (!partBySize.Defined() && !partByLoad.Defined()) { + if (!partBySize.has_value() && !partByLoad.has_value()) { return; } const auto partitionSizeMb = settings.GetPartitionSizeMb(); const auto minPartitions = settings.GetMinPartitionsCount(); const auto maxPartitions = settings.GetMaxPartitionsCount(); Cout << Endl << "Auto partitioning settings: " << Endl; - Cout << "Partitioning by size: " << (partBySize.GetRef() ? "true" : "false") << Endl; - Cout << "Partitioning by load: " << (partByLoad.GetRef() ? "true" : "false") << Endl; - if (partBySize.Defined() && partitionSizeMb) { + Cout << "Partitioning by size: " << (partBySize.value() ? "true" : "false") << Endl; + Cout << "Partitioning by load: " << (partByLoad.value() ? "true" : "false") << Endl; + if (partBySize.has_value() && partitionSizeMb) { Cout << "Preferred partition size (Mb): " << partitionSizeMb << Endl; } if (minPartitions) { @@ -855,8 +857,8 @@ namespace { } void PrintPartitionInfo(const NTable::TTableDescription& tableDescription, bool showBoundaries, bool showStats) { - const TVector& ranges = tableDescription.GetKeyRanges(); - const TVector& stats = tableDescription.GetPartitionStats(); + const std::vector& ranges = tableDescription.GetKeyRanges(); + const std::vector& stats = tableDescription.GetPartitionStats(); if (showBoundaries) { if (showStats) { Cout << Endl << "Partitions info:" << Endl; @@ -904,10 +906,10 @@ namespace { row.Column(j++, i + 1); if (showBoundaries) { const NTable::TKeyRange& keyRange = ranges[i]; - const TMaybe& from = keyRange.From(); - const TMaybe& to = keyRange.To(); - if (from.Defined()) { - const NTable::TKeyBound& bound = from.GetRef(); + const std::optional& from = keyRange.From(); + const std::optional& to = keyRange.To(); + if (from.has_value()) { + const NTable::TKeyBound& bound = from.value(); if (bound.IsInclusive()) { row.Column(j++, "["); } else { @@ -918,8 +920,8 @@ namespace { row.Column(j++, "("); row.Column(j++, "-Inf"); } - if (to.Defined()) { - const NTable::TKeyBound& bound = to.GetRef(); + if (to.has_value()) { + const NTable::TKeyBound& bound = to.value(); row.Column(j++, FormatValueJson(bound.GetValue(), EBinaryStringEncoding::Unicode)); if (bound.IsInclusive()) { row.Column(j++, "]"); @@ -950,9 +952,9 @@ int TCommandDescribe::PrintTableResponsePretty(const NTable::TTableDescription& PrintAttributes(tableDescription); PrintTtlSettings(tableDescription); PrintPartitioningSettings(tableDescription); - if (tableDescription.GetKeyBloomFilter().Defined()) { + if (tableDescription.GetKeyBloomFilter().has_value()) { Cout << Endl << "Bloom filter by key: " - << (tableDescription.GetKeyBloomFilter().GetRef() ? "true" : "false") << Endl; + << (tableDescription.GetKeyBloomFilter().value() ? "true" : "false") << Endl; } PrintReadReplicasSettings(tableDescription); PrintPermissionsIfNeeded(tableDescription); @@ -979,7 +981,7 @@ std::pair TCommandDescribe::ParseTopicConsumer() const { int TCommandDescribe::TryTopicConsumerDescribeOrFail(TDriver& driver, const NScheme::TDescribePathResult& result) { auto [topic, consumer] = ParseTopicConsumer(); if (!topic || !consumer) { - ThrowOnError(result); // no consumer can be found + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); // no consumer can be found } NScheme::TSchemeClient client(driver); @@ -988,13 +990,13 @@ int TCommandDescribe::TryTopicConsumerDescribeOrFail(TDriver& driver, const NSch FillSettings(NScheme::TDescribePathSettings()) ).GetValueSync(); if (!topicDescribeResult.IsSuccess() || topicDescribeResult.GetEntry().Type != NScheme::ESchemeEntryType::Topic && topicDescribeResult.GetEntry().Type != NScheme::ESchemeEntryType::PqGroup) { - ThrowOnError(result); // return previous error, this is not topic + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); // return previous error, this is not topic } // OK, this is topic, check the consumer NYdb::NTopic::TTopicClient topicClient(driver); auto consumerDescription = topicClient.DescribeConsumer(topic, consumer, NYdb::NTopic::TDescribeConsumerSettings().IncludeStats(ShowPartitionStats)).GetValueSync(); - ThrowOnError(consumerDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(consumerDescription); return PrintDescription(this, OutputFormat, consumerDescription.GetConsumerDescription(), &TCommandDescribe::PrintConsumerResponsePretty); } @@ -1129,7 +1131,7 @@ void TCommandPermissionGrant::Parse(TConfig& config) { TClientCommand::Parse(config); ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { + if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; } if (!PermissionsToGrant.size()) { @@ -1139,7 +1141,7 @@ void TCommandPermissionGrant::Parse(TConfig& config) { int TCommandPermissionGrant::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1170,7 +1172,7 @@ void TCommandPermissionRevoke::Parse(TConfig& config) { TClientCommand::Parse(config); ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { + if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; } if (!PermissionsToRevoke.size()) { @@ -1180,7 +1182,7 @@ void TCommandPermissionRevoke::Parse(TConfig& config) { int TCommandPermissionRevoke::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1211,7 +1213,7 @@ void TCommandPermissionSet::Parse(TConfig& config) { TClientCommand::Parse(config); ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { + if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; } if (!PermissionsToSet.size()) { @@ -1221,7 +1223,7 @@ void TCommandPermissionSet::Parse(TConfig& config) { int TCommandPermissionSet::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1256,7 +1258,7 @@ void TCommandChangeOwner::Parse(TConfig& config) { int TCommandChangeOwner::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1286,7 +1288,7 @@ void TCommandPermissionClear::Parse(TConfig& config) { int TCommandPermissionClear::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1316,7 +1318,7 @@ void TCommandPermissionSetInheritance::Parse(TConfig& config) { int TCommandPermissionSetInheritance::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1346,7 +1348,7 @@ void TCommandPermissionClearInheritance::Parse(TConfig& config) { int TCommandPermissionClearInheritance::Run(TConfig& config) { NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( Path, FillSettings( @@ -1381,7 +1383,7 @@ int TCommandPermissionList::Run(TConfig& config) { Path, FillSettings(NScheme::TDescribePathSettings()) ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); NScheme::TSchemeEntry entry = result.GetEntry(); Cout << Endl; PrintAllPermissions(entry.Owner, entry.Permissions, entry.EffectivePermissions); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h index edcf8da170b1..4e01a32fafa6 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h @@ -6,13 +6,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include namespace NYdb { @@ -47,9 +47,9 @@ class TCommandRemoveDirectory : public TYdbOperationCommand, public TCommandWith }; void PrintAllPermissions( - const TString& owner, - const TVector& permissions, - const TVector& effectivePermissions + const std::string& owner, + const std::vector& permissions, + const std::vector& effectivePermissions ); // Pretty print consumer info ('scheme describe' and 'topic consumer describe' commands) @@ -156,8 +156,8 @@ class TCommandPermissionGrant : public TYdbOperationCommand, public TCommandWith virtual int Run(TConfig& config) override; private: - TString Subject; - TVector PermissionsToGrant; + std::string Subject; + std::vector PermissionsToGrant; }; class TCommandPermissionRevoke : public TYdbOperationCommand, public TCommandWithPath { @@ -168,8 +168,8 @@ class TCommandPermissionRevoke : public TYdbOperationCommand, public TCommandWit virtual int Run(TConfig& config) override; private: - TString Subject; - TVector PermissionsToRevoke; + std::string Subject; + std::vector PermissionsToRevoke; }; class TCommandPermissionSet : public TYdbOperationCommand, public TCommandWithPath { @@ -180,8 +180,8 @@ class TCommandPermissionSet : public TYdbOperationCommand, public TCommandWithPa virtual int Run(TConfig& config) override; private: - TString Subject; - TVector PermissionsToSet; + std::string Subject; + std::vector PermissionsToSet; }; class TCommandChangeOwner : public TYdbOperationCommand, public TCommandWithPath { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index cbc0551fe9f6..3f750b5393d1 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -102,7 +102,7 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { auto result = client.ExplainYqlScript(Script, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); PrintExplainResult(result); } else { NScripting::TExecuteYqlRequestSettings settings; @@ -124,7 +124,7 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); PrintResponseHeader(result); PrintResponse(result); } @@ -134,7 +134,7 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { FillSettings(settings) ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); PrintResponseHeader(result); PrintResponse(result); } @@ -146,7 +146,7 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& result) { { TResultSetPrinter printer(OutputFormat); - const TVector& resultSets = result.GetResultSets(); + const std::vector& resultSets = result.GetResultSets(); for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { if (resultSetIt != resultSets.begin()) { printer.Reset(); @@ -155,8 +155,8 @@ void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& resu } } // TResultSetPrinter destructor should be called before printing stats - const TMaybe& stats = result.GetStats(); - if (stats.Defined()) { + const std::optional& stats = result.GetStats(); + if (stats.has_value()) { Cout << Endl << "Statistics:" << Endl << stats->ToString(); auto fullStats = stats->GetPlan(); @@ -164,11 +164,11 @@ void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& resu Cout << Endl << "Full statistics:" << Endl; TQueryPlanPrinter queryPlanPrinter(OutputFormat, /* analyzeMode */ true); - queryPlanPrinter.Print(*fullStats); + queryPlanPrinter.Print(TString{*fullStats}); if (FlameGraphPath) { try { - NKikimr::NVisual::GenerateFlameGraphSvg(*FlameGraphPath, *fullStats); + NKikimr::NVisual::GenerateFlameGraphSvg(*FlameGraphPath, TString{*fullStats}); Cout << "Resource usage flame graph is successfully saved to " << *FlameGraphPath << Endl; } catch (const yexception& ex) { @@ -181,7 +181,7 @@ void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& resu void TCommandExecuteYqlScript::PrintExplainResult(NScripting::TExplainYqlResult& result) { TQueryPlanPrinter queryPlanPrinter(OutputFormat); - queryPlanPrinter.Print(result.GetPlan()); + queryPlanPrinter.Print(TString{result.GetPlan()}); } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h index 3665d746d0eb..6a45b9779b79 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h @@ -3,7 +3,7 @@ #include "ydb_command.h" #include "ydb_common.h" -#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 0200655e536a..08437be17a18 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -81,7 +81,7 @@ void TTableCommand::Config(TConfig& config) { NTable::TSession TTableCommand::GetSession(TConfig& config) { NTable::TTableClient client(CreateDriver(config)); NTable::TCreateSessionResult result = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return result.GetSession(); } @@ -232,9 +232,9 @@ int TCommandCreateTable::Run(TConfig& config) { throw TMisuseException() << "Can't parse index \"" << index << "\". Need exactly one colon. Expected format: \":[,,...]\""; } - TVector columns = StringSplitter(parts[1]).Split(','); - for (TString& column : columns) { - if (!column) { + std::vector columns = StringSplitter(parts[1]).Split(','); + for (std::string& column : columns) { + if (column.empty()) { throw TMisuseException() << "Can't parse index \"" << index << "\". Empty column names found. Expected format: \":[,,...]\""; } @@ -291,7 +291,7 @@ int TCommandCreateTable::Run(TConfig& config) { replicationPolicy.AllowPromotion(AllowPromotion); } - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( GetSession(config).CreateTable( Path, builder.Build(), @@ -318,7 +318,7 @@ void TCommandDropTable::Parse(TConfig& config) { } int TCommandDropTable::Run(TConfig& config) { - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( GetSession(config).DropTable( Path, FillSettings(NTable::TDropTableSettings()) @@ -469,7 +469,7 @@ int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { }); }; auto status = client.RetryOperation(std::move(operation)).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); auto result = asyncResult.GetValueSync(); PrintDataQueryResponse(result); } @@ -488,7 +488,7 @@ int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { }); }; auto status = client.RetryOperation(std::move(operation)).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); auto result = asyncResult.GetValueSync(); PrintDataQueryResponse(result); } @@ -498,7 +498,7 @@ int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { void TCommandExecuteQuery::PrintDataQueryResponse(NTable::TDataQueryResult& result) { { TResultSetPrinter printer(OutputFormat); - const TVector& resultSets = result.GetResultSets(); + const std::vector& resultSets = result.GetResultSets(); for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { if (resultSetIt != resultSets.begin()) { printer.Reset(); @@ -507,19 +507,19 @@ void TCommandExecuteQuery::PrintDataQueryResponse(NTable::TDataQueryResult& resu } } // TResultSetPrinter destructor should be called before printing stats - const TMaybe& stats = result.GetStats(); - if (stats.Defined()) { + const std::optional& stats = result.GetStats(); + if (stats.has_value()) { Cout << Endl << "Statistics:" << Endl << stats->ToString(); PrintFlameGraph(stats->GetPlan()); } - if (FlameGraphPath && !stats.Defined()) + if (FlameGraphPath && !stats.has_value()) { Cout << Endl << "Flame graph is available for full or profile stats only" << Endl; } } int TCommandExecuteQuery::ExecuteSchemeQuery(TConfig& config) { - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( GetSession(config).ExecuteSchemeQuery( Query, FillSettings(NTable::TExecSchemeQuerySettings()) @@ -631,7 +631,7 @@ namespace { if constexpr (std::is_same_v) { return part.HasQueryStats(); } else if constexpr (std::is_same_v) { - return !part.GetStats().Empty(); + return part.GetStats().has_value(); } Y_UNREACHABLE(); } @@ -697,7 +697,7 @@ int TCommandExecuteQuery::ExecuteQueryImpl(TConfig& config) { }); }; auto status = RunOperation(client, operation).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); auto result = asyncResult.GetValueSync(); if (!PrintQueryResponse(result)) { return EXIT_FAILURE; @@ -719,7 +719,7 @@ int TCommandExecuteQuery::ExecuteQueryImpl(TConfig& config) { }); }; auto status = RunOperation(client, operation).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); auto result = asyncResult.GetValueSync(); if (!PrintQueryResponse(result)) { return EXIT_FAILURE; @@ -731,7 +731,7 @@ int TCommandExecuteQuery::ExecuteQueryImpl(TConfig& config) { template bool TCommandExecuteQuery::PrintQueryResponse(TIterator& result) { TMaybe stats; - TMaybe fullStats; + std::optional fullStats; { TResultSetPrinter printer(OutputFormat, &IsInterrupted); @@ -764,7 +764,7 @@ bool TCommandExecuteQuery::PrintQueryResponse(TIterator& result) { Cout << Endl << "Full statistics:" << Endl; TQueryPlanPrinter queryPlanPrinter(OutputFormat, /* analyzeMode */ true); - queryPlanPrinter.Print(*fullStats); + queryPlanPrinter.Print(TString{*fullStats}); } PrintFlameGraph(fullStats); @@ -776,7 +776,7 @@ bool TCommandExecuteQuery::PrintQueryResponse(TIterator& result) { return true; } -void TCommandExecuteQuery::PrintFlameGraph(const TMaybe& plan) +void TCommandExecuteQuery::PrintFlameGraph(const std::optional& plan) { if (!FlameGraphPath) { return; @@ -790,7 +790,7 @@ void TCommandExecuteQuery::PrintFlameGraph(const TMaybe& plan) return; } try { - NKikimr::NVisual::GenerateFlameGraphSvg(FlameGraphPath.GetRef(), *plan); + NKikimr::NVisual::GenerateFlameGraphSvg(FlameGraphPath.GetRef(), TString{*plan}); Cout << Endl << "Resource usage flame graph is successfully saved to " << FlameGraphPath << Endl; } catch (const yexception &ex) { @@ -840,7 +840,7 @@ void TCommandExplain::Config(TConfig& config) { config.SetFreeArgsNum(0); } -void TCommandExplain::SaveDiagnosticsToFile(const TString& diagnostics) { +void TCommandExplain::SaveDiagnosticsToFile(const std::string& diagnostics) { TFileOutput file(TStringBuilder() << "diagnostics_" << TGUID::Create().AsGuidString() << ".txt"); file << diagnostics; } @@ -877,7 +877,7 @@ int TCommandExplain::Run(TConfig& config) { } auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); TString diagnostics; @@ -919,7 +919,7 @@ int TCommandExplain::Run(TConfig& config) { Query, NQuery::TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); SetInterruptHandlers(); while (!IsInterrupted()) { @@ -946,7 +946,7 @@ int TCommandExplain::Run(TConfig& config) { NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(), FillSettings(settings) ).ExtractValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); planJson = result.GetQueryPlan(); if (auto stats = result.GetStats()) { auto proto = NYdb::TProtoAccessor::GetProto(*stats); @@ -962,7 +962,7 @@ int TCommandExplain::Run(TConfig& config) { Query, settings ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); planJson = result.GetPlan(); ast = result.GetAst(); @@ -1099,12 +1099,12 @@ int TCommandReadTable::Run(TConfig& config) { readTableSettings.Ordered(Ordered); } if (Columns) { - readTableSettings.Columns_ = StringSplitter(Columns).Split(',').ToList(); + readTableSettings.Columns_ = StringSplitter(Columns).Split(',').ToList(); } if (From || To) { NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(sessionResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(sessionResult); NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable(Path).GetValueSync(); NTable::TTableDescription tableDescription = tableResult.GetTableDescription(); @@ -1125,7 +1125,7 @@ int TCommandReadTable::Run(TConfig& config) { TMaybe tableIterator; - ThrowOnError(client.RetryOperationSync([this, &readTableSettings, &tableIterator](NTable::TSession session) { + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.RetryOperationSync([this, &readTableSettings, &tableIterator](NTable::TSession session) { NTable::TTablePartIterator result = session.ReadTable(Path, readTableSettings).GetValueSync(); if (result.IsSuccess()) { @@ -1198,16 +1198,16 @@ void TCommandIndexAddGlobal::Parse(TConfig& config) { int TCommandIndexAddGlobal::Run(TConfig& config) { NTable::TTableClient client(CreateDriver(config)); - auto columns = StringSplitter(Columns).Split(',').ToList(); - TVector dataColumns; + auto columns = StringSplitter(Columns).Split(',').ToList(); + std::vector dataColumns; if (DataColumns) { - dataColumns = StringSplitter(DataColumns).Split(',').ToList(); + dataColumns = StringSplitter(DataColumns).Split(',').ToList(); } auto settings = NTable::TAlterTableSettings() .AppendAddIndexes({NTable::TIndexDescription(IndexName, IndexType, columns, dataColumns)}); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto opResult = session.GetSession().AlterTableLong(Path, settings).GetValueSync(); ThrowOnError(opResult); PrintOperation(opResult, OutputFormat); @@ -1248,9 +1248,9 @@ int TCommandIndexDrop::Run(TConfig& config) { auto settings = NTable::TAlterTableSettings() .AppendDropIndexes({IndexName}); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -1286,9 +1286,9 @@ int TCommandIndexRename::Run(TConfig& config) { auto settings = NTable::TAlterTableSettings() .AppendRenameIndexes({IndexName, NewIndexName, Replace}); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -1322,9 +1322,9 @@ int TCommandAttributeAdd::Run(TConfig& config) { .AlterAttributes(Attributes); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -1359,9 +1359,9 @@ int TCommandAttributeDrop::Run(TConfig& config) { alterAttrs.EndAlterAttributes(); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -1431,9 +1431,9 @@ int TCommandTtlSet::Run(TConfig& config) { .EndAlterTtlSettings(); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -1462,9 +1462,9 @@ int TCommandTtlReset::Run(TConfig& config) { .EndAlterTtlSettings(); auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); + NStatusHelpers::ThrowOnErrorOrPrintIssues(session); auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index a0716f7dc322..041543abb3ef 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -5,9 +5,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include @@ -60,7 +60,7 @@ class TCommandCreateTable : public TTableCommand, public TCommandWithPath { private: TVector Columns; - TVector PrimaryKeys; + std::vector PrimaryKeys; TVector Indexes; TString PresetName; TString ExecutionPolicy; @@ -115,7 +115,7 @@ class TCommandExecuteQuery : public TTableCommand, TCommandQueryBase, TCommandWi template bool PrintQueryResponse(TIterator& result); - void PrintFlameGraph(const TMaybe& plan); + void PrintFlameGraph(const std::optional& plan); private: TString CollectStatsMode; @@ -135,7 +135,7 @@ class TCommandExplain : public TTableCommand, public TCommandWithOutput, TComman virtual int Run(TConfig& config) override; private: - static void SaveDiagnosticsToFile(const TString& diagnostics); + static void SaveDiagnosticsToFile(const std::string& diagnostics); bool PrintAst = false; TString QueryType; @@ -207,7 +207,7 @@ class TCommandIndexDrop : public TYdbCommand, public TCommandWithPath { virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; private: - TString IndexName; + std::string IndexName; }; class TCommandIndexRename : public TYdbCommand, public TCommandWithPath { @@ -229,7 +229,7 @@ class TCommandAttributeAdd : public TYdbCommand, public TCommandWithPath { virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; private: - THashMap Attributes; + std::unordered_map Attributes; }; class TCommandAttributeDrop : public TYdbCommand, public TCommandWithPath { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp index 265d99e33c59..a38151239a06 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include @@ -380,7 +380,7 @@ namespace NYdb::NConsoleClient { } auto status = topicClient.CreateTopic(TopicName, settings).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); return EXIT_SUCCESS; } @@ -484,14 +484,14 @@ namespace NYdb::NConsoleClient { NYdb::NTopic::TTopicClient topicClient(driver); auto topicDescription = topicClient.DescribeTopic(TopicName, {}).GetValueSync(); - ThrowOnError(topicDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(topicDescription); auto describeResult = topicClient.DescribeTopic(TopicName).GetValueSync(); - ThrowOnError(describeResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(describeResult); auto settings = PrepareAlterSettings(describeResult); auto result = topicClient.AlterTopic(TopicName, settings).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; } @@ -515,11 +515,11 @@ namespace NYdb::NConsoleClient { NTopic::TTopicClient topicClient(driver); auto topicDescription = topicClient.DescribeTopic(TopicName, {}).GetValueSync(); - ThrowOnError(topicDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(topicDescription); auto settings = NYdb::NTopic::TDropTopicSettings(); TStatus status = topicClient.DropTopic(TopicName, settings).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); return EXIT_SUCCESS; } @@ -570,7 +570,7 @@ namespace NYdb::NConsoleClient { NTopic::TTopicClient topicClient(driver); auto topicDescription = topicClient.DescribeTopic(TopicName, {}).GetValueSync(); - ThrowOnError(topicDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(topicDescription); NYdb::NTopic::TAlterTopicSettings readRuleSettings = NYdb::NTopic::TAlterTopicSettings(); NYdb::NTopic::TConsumerSettings consumerSettings(readRuleSettings); @@ -588,7 +588,7 @@ namespace NYdb::NConsoleClient { readRuleSettings.AppendAddConsumers(consumerSettings); TStatus status = topicClient.AlterTopic(TopicName, readRuleSettings).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); return EXIT_SUCCESS; } @@ -615,7 +615,7 @@ namespace NYdb::NConsoleClient { NYdb::NTopic::TTopicClient topicClient(driver); auto topicDescription = topicClient.DescribeTopic(TopicName, {}).GetValueSync(); - ThrowOnError(topicDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(topicDescription); auto consumers = topicDescription.GetTopicDescription().GetConsumers(); if (!std::any_of(consumers.begin(), consumers.end(), [&](const auto& consumer) { return consumer.GetConsumerName() == ConsumerName_; })) @@ -628,7 +628,7 @@ namespace NYdb::NConsoleClient { removeReadRuleSettings.AppendDropConsumers(ConsumerName_); TStatus status = topicClient.AlterTopic(TopicName, removeReadRuleSettings).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); return EXIT_SUCCESS; } @@ -659,7 +659,7 @@ namespace NYdb::NConsoleClient { NYdb::NTopic::TTopicClient topicClient(driver); auto consumerDescription = topicClient.DescribeConsumer(TopicName, ConsumerName_, NYdb::NTopic::TDescribeConsumerSettings().IncludeStats(ShowPartitionStats_)).GetValueSync(); - ThrowOnError(consumerDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(consumerDescription); return PrintDescription(this, OutputFormat, consumerDescription.GetConsumerDescription(), &TCommandTopicConsumerDescribe::PrintPrettyResult); } @@ -700,7 +700,7 @@ namespace NYdb::NConsoleClient { NYdb::NTopic::TTopicClient topicClient(driver); auto topicDescription = topicClient.DescribeTopic(TopicName, {}).GetValueSync(); - ThrowOnError(topicDescription); + NStatusHelpers::ThrowOnErrorOrPrintIssues(topicDescription); auto consumers = topicDescription.GetTopicDescription().GetConsumers(); if (!std::any_of(consumers.begin(), consumers.end(), [&](const auto& consumer) { return consumer.GetConsumerName() == ConsumerName_; })) @@ -710,7 +710,7 @@ namespace NYdb::NConsoleClient { } TStatus status = topicClient.CommitOffset(TopicName, PartitionId_, ConsumerName_, Offset_).GetValueSync(); - ThrowOnError(status); + NStatusHelpers::ThrowOnErrorOrPrintIssues(status); return EXIT_SUCCESS; } @@ -901,7 +901,7 @@ namespace NYdb::NConsoleClient { ValidateConfig(); auto driver = - std::make_unique(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)))); + std::make_unique(CreateDriver(config, std::unique_ptr(CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)).Release()))); NTopic::TTopicClient topicClient(*driver); auto readSession = topicClient.CreateReadSession(PrepareReadSessionSettings()); @@ -1032,7 +1032,7 @@ namespace NYdb::NConsoleClient { SetInterruptHandlers(); auto driver = - std::make_unique(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)))); + std::make_unique(CreateDriver(config, std::unique_ptr(CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)).Release()))); NTopic::TTopicClient topicClient(*driver); { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h index a87d8486490a..7b4d37e3709f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h @@ -5,9 +5,9 @@ #include #include -#include +#include -#include +#include namespace NYdb::NConsoleClient { TString PrepareAllowedCodecsDescription(const TString& descriptionPrefix, const TVector& codecs); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_sql.cpp b/ydb/public/lib/ydb_cli/commands/ydb_sql.cpp index abfd8377149c..b11f65a2ef3f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_sql.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_sql.cpp @@ -2,13 +2,13 @@ #include #include -#include +#include #include #include #include #include #include -#include +#include #include #include @@ -158,7 +158,7 @@ int TCommandSql::RunCommand(TConfig& config) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); int printResult = PrintResponse(result); if (printResult != EXIT_SUCCESS) { return printResult; @@ -173,16 +173,16 @@ int TCommandSql::RunCommand(TConfig& config) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return PrintResponse(result); } return EXIT_SUCCESS; } int TCommandSql::PrintResponse(NQuery::TExecuteQueryIterator& result) { - TMaybe stats; - TMaybe plan; - TMaybe ast; + std::optional stats; + std::optional plan; + std::optional ast; { TResultSetPrinter printer(OutputFormat, &IsInterrupted); @@ -196,7 +196,7 @@ int TCommandSql::PrintResponse(NQuery::TExecuteQueryIterator& result) { printer.Print(streamPart.GetResultSet()); } - if (!streamPart.GetStats().Empty()) { + if (streamPart.GetStats().has_value()) { const auto& queryStats = *streamPart.GetStats(); stats = queryStats.ToString(); ast = queryStats.GetAst(); @@ -232,7 +232,7 @@ int TCommandSql::PrintResponse(NQuery::TExecuteQueryIterator& result) { && (ExplainMode || ExplainAnalyzeMode) ? EDataFormat::PrettyTable : OutputFormat; TQueryPlanPrinter queryPlanPrinter(format, /* show actual costs */ !ExplainMode); - queryPlanPrinter.Print(*plan); + queryPlanPrinter.Print(TString{*plan}); } if (IsInterrupted()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_sql.h b/ydb/public/lib/ydb_cli/commands/ydb_sql.h index 19dccfd01e61..fc86a9dc3bc4 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_sql.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_sql.h @@ -3,7 +3,7 @@ #include "ydb_command.h" #include "ydb_common.h" -#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 1cf3cad99a8d..87f17b751a2c 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -1,6 +1,6 @@ #include "ydb_storage_config.h" -#include +#include #include #include @@ -47,15 +47,15 @@ int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); auto result = client.FetchStorageConfig().GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnError(result); auto cfg = result.GetConfig(); - if (!cfg) { + if (cfg.empty()) { Cerr << "YAML config is absent on this cluster." << Endl; return EXIT_FAILURE; } - Cout << WrapYaml(cfg); + Cout << WrapYaml(TString{cfg}); return EXIT_SUCCESS; } @@ -93,7 +93,7 @@ int TCommandStorageConfigReplace::Run(TConfig& config) { return client.ReplaceStorageConfig(StorageConfig).GetValueSync(); }; auto status = exec(); - ThrowOnError(status); + NStatusHelpers::ThrowOnError(status); if (!status.GetIssues()) { Cout << status << Endl; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index ad148c53b5e7..c24e8c639030 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -1,7 +1,7 @@ #include "ydb_tools.h" #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H #include @@ -110,7 +110,7 @@ int TCommandDump::Run(TConfig& config) { log->SetFormatter(GetPrefixLogFormatter("")); NDump::TClient client(CreateDriver(config), std::move(log)); - ThrowOnError(client.Dump(Path, FilePath, settings)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Dump(Path, FilePath, settings)); return EXIT_SUCCESS; } @@ -241,7 +241,7 @@ int TCommandRestore::Run(TConfig& config) { log->SetFormatter(GetPrefixLogFormatter("")); NDump::TClient client(CreateDriver(config), std::move(log)); - ThrowOnError(client.Restore(FilePath, Path, settings)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.Restore(FilePath, Path, settings)); return EXIT_SUCCESS; } @@ -292,7 +292,7 @@ int TCommandCopy::Run(TConfig& config) { for (auto& item : Items) { copyItems.emplace_back(item.Source, item.Destination); } - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( GetSession(config).CopyTables( copyItems, FillSettings(NTable::TCopyTablesSettings()) @@ -385,7 +385,7 @@ int TCommandRename::Run(TConfig& config) { renameItems.back().SetReplaceDestination(); } } - ThrowOnError( + NStatusHelpers::ThrowOnErrorOrPrintIssues( GetSession(config).RenameTables( renameItems, FillSettings(NTable::TRenameTablesSettings()) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 8cc76d67ac15..5d173edab4dd 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include @@ -400,7 +400,7 @@ void TWorkloadCommandBase::CleanTables(NYdbWorkload::IWorkloadQueryGenerator& wo if (DryRun) { Cout << "Remove " << fullPath << Endl; } else { - ThrowOnError(RemovePathRecursive(*SchemeClient, *TableClient, TopicClient.Get(), QueryClient.Get(), fullPath, ERecursiveRemovePrompt::Never, settings)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(RemovePathRecursive(*SchemeClient, *TableClient, TopicClient.Get(), QueryClient.Get(), fullPath, ERecursiveRemovePrompt::Never, settings)); } Cout << "Remove path " << path << "...Ok" << Endl; } @@ -466,7 +466,7 @@ int TWorkloadCommandInit::DoRun(NYdbWorkload::IWorkloadQueryGenerator& workloadG auto result = TableClient->RetryOperationSync([ddlQueries](NTable::TSession session) { return session.ExecuteSchemeQuery(ddlQueries.c_str()).GetValueSync(); }); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); } Cout << "Init tables ...Ok" << Endl; } @@ -511,7 +511,7 @@ int TWorkloadCommandClean::DoRun(NYdbWorkload::IWorkloadQueryGenerator& workload NTable::TSession TWorkloadCommandInit::GetSession() { NTable::TCreateSessionResult result = TableClient->GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return result.GetSession(); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.h b/ydb/public/lib/ydb_cli/commands/ydb_workload.h index 09f90159a6ae..87b3fe8fcc2e 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.h @@ -2,8 +2,8 @@ #include #include -#include -#include +#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload_import.cpp index 6ab2976cdf24..e1845b8945d7 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload_import.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload_import.cpp @@ -90,7 +90,7 @@ class TWorkloadCommandImport::TUploadCommand::TDbWriter: public IWriter { TAsyncStatus WriteDataPortion(NYdbWorkload::IBulkDataGenerator::TDataPortionPtr portion) override { if (std::holds_alternative(portion->MutableData())) { - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } if (auto* value = std::get_if(&portion->MutableData())) { return Owner.TableClient->BulkUpsert(portion->GetTable(), std::move(*value)).Apply(ConvertResult); @@ -116,11 +116,11 @@ class TWorkloadCommandImport::TUploadCommand::TDbWriter: public IWriter { } auto arrowCsv = NKikimr::NFormats::TArrowCSVTable::Create(param.Columns, true); if (!arrowCsv.ok()) { - return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues({NYql::TIssue(arrowCsv.status().ToString())}))); + return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues({NYdb::NIssue::TIssue(arrowCsv.status().ToString())}))); } Ydb::Formats::CsvSettings csvSettings; if (!csvSettings.ParseFromString(value->FormatString)) { - return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues({NYql::TIssue("Invalid format string")}))); + return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues({NYdb::NIssue::TIssue("Invalid format string")}))); }; auto writeOptions = arrow::ipc::IpcWriteOptions::Defaults(); @@ -129,7 +129,7 @@ class TWorkloadCommandImport::TUploadCommand::TDbWriter: public IWriter { TString error; if (auto batch = arrowCsv->ReadSingleBatch(value->Data, csvSettings, error)) { if (error) { - return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues({NYql::TIssue(error)}))); + return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues({NYdb::NIssue::TIssue(error)}))); } return Owner.TableClient->RetryOperation([ parquet = NYdb_cli::NArrow::SerializeBatch(batch, writeOptions), @@ -140,9 +140,9 @@ class TWorkloadCommandImport::TUploadCommand::TDbWriter: public IWriter { }, RetrySettings); } if (error) { - return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues({NYql::TIssue(error)}))); + return NThreading::MakeFuture(TStatus(EStatus::INTERNAL_ERROR, NYdb::NIssue::TIssues({NYdb::NIssue::TIssue(error)}))); } - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } static TStatus ConvertResult(const NTable::TAsyncBulkUpsertResult& result) { @@ -150,8 +150,8 @@ class TWorkloadCommandImport::TUploadCommand::TDbWriter: public IWriter { } struct TArrowCSVParams { - TStatus Status = TStatus(EStatus::SUCCESS, NYql::TIssues()); - TVector Columns; + TStatus Status = TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()); + std::vector Columns; }; TArrowCSVParams GetCSVParams(const TString& table) { @@ -193,7 +193,7 @@ class TWorkloadCommandImport::TUploadCommand::TFileWriter: public IWriter { TAsyncStatus WriteDataPortion(NYdbWorkload::IBulkDataGenerator::TDataPortionPtr portion) override { if (std::holds_alternative(portion->MutableData())) { - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } if (auto* value = std::get_if(&portion->MutableData())) { return NThreading::MakeErrorFuture(std::make_exception_ptr(yexception() << "Not implemented")); @@ -207,13 +207,13 @@ class TWorkloadCommandImport::TUploadCommand::TFileWriter: public IWriter { toWrite.ReadLine(firstLine); } out->Write(toWrite); - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } if (auto* value = std::get_if(&portion->MutableData())) { auto g = Guard(Lock); auto [out, created] = GetOutput(portion->GetTable()); out->Write(value->Data); - return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYql::TIssues())); + return NThreading::MakeFuture(TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues())); } Y_FAIL_S("Invalid data portion"); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp index 402609b2bbbe..e86a812832a1 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp @@ -109,7 +109,7 @@ int TCommandYql::RunCommand(TConfig& config, const TString& script) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); if (!PrintResponse(result)) { return EXIT_FAILURE; } @@ -121,7 +121,7 @@ int TCommandYql::RunCommand(TConfig& config, const TString& script) { ); auto result = asyncResult.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); if (!PrintResponse(result)) { return EXIT_FAILURE; } @@ -131,7 +131,7 @@ int TCommandYql::RunCommand(TConfig& config, const TString& script) { bool TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { TStringStream statsStr; - TMaybe fullStats; + std::optional fullStats; { ui32 currentIndex = 0; TResultSetPrinter printer(OutputFormat, &IsInterrupted); @@ -173,11 +173,11 @@ bool TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { Cout << Endl << "Full statistics:" << Endl; TQueryPlanPrinter queryPlanPrinter(OutputFormat, /* analyzeMode */ true); - queryPlanPrinter.Print(*fullStats); + queryPlanPrinter.Print(TString{*fullStats}); if (FlameGraphPath) { try { - NKikimr::NVisual::GenerateFlameGraphSvg(*FlameGraphPath, *fullStats); + NKikimr::NVisual::GenerateFlameGraphSvg(*FlameGraphPath, TString{*fullStats}); Cout << "Resource usage flame graph is successfully saved to " << *FlameGraphPath << Endl; } catch (const yexception& ex) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.h b/ydb/public/lib/ydb_cli/commands/ydb_yql.h index 10e95dbd506b..aba74ef77b7c 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.h @@ -3,7 +3,7 @@ #include "ydb_command.h" #include "ydb_common.h" -#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/common/aws.cpp b/ydb/public/lib/ydb_cli/common/aws.cpp index 7bc8ff8750a5..05529ba7db0f 100644 --- a/ydb/public/lib/ydb_cli/common/aws.cpp +++ b/ydb/public/lib/ydb_cli/common/aws.cpp @@ -1,6 +1,6 @@ #include "aws.h" -#include +#include #if !defined(_win32_) #include diff --git a/ydb/public/lib/ydb_cli/common/aws.h b/ydb/public/lib/ydb_cli/common/aws.h index a0dbbb49514e..814b997d6a64 100644 --- a/ydb/public/lib/ydb_cli/common/aws.h +++ b/ydb/public/lib/ydb_cli/common/aws.h @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NImport { +namespace NYdb::inline V3::NImport { struct TImportFromS3Settings; } diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index d9aadcf85dad..c1e474275be2 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -2,8 +2,8 @@ #include "common.h" -#include -#include +#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/common/csv_parser.cpp b/ydb/public/lib/ydb_cli/common/csv_parser.cpp index 819bab0fd074..9b99bbfa6d1f 100644 --- a/ydb/public/lib/ydb_cli/common/csv_parser.cpp +++ b/ydb/public/lib/ydb_cli/common/csv_parser.cpp @@ -98,7 +98,7 @@ class TCsvToYdbConverter { Builder.Yson(token); break; case EPrimitiveType::Uuid: - Builder.Uuid(token); + Builder.Uuid(TUuidValue{token}); break; case EPrimitiveType::Float: Builder.Float(GetArithmetic(token)); @@ -183,11 +183,11 @@ class TCsvToYdbConverter { void BuildValue(const TStringBuf& token) { switch (Parser.GetKind()) { case TTypeParser::ETypeKind::Primitive: { - BuildPrimitive(TString(token)); + BuildPrimitive(std::string{token}); break; } case TTypeParser::ETypeKind::Decimal: { - Builder.Decimal(TDecimalValue(TString(token), Parser.GetDecimal().Precision, Parser.GetDecimal().Scale)); + Builder.Decimal(TDecimalValue(std::string(token), Parser.GetDecimal().Precision, Parser.GetDecimal().Scale)); break; } case TTypeParser::ETypeKind::Optional: { @@ -418,7 +418,7 @@ class TCsvToYdbConverter { TCsvParseException FormatError(const std::exception& inputError, const TCsvParser::TParseMetadata& meta, - std::optional columnName = {}) { + const std::optional& columnName = {}) { auto outputError = TCsvParseException() << "Error during CSV parsing"; if (meta.Line.has_value()) { outputError << " in line " << meta.Line.value(); @@ -437,7 +437,7 @@ TValue FieldToValue(TTypeParser& parser, const TStringBuf& token, const std::optional& nullValue, const TCsvParser::TParseMetadata& meta, - const TString& columnName) { + const std::string& columnName) { try { TCsvToYdbConverter converter(parser, nullValue); return converter.Convert(token); @@ -469,7 +469,7 @@ TStringBuf Consume(NCsvFormat::CsvSplitter& splitter, } TCsvParser::TCsvParser(TString&& headerRow, const char delimeter, const std::optional& nullValue, - const std::map* destinationTypes, + const std::map* destinationTypes, const std::map* paramSources) : HeaderRow(std::move(headerRow)) , Delimeter(delimeter) @@ -482,7 +482,7 @@ TCsvParser::TCsvParser(TString&& headerRow, const char delimeter, const std::opt } TCsvParser::TCsvParser(TVector&& header, const char delimeter, const std::optional& nullValue, - const std::map* destinationTypes, + const std::map* destinationTypes, const std::map* paramSources) : Header(std::move(header)) , Delimeter(delimeter) @@ -525,7 +525,7 @@ void TCsvParser::BuildParams(TString& data, TParamsBuilder& builder, const TPars void TCsvParser::BuildValue(TString& data, TValueBuilder& builder, const TType& type, const TParseMetadata& meta) const { NCsvFormat::CsvSplitter splitter(data, Delimeter); auto headerIt = Header.cbegin(); - std::map fields; + std::map fields; do { if (headerIt == Header.cend()) { throw FormatError(yexception() << "Header contains less fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"", meta); @@ -543,7 +543,7 @@ void TCsvParser::BuildValue(TString& data, TValueBuilder& builder, const TType& TTypeParser parser(type); parser.OpenStruct(); while (parser.TryNextMember()) { - TString name = parser.GetMemberName(); + std::string name = parser.GetMemberName(); if (name == "__ydb_skip_column_name") { continue; } diff --git a/ydb/public/lib/ydb_cli/common/csv_parser.h b/ydb/public/lib/ydb_cli/common/csv_parser.h index 301d4cb601c8..5fea82930bd2 100644 --- a/ydb/public/lib/ydb_cli/common/csv_parser.h +++ b/ydb/public/lib/ydb_cli/common/csv_parser.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include @@ -58,10 +58,10 @@ class TCsvParser { ~TCsvParser() = default; TCsvParser(TString&& headerRow, const char delimeter, const std::optional& nullValue, - const std::map* destinationTypes = nullptr, + const std::map* destinationTypes = nullptr, const std::map* paramSources = nullptr); TCsvParser(TVector&& header, const char delimeter, const std::optional& nullValue, - const std::map* destinationTypes = nullptr, + const std::map* destinationTypes = nullptr, const std::map* paramSources = nullptr); void BuildParams(TString& data, TParamsBuilder& builder, const TParseMetadata& meta) const; @@ -79,7 +79,7 @@ class TCsvParser { std::optional NullValue; // Types of destination table or query parameters // Column name -> column type - const std::map* DestinationTypes; + const std::map* DestinationTypes; const std::map* ParamSources; // Type of a single row in resulting TValue. // Column order according to the header, though can have less elements than the Header diff --git a/ydb/public/lib/ydb_cli/common/csv_parser_ut.cpp b/ydb/public/lib/ydb_cli/common/csv_parser_ut.cpp index 847902d1c343..ffe3de88bf27 100644 --- a/ydb/public/lib/ydb_cli/common/csv_parser_ut.cpp +++ b/ydb/public/lib/ydb_cli/common/csv_parser_ut.cpp @@ -17,7 +17,7 @@ Y_UNIT_TEST_SUITE(YdbCliCsvParserTests) { } void CommonTestParams(TString&& header, TString&& data, const std::map& estimatedResult) { - std::map paramTypes; + std::map paramTypes; for (const auto& [name, value] : estimatedResult) { paramTypes.insert({name, value.GetType()}); } @@ -35,7 +35,7 @@ Y_UNIT_TEST_SUITE(YdbCliCsvParserTests) { } void CommonTestValue(TString&& header, TString&& data, const TValue& estimatedResult) { - std::map paramTypes; + std::map paramTypes; for (auto member : estimatedResult.GetType().GetProto().struct_type().members()) { paramTypes.insert({member.name(), member.type()}); } @@ -47,7 +47,7 @@ Y_UNIT_TEST_SUITE(YdbCliCsvParserTests) { } void CommonTestBuildList(TString&& header, std::vector&& data, const TValue& estimatedResult) { - std::map columnTypes; + std::map columnTypes; for (auto member : estimatedResult.GetType().GetProto().list_type().item().struct_type().members()) { columnTypes.insert({member.name(), member.type()}); } @@ -209,7 +209,6 @@ Y_UNIT_TEST_SUITE(YdbCliCsvParserTests) { CommonTestValue("name", "550e8400-e29b-41d4-a716-446655440000", MakeStruct("name", TValueBuilder().Uuid(TUuidValue("550e8400-e29b-41d4-a716-446655440000")).Build())); CommonTestValue("name", "\"{\"\"a\"\":10, \"\"b\"\":\"\"string\"\"}\"", MakeStruct("name", TValueBuilder().Json("{\"a\":10, \"b\":\"string\"}").Build())); CommonTestValue("name", "строка", MakeStruct("name", TValueBuilder().OptionalUtf8("строка").Build())); - CommonTestValue("name", "\"\"", MakeStruct("name", TValueBuilder().OptionalUtf8({}).Build())); CommonTestValue("name", "данные", MakeStruct("name", TValueBuilder().Pg(TPgValue(TPgValue::VK_TEXT, "данные", TPgType("some_type"))).Build())); } @@ -313,7 +312,7 @@ Y_UNIT_TEST_SUITE(YdbCliCsvParserTests) { } Y_UNIT_TEST(ShuffledColumns) { - std::map tableColumnTypes = { + std::map tableColumnTypes = { {"col1", TTypeBuilder().Primitive(EPrimitiveType::Utf8).Build()}, {"col2", TTypeBuilder().BeginOptional().Primitive(EPrimitiveType::Int64).EndOptional().Build()}, {"col3", TTypeBuilder().Primitive(EPrimitiveType::Bool).Build()}, diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index d5de4673865d..20ef10336670 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -753,11 +753,11 @@ void TResultSetPrinter::EndLineBeforeNextResult() { } void TResultSetPrinter::PrintPretty(const TResultSet& resultSet) { - const TVector& columns = resultSet.GetColumnsMeta(); + const std::vector& columns = resultSet.GetColumnsMeta(); TResultSetParser parser(resultSet); TVector columnNames; for (const auto& column : columns) { - columnNames.push_back(column.Name); + columnNames.push_back(TString{column.Name}); } TPrettyTableConfig tableConfig; @@ -802,7 +802,7 @@ void TResultSetPrinter::PrintJsonArray(const TResultSet& resultSet, EBinaryStrin } void TResultSetPrinter::PrintCsv(const TResultSet& resultSet, const char* delim) { - const TVector& columns = resultSet.GetColumnsMeta(); + const std::vector& columns = resultSet.GetColumnsMeta(); TResultSetParser parser(resultSet); if (Settings.IsCsvWithHeader()) { for (ui32 i = 0; i < columns.size(); ++i) { diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index 2a3f8f9254fe..5452ed7a1a0e 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -5,8 +5,8 @@ #include "pretty_table.h" #include -#include -#include +#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/common/interactive.h b/ydb/public/lib/ydb_cli/common/interactive.h index c0bfc68dcb6e..b041006b0cd6 100644 --- a/ydb/public/lib/ydb_cli/common/interactive.h +++ b/ydb/public/lib/ydb_cli/common/interactive.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/interruptible.h b/ydb/public/lib/ydb_cli/common/interruptible.h index 84647316c503..1d831b73bf61 100644 --- a/ydb/public/lib/ydb_cli/common/interruptible.h +++ b/ydb/public/lib/ydb_cli/common/interruptible.h @@ -3,8 +3,8 @@ #include "command.h" #include "formats.h" -#include -#include +#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/parameters.cpp b/ydb/public/lib/ydb_cli/common/parameters.cpp index ae843a3703f1..aa1413744630 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.cpp +++ b/ydb/public/lib/ydb_cli/common/parameters.cpp @@ -343,7 +343,7 @@ void TCommandWithParameters::GetParamTypes(const TDriver& driver, const TString& queryText, explainSettings ).GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); ParamTypes = result.GetParameterTypes(); } diff --git a/ydb/public/lib/ydb_cli/common/parameters.h b/ydb/public/lib/ydb_cli/common/parameters.h index c426f24727fe..b33886131b1c 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.h +++ b/ydb/public/lib/ydb_cli/common/parameters.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include namespace NYdb { @@ -46,7 +46,7 @@ class TCommandWithParameters : public TCommandWithExamples, public TCommandWithI TMaybe ReadData(); - std::map ParamTypes; + std::map ParamTypes; TVector Header; TString Columns; THolder InputFileHolder; diff --git a/ydb/public/lib/ydb_cli/common/print_operation.cpp b/ydb/public/lib/ydb_cli/common/print_operation.cpp index 659d6004d9d1..3b1dddd07986 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.cpp +++ b/ydb/public/lib/ydb_cli/common/print_operation.cpp @@ -1,8 +1,8 @@ #include "print_operation.h" #include "pretty_table.h" -#include -#include +#include +#include #include #include @@ -24,7 +24,7 @@ namespace { auto& row = table.AddRow(); row - .Column(0, ProtoToString(operation.Id())) + .Column(0, operation.Id().ToString()) .Column(1, operation.Ready() ? "true" : "false") .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())); @@ -61,7 +61,7 @@ namespace { return result; } - if (!metadata.ItemsProgress) { + if (metadata.ItemsProgress.empty()) { return result; } @@ -94,11 +94,11 @@ namespace { auto& row = table.AddRow(); row - .Column(0, ProtoToString(operation.Id())) + .Column(0, operation.Id().ToString()) .Column(1, operation.Ready() ? "true" : "false") .Column(2, status.GetStatus()) .Column(3, PrintProgress(metadata)) - .Column(4, TStringBuilder() << settings.Host_ << ":" << settings.Port_.GetOrElse(80)); + .Column(4, TStringBuilder() << settings.Host_ << ":" << settings.Port_.value_or(80)); TStringBuilder freeText; @@ -117,11 +117,11 @@ namespace { } if (settings.Description_) { - freeText << "Description: " << settings.Description_.GetRef() << Endl; + freeText << "Description: " << settings.Description_.value() << Endl; } if (settings.NumberOfRetries_) { - freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; + freeText << "Number of retries: " << settings.NumberOfRetries_.value() << Endl; } freeText << "TypeV3: " << (settings.UseTypeV3_ ? "true" : "false") << Endl; @@ -154,7 +154,7 @@ namespace { auto& row = table.AddRow(); row - .Column(0, ProtoToString(operation.Id())) + .Column(0, operation.Id().ToString()) .Column(1, operation.Ready() ? "true" : "false") .Column(2, status.GetStatus()) .Column(3, PrintProgress(metadata)) @@ -172,11 +172,11 @@ namespace { if constexpr (std::is_same_v) { if (settings.NoACL_) { - freeText << "NoACL: " << settings.NoACL_ << Endl; + freeText << "NoACL: " << *settings.NoACL_ << Endl; } if (settings.SkipChecksumValidation_) { - freeText << "SkipChecksumValidation: " << settings.SkipChecksumValidation_ << Endl; + freeText << "SkipChecksumValidation: " << *settings.SkipChecksumValidation_ << Endl; } } @@ -195,11 +195,11 @@ namespace { } if (settings.Description_) { - freeText << "Description: " << settings.Description_.GetRef() << Endl; + freeText << "Description: " << settings.Description_.value() << Endl; } if (settings.NumberOfRetries_) { - freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; + freeText << "Number of retries: " << settings.NumberOfRetries_.value() << Endl; } if (!operation.CreatedBy().empty()) { @@ -246,7 +246,7 @@ namespace { auto& row = table.AddRow(); row - .Column(0, ProtoToString(operation.Id())) + .Column(0, operation.Id().ToString()) .Column(1, operation.Ready() ? "true" : "false") .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())) .Column(3, metadata.State) @@ -277,7 +277,7 @@ namespace { auto& row = table.AddRow(); row - .Column(0, ProtoToString(operation.Id())) + .Column(0, operation.Id().ToString()) .Column(1, operation.Ready() ? "true" : "false") .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())) .Column(3, metadata.ExecutionId) @@ -327,14 +327,14 @@ namespace { switch (format) { case EDataFormat::Default: case EDataFormat::Pretty: - if (operations.GetList()) { + if (!operations.GetList().empty()) { auto table = MakeTable(operations.GetList().front()); for (const auto& operation : operations.GetList()) { PrettyPrint(operation, table); } Cout << table; } - if (operations.NextPageToken()) { + if (!operations.NextPageToken().empty()) { Cout << Endl << "Next page token: " << operations.NextPageToken() << Endl; } break; diff --git a/ydb/public/lib/ydb_cli/common/print_operation.h b/ydb/public/lib/ydb_cli/common/print_operation.h index 26fbd75aebab..64329c7c7f9d 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.h +++ b/ydb/public/lib/ydb_cli/common/print_operation.h @@ -1,11 +1,11 @@ #pragma once #include "formats.h" -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/print_utils.h b/ydb/public/lib/ydb_cli/common/print_utils.h index dfd73ce8cbe4..a67d0a190264 100644 --- a/ydb/public/lib/ydb_cli/common/print_utils.h +++ b/ydb/public/lib/ydb_cli/common/print_utils.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace NYdb { diff --git a/ydb/public/lib/ydb_cli/common/query_stats.h b/ydb/public/lib/ydb_cli/common/query_stats.h index 4b19d367b17f..26d86e0c078b 100644 --- a/ydb/public/lib/ydb_cli/common/query_stats.h +++ b/ydb/public/lib/ydb_cli/common/query_stats.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/recursive_list.h b/ydb/public/lib/ydb_cli/common/recursive_list.h index 7168a962f7f2..c611a917d99d 100644 --- a/ydb/public/lib/ydb_cli/common/recursive_list.h +++ b/ydb/public/lib/ydb_cli/common/recursive_list.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/ydb/public/lib/ydb_cli/common/recursive_remove.cpp b/ydb/public/lib/ydb_cli/common/recursive_remove.cpp index 0d25cfcd5a6c..38a49c68ba14 100644 --- a/ydb/public/lib/ydb_cli/common/recursive_remove.cpp +++ b/ydb/public/lib/ydb_cli/common/recursive_remove.cpp @@ -70,9 +70,9 @@ TStatus RemoveTopic(TTopicClient& client, const TString& path, const TDropTopicS }); } -NYql::TIssues MakeIssues(const TString& error) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); +NYdb::NIssue::TIssues MakeIssues(const TString& error) { + NYdb::NIssue::TIssues issues; + issues.AddIssue(NYdb::NIssue::TIssue(error)); return issues; } @@ -179,7 +179,7 @@ TStatus RemoveDirectoryRecursive( // output order is: Root, Recursive(children)... // we need to reverse it to delete recursively for (auto it = recursiveListResult.Entries.rbegin(); it != recursiveListResult.Entries.rend(); ++it) { - if (auto result = Remove(schemeClient, tableClient, topicClient, queryClient, it->Type, it->Name, prompt, settings); !result.IsSuccess()) { + if (auto result = Remove(schemeClient, tableClient, topicClient, queryClient, it->Type, TString{it->Name}, prompt, settings); !result.IsSuccess()) { return result; } if (createProgressBar) { diff --git a/ydb/public/lib/ydb_cli/common/recursive_remove.h b/ydb/public/lib/ydb_cli/common/recursive_remove.h index a1de1968655e..3f0f2dfe157a 100644 --- a/ydb/public/lib/ydb_cli/common/recursive_remove.h +++ b/ydb/public/lib/ydb_cli/common/recursive_remove.h @@ -1,9 +1,9 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include namespace NYdb::NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/retry_func.h b/ydb/public/lib/ydb_cli/common/retry_func.h index 85a2fd7c8cfe..cfa65c8cc10e 100644 --- a/ydb/public/lib/ydb_cli/common/retry_func.h +++ b/ydb/public/lib/ydb_cli/common/retry_func.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/ydb/public/lib/ydb_cli/common/scheme_printers.cpp b/ydb/public/lib/ydb_cli/common/scheme_printers.cpp index 9175afc7e591..9fbf42d67ac2 100644 --- a/ydb/public/lib/ydb_cli/common/scheme_printers.cpp +++ b/ydb/public/lib/ydb_cli/common/scheme_printers.cpp @@ -30,7 +30,7 @@ NThreading::TFuture TSchemePrinterBase::PrintDirectoryRecursive(const TStr Settings.ListDirectorySettings ).Apply([this, fullPath, relativePath](const NScheme::TAsyncListDirectoryResult& resultFuture) { const auto& result = resultFuture.GetValueSync(); - ThrowOnError(result); + NStatusHelpers::ThrowOnErrorOrPrintIssues(result); if (relativePath || IsDirectoryLike(result.GetEntry())) { std::lock_guard g(Lock); @@ -65,13 +65,13 @@ NTable::TDescribeTableResult TSchemePrinterBase::DescribeTable(const TString& re NTable::TCreateSessionResult sessionResult = TableClient.GetSession( NTable::TCreateSessionSettings() ).GetValueSync(); - ThrowOnError(sessionResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(sessionResult); NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable( Settings.Path + (relativePath ? ("/" + relativePath) : ""), Settings.DescribeTableSettings ).GetValueSync(); - ThrowOnError(tableResult); + NStatusHelpers::ThrowOnErrorOrPrintIssues(tableResult); return tableResult; } @@ -83,7 +83,7 @@ void TDefaultSchemePrinter::PrintDirectory( const TString& relativePath, const NScheme::TListDirectoryResult& entryResult) { - TVector children = entryResult.GetChildren(); + std::vector children = entryResult.GetChildren(); NScheme::TSchemeEntry entry = entryResult.GetEntry(); if (Settings.Recursive) { @@ -140,7 +140,7 @@ void TTableSchemePrinter::PrintDirectory( // Do not print target directory itself if (!Settings.Recursive) { for (const auto& child : entryResult.GetChildren()) { - PrintEntry(child.Name, child); + PrintEntry(TString{child.Name}, child); } } } @@ -159,7 +159,7 @@ void TTableSchemePrinter::PrintTable(const TString& relativePath, const NScheme: auto tableDescription = tableResult.GetTableDescription(); // Empty relative path in case of a single non-directory object in Path - TString actualRelativePath = relativePath ? relativePath : entry.Name; + TString actualRelativePath = relativePath ? relativePath : TString{entry.Name}; Table.AddRow() .Column(0, EntryTypeToString(entry.Type)) @@ -173,7 +173,7 @@ void TTableSchemePrinter::PrintTable(const TString& relativePath, const NScheme: void TTableSchemePrinter::PrintOther(const TString& relativePath, const NScheme::TSchemeEntry& entry) { // Empty relative path in case of a single non-directory object in Path - TString actualRelativePath = relativePath ? relativePath : entry.Name; + TString actualRelativePath = relativePath ? relativePath : TString{entry.Name}; Table.AddRow() .Column(0, EntryTypeToString(entry.Type)) .Column(1, entry.Owner) @@ -210,7 +210,7 @@ void TJsonSchemePrinter::PrintDirectory(const TString& relativePath, const NSche NeedToCloseList = true; } else { for (const auto& child : entryResult.GetChildren()) { - PrintEntry(child.Name, child); + PrintEntry(TString{child.Name}, child); } Writer.EndList(); Cout << Writer.Str() << Endl; @@ -255,7 +255,7 @@ void TJsonSchemePrinter::PrintOther(const TString& relativePath, const NScheme:: void TJsonSchemePrinter::PrintCommonInfo(const TString& relativePath, const NScheme::TSchemeEntry& entry) { // Empty relative path in case of a single non-directory object in Path - TString actualRelativePath = relativePath ? relativePath : entry.Name; + TString actualRelativePath = relativePath ? relativePath : TString{entry.Name}; Writer.WriteKey("type"); Writer.WriteString(EntryTypeToString(entry.Type)); Writer.WriteKey("path"); diff --git a/ydb/public/lib/ydb_cli/common/scheme_printers.h b/ydb/public/lib/ydb_cli/common/scheme_printers.h index 9648a70a0321..562488764c71 100644 --- a/ydb/public/lib/ydb_cli/common/scheme_printers.h +++ b/ydb/public/lib/ydb_cli/common/scheme_printers.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/common/sys.cpp b/ydb/public/lib/ydb_cli/common/sys.cpp index d13edd492ed9..9878c2287053 100644 --- a/ydb/public/lib/ydb_cli/common/sys.cpp +++ b/ydb/public/lib/ydb_cli/common/sys.cpp @@ -1,6 +1,6 @@ #include "sys.h" -#include +#include namespace NYdb::NConsoleClient { @@ -9,9 +9,9 @@ bool IsSystemObject(const NScheme::TSchemeEntry& entry) { return false; } - return entry.Name.StartsWith("~") - || entry.Name.StartsWith(".sys") - || entry.Name.StartsWith(".metadata"); + return entry.Name.starts_with("~") + || entry.Name.starts_with(".sys") + || entry.Name.starts_with(".metadata"); } } diff --git a/ydb/public/lib/ydb_cli/common/sys.h b/ydb/public/lib/ydb_cli/common/sys.h index 4a4b19717b81..5d265603f5b4 100644 --- a/ydb/public/lib/ydb_cli/common/sys.h +++ b/ydb/public/lib/ydb_cli/common/sys.h @@ -2,9 +2,11 @@ namespace NYdb { +inline namespace V3 { namespace NScheme { struct TSchemeEntry; } +} namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp index eb53b145e169..ef2cd0f2dc6e 100644 --- a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp +++ b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp @@ -9,7 +9,7 @@ namespace NYdb { namespace NConsoleClient { -TAdaptiveTabbedTable::TAdaptiveTabbedTable(const TVector& entries) +TAdaptiveTabbedTable::TAdaptiveTabbedTable(const std::vector& entries) : Entries(entries) { CalculateColumns(); diff --git a/ydb/public/lib/ydb_cli/common/tabbed_table.h b/ydb/public/lib/ydb_cli/common/tabbed_table.h index 32e7daea4e61..bdfe7edf997f 100644 --- a/ydb/public/lib/ydb_cli/common/tabbed_table.h +++ b/ydb/public/lib/ydb_cli/common/tabbed_table.h @@ -1,13 +1,13 @@ #pragma once -#include +#include namespace NYdb { namespace NConsoleClient { class TAdaptiveTabbedTable { public: - TAdaptiveTabbedTable(const TVector& entries); + TAdaptiveTabbedTable(const std::vector& entries); void Print(IOutputStream& o) const; private: @@ -21,7 +21,7 @@ class TAdaptiveTabbedTable { void InitializeColumnInfo(size_t maxCols, size_t minColumnWidth); void CalculateColumns(); - const TVector& Entries; + const std::vector& Entries; TVector ColumnInfo; size_t ColumnCount; }; diff --git a/ydb/public/lib/ydb_cli/common/ya.make b/ydb/public/lib/ydb_cli/common/ya.make index 772e7996fa88..a4c70d4d653a 100644 --- a/ydb/public/lib/ydb_cli/common/ya.make +++ b/ydb/public/lib/ydb_cli/common/ya.make @@ -40,17 +40,17 @@ PEERDIR( library/cpp/yaml/as library/cpp/string_utils/csv ydb/public/lib/json_value - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_types - ydb/public/sdk/cpp/client/ydb_types/credentials - ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/types + ydb/public/sdk/cpp/src/client/types/credentials + ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange ydb/library/arrow_parquet ) diff --git a/ydb/public/lib/ydb_cli/dump/dump.cpp b/ydb/public/lib/ydb_cli/dump/dump.cpp index 7554f95c9743..c4acb28619bb 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.cpp +++ b/ydb/public/lib/ydb_cli/dump/dump.cpp @@ -2,7 +2,7 @@ #include "dump_impl.h" #include "restore_impl.h" -#include +#include #include diff --git a/ydb/public/lib/ydb_cli/dump/dump.h b/ydb/public/lib/ydb_cli/dump/dump.h index d2c80498bcce..1f85a882301b 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.h +++ b/ydb/public/lib/ydb_cli/dump/dump.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include @@ -12,7 +12,9 @@ class TLog; namespace NYdb { +inline namespace V3 { class TDriver; +} namespace NDump { diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp index 0325665c9f4e..816a9f5d446f 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp +++ b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp @@ -3,7 +3,7 @@ #include "restore_compat.h" #include -#include +#include #include #include #include @@ -22,6 +22,8 @@ #include #include +#include + namespace NYdb { namespace NDump { @@ -152,7 +154,7 @@ bool IsDatabase(TSchemeClient& client, const TString& path) { } bool RewriteTablePathPrefix(TString& query, TStringBuf backupRoot, TStringBuf restoreRoot, - bool restoreRootIsDatabase, NYql::TIssues& issues + bool restoreRootIsDatabase, NIssue::TIssues& issues ) { if (backupRoot == restoreRoot) { return true; @@ -292,7 +294,7 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP THashSet oldEntries; for (const auto& entry : oldDirectoryList.Entries) { - oldEntries.insert(entry.Name); + oldEntries.insert(TString{entry.Name}); } // restore @@ -350,9 +352,9 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP switch (entry.Type) { case ESchemeEntryType::Directory: { - auto result = RemoveDirectoryRecursive(SchemeClient, TableClient, nullptr, &QueryClient, fullPath, ERecursiveRemovePrompt::Never, {}, true, false); + auto result = NConsoleClient::RemoveDirectoryRecursive(SchemeClient, TableClient, nullptr, &QueryClient, TString{fullPath}, ERecursiveRemovePrompt::Never, {}, true, false); if (!result.IsSuccess()) { - LOG_E("Error removing directory: " << fullPath.Quote() << ": " << result.GetIssues().ToOneLineString()); + LOG_E("Error removing directory: " << TString{fullPath}.Quote() << ": " << result.GetIssues().ToOneLineString()); return restoreResult; } break; @@ -362,7 +364,7 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP return session.DropTable(path).GetValueSync(); }); if (!result.IsSuccess()) { - LOG_E("Error removing table: " << fullPath.Quote() << ": " << result.GetIssues().ToOneLineString()); + LOG_E("Error removing table: " << TString{fullPath}.Quote() << ": " << result.GetIssues().ToOneLineString()); return restoreResult; } break; @@ -374,13 +376,13 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP )", fullPath.c_str()), NQuery::TTxControl::NoTx()).ExtractValueSync(); }); if (!result.IsSuccess()) { - LOG_E("Error removing view: " << fullPath.Quote() << ": " << result.GetIssues().ToOneLineString()); + LOG_E("Error removing view: " << TString{fullPath}.Quote() << ": " << result.GetIssues().ToOneLineString()); return restoreResult; } break; } default: - LOG_E("Error removing unexpected object: " << fullPath.Quote()); + LOG_E("Error removing unexpected object: " << TString{fullPath}.Quote()); return restoreResult; } } @@ -483,7 +485,7 @@ TRestoreResult TRestoreClient::RestoreView( const auto backupRoot = GetBackupRoot(query); { - NYql::TIssues issues; + NIssue::TIssues issues; if (!RewriteTablePathPrefix(query, backupRoot, dbRestoreRoot, IsDatabase(SchemeClient, dbRestoreRoot), issues)) { // hard fail since we want to avoid silent fails with wrong table path prefixes return Result(dbPath, TStatus(EStatus::BAD_REQUEST, std::move(issues))); @@ -501,7 +503,7 @@ TRestoreResult TRestoreClient::RestoreView( constexpr TStringBuf pattern = R"(CREATE VIEW IF NOT EXISTS `\S+` )"; if (!re2::RE2::Replace(&query, pattern, std::format(R"(CREATE VIEW IF NOT EXISTS `{}` )", dbPath.c_str()))) { - NYql::TIssues issues; + NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Cannot restore a view from the file: " << createViewFile.GetPath().Quote() << ". Pattern: \"" << pattern << "\", was not found in the create view statement: " << query.Quote() ); @@ -628,7 +630,7 @@ TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDe return Result(dbPath, std::move(descResult)); } - auto unorderedColumns = [](const TVector& orderedColumns) { + auto unorderedColumns = [](const std::vector& orderedColumns) { THashMap result; for (const auto& column : orderedColumns) { result.emplace(column.Name, column); @@ -636,7 +638,7 @@ TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDe return result; }; - auto unorderedIndexes = [](const TVector& orderedIndexes) { + auto unorderedIndexes = [](const std::vector& orderedIndexes) { THashMap result; for (const auto& index : orderedIndexes) { result.emplace(index.GetIndexName(), index); @@ -844,7 +846,7 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl continue; } - LOG_D("Restore index " << index.GetIndexName().Quote() << " on " << dbPath.Quote()); + LOG_D("Restore index " << TString{index.GetIndexName()}.Quote() << " on " << dbPath.Quote()); TOperation::TOperationId buildIndexId; auto buildIndexStatus = TableClient.RetryOperationSync([&, &outId = buildIndexId](TSession session) { @@ -857,13 +859,13 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl }); if (!IsOperationStarted(buildIndexStatus)) { - LOG_E("Error building index " << index.GetIndexName().Quote() << " on " << dbPath.Quote()); + LOG_E("Error building index " << TString{index.GetIndexName()}.Quote() << " on " << dbPath.Quote()); return Result(dbPath, std::move(buildIndexStatus)); } auto waitForIndexBuildStatus = WaitForIndexBuild(OperationClient, buildIndexId); if (!waitForIndexBuildStatus.IsSuccess()) { - LOG_E("Error building index " << index.GetIndexName().Quote() << " on " << dbPath.Quote()); + LOG_E("Error building index " << TString{index.GetIndexName()}.Quote() << " on " << dbPath.Quote()); return Result(dbPath, std::move(waitForIndexBuildStatus)); } @@ -871,7 +873,7 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl return OperationClient.Forget(buildIndexId).GetValueSync(); }); if (!forgetStatus.IsSuccess()) { - LOG_E("Error building index " << index.GetIndexName().Quote() << " on " << dbPath.Quote()); + LOG_E("Error building index " << TString{index.GetIndexName()}.Quote() << " on " << dbPath.Quote()); return Result(dbPath, std::move(forgetStatus)); } } @@ -907,7 +909,7 @@ TRestoreResult TRestoreClient::RestoreChangefeeds(const TFsPath& fsPath, const T return RestoreConsumers(Join("/", dbPath, fsPath.GetName()), topicDesc.GetConsumers());; } -TRestoreResult TRestoreClient::RestoreConsumers(const TString& topicPath, const TVector& consumers) { +TRestoreResult TRestoreClient::RestoreConsumers(const TString& topicPath, const std::vector& consumers) { for (const auto& consumer : consumers) { auto createResult = TopicClient.AlterTopic(topicPath, NTopic::TAlterTopicSettings() @@ -918,9 +920,9 @@ TRestoreResult TRestoreClient::RestoreConsumers(const TString& topicPath, const .EndAddConsumer() ).GetValueSync(); if (createResult.IsSuccess()) { - LOG_D("Created consumer " << consumer.GetConsumerName().Quote() << " for " << topicPath.Quote()); + LOG_D("Created consumer " << TString{consumer.GetConsumerName()}.Quote() << " for " << topicPath.Quote()); } else { - LOG_E("Failed to create " << consumer.GetConsumerName().Quote() << " for " << topicPath.Quote()); + LOG_E("Failed to create " << TString{consumer.GetConsumerName()}.Quote() << " for " << topicPath.Quote()); return Result(topicPath, std::move(createResult)); } } diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.h b/ydb/public/lib/ydb_cli/dump/restore_impl.h index 7d7c6787b59e..fb64657aff1b 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_impl.h +++ b/ydb/public/lib/ydb_cli/dump/restore_impl.h @@ -2,12 +2,12 @@ #include "dump.h" -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -134,7 +134,7 @@ class TRestoreClient { TRestoreResult RestoreIndexes(const TString& dbPath, const NTable::TTableDescription& desc); TRestoreResult RestoreChangefeeds(const TFsPath& path, const TString& dbPath); TRestoreResult RestorePermissions(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, bool isAlreadyExisting); - TRestoreResult RestoreConsumers(const TString& topicPath, const TVector& consumers); + TRestoreResult RestoreConsumers(const TString& topicPath, const std::vector& consumers); THolder CreateDataWriter(const TString& dbPath, const TRestoreSettings& settings, const NTable::TTableDescription& desc, const TVector>& accumulators); diff --git a/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp b/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp index e28e9d64f648..e024511e7818 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp +++ b/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp @@ -94,7 +94,7 @@ class TValue { ui64 MemSize() const { switch (GetType()) { case EType::String: - return sizeof(Value) + std::get(Value).size(); + return sizeof(Value) + std::get(Value).size(); default: return sizeof(Value); } @@ -104,7 +104,7 @@ class TValue { std::variant< TInf, TNull, - TString, + std::string, bool, ui8, i32, @@ -351,7 +351,7 @@ class TKey: public TVector { using TSplitPoint = TKey; class TKeyBuilder { - static auto MakeKeyColumnIds(const TVector& keyColumns) { + static auto MakeKeyColumnIds(const std::vector& keyColumns) { THashMap keyColumnIds; for (ui32 i = 0; i < keyColumns.size(); ++i) { @@ -380,8 +380,8 @@ class TKeyBuilder { public: explicit TKeyBuilder( - const TVector& columns, - const TVector& keyColumns, + const std::vector& columns, + const std::vector& keyColumns, const std::shared_ptr& log) : Columns(columns) , KeyColumnIds(MakeKeyColumnIds(keyColumns)) @@ -418,7 +418,7 @@ class TKeyBuilder { } private: - const TVector Columns; + const std::vector Columns; const THashMap KeyColumnIds; const std::shared_ptr Log; @@ -517,7 +517,7 @@ class TTableRows { using TRows = TMap; using TRowsBy = TMap>, TGreater>; - static auto MakeSplitPoints(const TVector& keyRanges) { + static auto MakeSplitPoints(const std::vector& keyRanges) { Y_ENSURE(!keyRanges.empty()); TVector splitPoints; @@ -526,7 +526,7 @@ class TTableRows { while (++it != keyRanges.end()) { const auto& from = it->From(); - Y_ENSURE(from.Defined()); + Y_ENSURE(from.has_value()); Y_ENSURE(from->IsInclusive()); TValueParser parser(from->GetValue()); @@ -584,7 +584,7 @@ class TTableRows { } public: - explicit TTableRows(const TVector& keyRanges) + explicit TTableRows(const std::vector& keyRanges) : ByPartition(MakeEmptyRows(MakeSplitPoints(keyRanges))) , MemSize(0) { @@ -630,7 +630,7 @@ class TTableRows { } } - void Reshard(const TVector& keyRanges) { + void Reshard(const std::vector& keyRanges) { auto newByPartition = MakeEmptyRows(MakeSplitPoints(keyRanges)); for (auto& [_, rows] : ByPartition) { @@ -770,7 +770,7 @@ class TDataAccumulator: public NPrivate::IDataAccumulator { return batch; } - void Reshard(const TVector& keyRanges) { + void Reshard(const std::vector& keyRanges) { TGuard lock(Mutex); Rows.Reshard(keyRanges); } @@ -815,7 +815,7 @@ class TDataWriter: public NPrivate::IDataWriter { RequestLimiter.Use(1); - auto importResult = ImportClient.ImportData(Path, data, Settings).GetValueSync(); + auto importResult = ImportClient.ImportData(Path, TString{data}, Settings).GetValueSync(); if (importResult.IsSuccess()) { return true; diff --git a/ydb/public/lib/ydb_cli/dump/util/util.h b/ydb/public/lib/ydb_cli/dump/util/util.h index 4f9256aa3cbb..71c60b749dfa 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.h +++ b/ydb/public/lib/ydb_cli/dump/util/util.h @@ -1,27 +1,27 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include namespace NYdb::NDump { -inline void AddPath(NYql::TIssues& issues, const TString& path) { - issues.AddIssue(NYql::TIssue(TStringBuilder() << "Path: " << path) - .SetCode(NYql::DEFAULT_ERROR, NYql::TSeverityIds::S_INFO)); +inline void AddPath(NYdb::NIssue::TIssues& issues, const TString& path) { + issues.AddIssue(NYdb::NIssue::TIssue(TStringBuilder() << "Path: " << path) + .SetCode(NYdb::NIssue::DEFAULT_ERROR, NYdb::NIssue::ESeverity::Info)); } template inline TResult Result(const TMaybe& path = Nothing(), EStatus code = EStatus::SUCCESS, const TString& error = {}) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; if (path) { AddPath(issues, *path); } if (error) { - issues.AddIssue(NYql::TIssue(error)); + issues.AddIssue(NYdb::NIssue::TIssue(error)); } return TResult(TStatus(code, std::move(issues))); } @@ -33,7 +33,7 @@ inline TResult Result(EStatus code, const TString& error) { template inline TResult Result(const TString& path, TStatus&& status) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; AddPath(issues, path); issues.AddIssues(status.GetIssues()); return TResult(TStatus(status.GetStatus(), std::move(issues))); diff --git a/ydb/public/lib/ydb_cli/dump/util/ya.make b/ydb/public/lib/ydb_cli/dump/util/ya.make index 4d58ac92ed49..4f80e1416d43 100644 --- a/ydb/public/lib/ydb_cli/dump/util/ya.make +++ b/ydb/public/lib/ydb_cli/dump/util/ya.make @@ -8,9 +8,9 @@ SRCS( PEERDIR( library/cpp/protobuf/util ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_types/status + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/types/status yql/essentials/parser/proto_ast/gen/v1 yql/essentials/parser/proto_ast/gen/v1_proto_split yql/essentials/sql/settings diff --git a/ydb/public/lib/ydb_cli/dump/ya.make b/ydb/public/lib/ydb_cli/dump/ya.make index 670d15ca80a0..82dc9b7c6a6c 100644 --- a/ydb/public/lib/ydb_cli/dump/ya.make +++ b/ydb/public/lib/ydb_cli/dump/ya.make @@ -19,9 +19,9 @@ PEERDIR( ydb/public/lib/ydb_cli/common ydb/public/lib/ydb_cli/dump/files ydb/public/lib/ydb_cli/dump/util - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/topic ) GENERATE_ENUM_SERIALIZATION(dump.h) diff --git a/ydb/public/lib/ydb_cli/import/import.cpp b/ydb/public/lib/ydb_cli/import/import.cpp index 52d3240e9774..98607182c993 100644 --- a/ydb/public/lib/ydb_cli/import/import.cpp +++ b/ydb/public/lib/ydb_cli/import/import.cpp @@ -1,12 +1,12 @@ #include "import.h" #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -56,9 +56,9 @@ constexpr ui64 rowsToAnalyze = 100000; inline TStatus MakeStatus(EStatus code = EStatus::SUCCESS, const TString& error = {}) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; if (error) { - issues.AddIssue(NYql::TIssue(error)); + issues.AddIssue(NYdb::NIssue::TIssue(error)); } return TStatus(code, std::move(issues)); } @@ -94,7 +94,7 @@ void InitCsvParser(TCsvParser& parser, NCsvFormat::TLinesSplitter& csvSource, const TImportFileSettings& settings, const TString& headerRow, - const std::map* columnTypes = nullptr, + const std::map* columnTypes = nullptr, const NTable::TTableDescription* dbTableInfo = nullptr) { if (settings.Header_ || headerRow) { TString newHeaderRow; @@ -121,7 +121,7 @@ void InitCsvParser(TCsvParser& parser, throw yexception() << "Need to specify column names"; } for (const auto& column : dbTableInfo->GetColumns()) { - columns.push_back(column.Name); + columns.push_back(TString{column.Name}); } parser = TCsvParser(std::move(columns), settings.Delimiter_[0], settings.NullValue_, columnTypes); } @@ -508,7 +508,7 @@ class TImportFileClient::TImpl { TStatus UpsertParquet(const TString& filename, const TString& dbPath, ProgressCallbackFunc & progressCallback); TAsyncStatus UpsertParquetBuffer(const TString& dbPath, const TString& buffer, const TString& strSchema); TType GetTableType(); - std::map GetColumnTypes(); + std::map GetColumnTypes(); void ValidateTValueUpsertTable(); std::shared_ptr LoadOrStartImportProgress(const TString& filePath); TStatus GenerateCreateTableFromCsv(IInputStream& input, @@ -1603,8 +1603,8 @@ TType TImportFileClient::TImpl::GetTableType() { return typeBuilder.Build(); } -std::map TImportFileClient::TImpl::GetColumnTypes() { - std::map columnTypes; +std::map TImportFileClient::TImpl::GetColumnTypes() { + std::map columnTypes; Y_ENSURE_BT(DbTableInfo); const auto& columns = DbTableInfo->GetTableColumns(); for (auto it = columns.begin(); it != columns.end(); it++) { diff --git a/ydb/public/lib/ydb_cli/import/import.h b/ydb/public/lib/ydb_cli/import/import.h index d46780e39b53..0c90e14830e7 100644 --- a/ydb/public/lib/ydb_cli/import/import.h +++ b/ydb/public/lib/ydb_cli/import/import.h @@ -8,14 +8,15 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include namespace NYdb { +inline namespace V3 { class TDriver; namespace NOperation { @@ -30,6 +31,7 @@ class TTableClient; namespace NImport { class TImportClient; } +} namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/import/ya.make b/ydb/public/lib/ydb_cli/import/ya.make index 79e0d4d92cda..b6b4ff8c1754 100644 --- a/ydb/public/lib/ydb_cli/import/ya.make +++ b/ydb/public/lib/ydb_cli/import/ya.make @@ -8,7 +8,7 @@ SRCS( PEERDIR( ydb/public/api/protos ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/proto ydb/public/lib/json_value contrib/libs/apache/arrow library/cpp/string_utils/csv diff --git a/ydb/public/lib/ydb_cli/topic/topic_read.cpp b/ydb/public/lib/ydb_cli/topic/topic_read.cpp index 4586e04df7cf..97c2f61b9436 100644 --- a/ydb/public/lib/ydb_cli/topic/topic_read.cpp +++ b/ydb/public/lib/ydb_cli/topic/topic_read.cpp @@ -75,7 +75,7 @@ namespace NYdb::NConsoleClient { } namespace { - const TString FormatBody(const TString& body, ETransformBody transform) { + const std::string FormatBody(const std::string& body, ETransformBody transform) { if (transform == ETransformBody::Base64) { return Base64Encode(body); } @@ -309,7 +309,7 @@ namespace NYdb::NConsoleClient { } else if (auto* endPartitionSessionEvent = std::get_if(&event)) { return HandleEndPartitionSessionEvent(endPartitionSessionEvent); } else if (auto* sessionClosedEvent = std::get_if(&event)) { - ThrowOnError(*sessionClosedEvent); + NStatusHelpers::ThrowOnErrorOrPrintIssues(*sessionClosedEvent); return 1; } return 0; @@ -329,7 +329,7 @@ namespace NYdb::NConsoleClient { if (future.HasValue()) { // TODO(shmel1k@): throttling? // TODO(shmel1k@): think about limiting size of events - TVector events = ReadSession_->GetEvents(true); + std::vector events = ReadSession_->GetEvents(true); for (auto& event : events) { if (int status = HandleEvent(event, output); status) { return status; diff --git a/ydb/public/lib/ydb_cli/topic/topic_read.h b/ydb/public/lib/ydb_cli/topic/topic_read.h index 80f3742f6f8b..0eed95c11410 100644 --- a/ydb/public/lib/ydb_cli/topic/topic_read.h +++ b/ydb/public/lib/ydb_cli/topic/topic_read.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace NYdb::NConsoleClient { #define GETTER(TYPE, NAME) \ diff --git a/ydb/public/lib/ydb_cli/topic/topic_read_ut.cpp b/ydb/public/lib/ydb_cli/topic/topic_read_ut.cpp index cbe1ae66c247..e444663f662e 100644 --- a/ydb/public/lib/ydb_cli/topic/topic_read_ut.cpp +++ b/ydb/public/lib/ydb_cli/topic/topic_read_ut.cpp @@ -1,6 +1,6 @@ #include "topic_read.h" #include -#include +#include #include namespace NYdb::NConsoleClient { @@ -142,7 +142,7 @@ namespace NYdb::NConsoleClient { } } - NTopic::TReadSessionSettings PrepareReadSessionSettings(const TString& topicPath) { + NTopic::TReadSessionSettings PrepareReadSessionSettings(const std::string& topicPath) { NTopic::TReadSessionSettings settings; settings.ConsumerName("cli"); settings.AppendTopics(topicPath); diff --git a/ydb/public/lib/ydb_cli/topic/topic_write.cpp b/ydb/public/lib/ydb_cli/topic/topic_write.cpp index ac0820e68447..2d1c06a68821 100644 --- a/ydb/public/lib/ydb_cli/topic/topic_write.cpp +++ b/ydb/public/lib/ydb_cli/topic/topic_write.cpp @@ -70,7 +70,7 @@ namespace NYdb::NConsoleClient { int TTopicWriter::Init() { TInstant endPreparationTime = Now() + DefaultMessagesWaitTimeout; - NThreading::TFuture initSeqNo = WriteSession_->GetInitSeqNo(); + NThreading::TFuture initSeqNo = WriteSession_->GetInitSeqNo(); while (Now() < endPreparationTime) { // TODO(shmel1k@): handle situation if seqNo already exists but with exception. @@ -87,7 +87,7 @@ namespace NYdb::NConsoleClient { if (initSeqNo.HasException()) { // NOTE(shmel1k@): SessionClosedEvent is stored in EventsQueue, so we can try to get it. auto event = WriteSession_->GetEvent(true); - if (event.Defined()) { + if (event.has_value()) { return HandleEvent(*event); } initSeqNo.TryRethrow(); @@ -110,7 +110,7 @@ namespace NYdb::NConsoleClient { } int TTopicWriter::HandleSessionClosedEvent(const NTopic::TSessionClosedEvent* event) { - ThrowOnError(*event); + NStatusHelpers::ThrowOnErrorOrPrintIssues(*event); return EXIT_FAILURE; } @@ -172,8 +172,8 @@ namespace NYdb::NConsoleClient { bool continueSending = true; while (continueSending) { while (!ContinuationToken_.Defined()) { - TMaybe event = WriteSession_->GetEvent(true); - if (event.Empty()) { + std::optional event = WriteSession_->GetEvent(true); + if (!event.has_value()) { continue; } if (int status = HandleEvent(*event); status) { @@ -197,7 +197,7 @@ namespace NYdb::NConsoleClient { if (WriteSession_->Close(closeTimeout)) { return true; } - TVector events = WriteSession_->GetEvents(true); + std::vector events = WriteSession_->GetEvents(true); if (events.empty()) { return false; } diff --git a/ydb/public/lib/ydb_cli/topic/topic_write.h b/ydb/public/lib/ydb_cli/topic/topic_write.h index e45bd0bdffad..852e37c9f4e1 100644 --- a/ydb/public/lib/ydb_cli/topic/topic_write.h +++ b/ydb/public/lib/ydb_cli/topic/topic_write.h @@ -1,7 +1,7 @@ #pragma once #include "ydb/public/lib/ydb_cli/commands/ydb_command.h" -#include +#include #include #include #include diff --git a/ydb/public/lib/ydb_cli/topic/ut/ya.make b/ydb/public/lib/ydb_cli/topic/ut/ya.make index 397dd282f463..f4b36d3c950e 100644 --- a/ydb/public/lib/ydb_cli/topic/ut/ya.make +++ b/ydb/public/lib/ydb_cli/topic/ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/public/lib/ydb_cli/topic) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) @@ -21,13 +25,13 @@ PEERDIR( library/cpp/histogram/hdr library/cpp/threading/local_executor ydb/core/fq/libs/private_client - ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public ydb/public/lib/experimental ydb/public/lib/ydb_cli/commands ydb/public/lib/ydb_cli/common ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ) YQL_LAST_ABI_VERSION() diff --git a/ydb/public/lib/ydb_cli/topic/ya.make b/ydb/public/lib/ydb_cli/topic/ya.make index 5c187fee7ed5..fc0678f37185 100644 --- a/ydb/public/lib/ydb_cli/topic/ya.make +++ b/ydb/public/lib/ydb_cli/topic/ya.make @@ -7,9 +7,9 @@ SRCS( PEERDIR( ydb/public/lib/ydb_cli/common - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/topic ) GENERATE_ENUM_SERIALIZATION(topic_metadata_fields.h) diff --git a/ydb/public/lib/yson_value/ya.make b/ydb/public/lib/yson_value/ya.make index 6e7149092179..189afb91902c 100644 --- a/ydb/public/lib/yson_value/ya.make +++ b/ydb/public/lib/yson_value/ya.make @@ -7,8 +7,8 @@ SRCS( PEERDIR( library/cpp/yson library/cpp/yson/node - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/value yql/essentials/types/uuid ) diff --git a/ydb/public/lib/yson_value/ydb_yson_value.cpp b/ydb/public/lib/yson_value/ydb_yson_value.cpp index 04035e7c3f5d..5a2769b0afa2 100644 --- a/ydb/public/lib/yson_value/ydb_yson_value.cpp +++ b/ydb/public/lib/yson_value/ydb_yson_value.cpp @@ -1,7 +1,7 @@ #include "ydb_yson_value.h" -#include -#include +#include +#include #include diff --git a/ydb/public/lib/yson_value/ydb_yson_value.h b/ydb/public/lib/yson_value/ydb_yson_value.h index c20bdf35e708..94ab10555cc3 100644 --- a/ydb/public/lib/yson_value/ydb_yson_value.h +++ b/ydb/public/lib/yson_value/ydb_yson_value.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include diff --git a/ydb/public/sdk/cpp/CMakeLists.txt b/ydb/public/sdk/cpp/CMakeLists.txt new file mode 100644 index 000000000000..d0b25570784f --- /dev/null +++ b/ydb/public/sdk/cpp/CMakeLists.txt @@ -0,0 +1,99 @@ +cmake_minimum_required(VERSION 3.15) +project(YDB-CPP-SDK LANGUAGES C CXX ASM) + +option(YDB_SDK_INSTALL "Install YDB C++ SDK" Off) +option(YDB_SDK_TESTS "Build YDB C++ SDK tests" Off) +option(YDB_SDK_EXAMPLES "Build YDB C++ SDK examples" On) +set(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET "" CACHE STRING "Name of cmake target preparing google common proto library") +option(YDB_SDK_USE_RAPID_JSON "Search for rapid json library in system" ON) + +set(BUILD_SHARED_LIBS Off) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED On) +set(CMAKE_CXX_EXTENSIONS On) + +set(YDB_SDK_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(YDB_SDK_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) +set(YDB-CPP-SDK_AVAILABLE_COMPONENTS "" CACHE INTERNAL "") +set(YDB-CPP-SDK_COMPONENT_TARGETS "" CACHE INTERNAL "") +file(READ "src/client/resources/ydb_sdk_version.txt" YDB_SDK_VERSION) + +#[=============================================================================[ + NOTE: if `ccache` is used with the environment variable `CCACHE_BASEDIR`, + these cached variable should be set manually by passing them to `cmake` as + `-DARCADIA_ROOT=source/path/relative/to/build/dir` and + `-DARCADIA_BUILD_ROOT=.`, because in that case the macro `__FILE__` will be + expanded to a relative path, even if the source code file was specified as + an absolute path, and we have to know the proper prefix of that path. + See details: https://ccache.dev/manual/3.1.html#_compiling_in_different_directories +#]=============================================================================] +set(ARCADIA_ROOT ${YDB_SDK_SOURCE_DIR} CACHE INTERNAL "The source root directory") +set(ARCADIA_BUILD_ROOT ${YDB_SDK_BINARY_DIR} CACHE INTERNAL "The build root directory") + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +list(APPEND CMAKE_MODULE_PATH ${YDB_SDK_SOURCE_DIR}/cmake) + +include(cmake/global_flags.cmake) +include(cmake/global_vars.cmake) +include(cmake/install.cmake) +include(cmake/common.cmake) +include(cmake/ccache.cmake) +include(cmake/protobuf.cmake) +include(cmake/testing.cmake) +include(cmake/external_libs.cmake) + +if (YDB_SDK_TESTS) + enable_testing() +endif() + +add_subdirectory(tools) +add_subdirectory(contrib/libs) +add_subdirectory(library/cpp) +add_subdirectory(include/ydb-cpp-sdk/client) +add_subdirectory(src) +add_subdirectory(util) + +#_ydb_sdk_validate_public_headers() + +if (YDB_SDK_EXAMPLES) + add_subdirectory(examples) +endif() + +if (YDB_SDK_TESTS) + add_subdirectory(tests) +endif() + +if (YDB_SDK_INSTALL) + _ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR}) + install(EXPORT ydb-cpp-sdk-targets + FILE ydb-cpp-sdk-targets.cmake + CONFIGURATIONS RELEASE + NAMESPACE YDB-CPP-SDK:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk/release + ) + configure_package_config_file( + ${YDB_SDK_SOURCE_DIR}/cmake/ydb-cpp-sdk-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/ydb-cpp-sdk-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk + ) + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/ydb-cpp-sdk-config-version.cmake + VERSION ${YDB_SDK_VERSION} COMPATIBILITY SameMajorVersion + ) + _ydb_sdk_directory_install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/ydb-cpp-sdk-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/ydb-cpp-sdk-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk + ) + _ydb_sdk_directory_install(FILES + ${YDB_SDK_SOURCE_DIR}/cmake/FindBrotli.cmake + ${YDB_SDK_SOURCE_DIR}/cmake/FindgRPC.cmake + ${YDB_SDK_SOURCE_DIR}/cmake/FindIDN.cmake + ${YDB_SDK_SOURCE_DIR}/cmake/FindLZ4.cmake + ${YDB_SDK_SOURCE_DIR}/cmake/FindxxHash.cmake + ${YDB_SDK_SOURCE_DIR}/cmake/FindZSTD.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk/Modules + ) +endif() diff --git a/ydb/public/sdk/cpp/adapters/issue/issue.cpp b/ydb/public/sdk/cpp/adapters/issue/issue.cpp new file mode 100644 index 000000000000..4c5a3febccbc --- /dev/null +++ b/ydb/public/sdk/cpp/adapters/issue/issue.cpp @@ -0,0 +1,40 @@ +#include "issue.h" + +#include + +#include + + +namespace NYdb::NAdapters { + +NYql::TIssue ToYqlIssue(const NYdb::NIssue::TIssue& sdkIssue) { + Ydb::Issue::IssueMessage message; + NYdb::NIssue::IssueToMessage(sdkIssue, &message); + return NYql::IssueFromMessage(message); +} + +NYql::TIssues ToYqlIssues(const NYdb::NIssue::TIssues& sdkIssues) { + google::protobuf::RepeatedPtrField message; + NYdb::NIssue::IssuesToMessage(sdkIssues, &message); + + NYql::TIssues yqlIssues; + NYql::IssuesFromMessage(message, yqlIssues); + return yqlIssues; +} + +NYdb::NIssue::TIssue ToSdkIssue(const NYql::TIssue& yqlIssue) { + Ydb::Issue::IssueMessage message; + NYql::IssueToMessage(yqlIssue, &message); + return NYdb::NIssue::IssueFromMessage(message); +} + +NYdb::NIssue::TIssues ToSdkIssues(const NYql::TIssues& yqlIssues) { + google::protobuf::RepeatedPtrField message; + NYql::IssuesToMessage(yqlIssues, &message); + + NYdb::NIssue::TIssues sdkIssues; + NYdb::NIssue::IssuesFromMessage(message, sdkIssues); + return sdkIssues; +} + +} diff --git a/ydb/public/sdk/cpp/adapters/issue/issue.h b/ydb/public/sdk/cpp/adapters/issue/issue.h new file mode 100644 index 000000000000..8a9b48711dbf --- /dev/null +++ b/ydb/public/sdk/cpp/adapters/issue/issue.h @@ -0,0 +1,13 @@ +#include +#include + + +namespace NYdb::NAdapters { + +NYql::TIssue ToYqlIssue(const NYdb::NIssue::TIssue& sdkIssue); +NYql::TIssues ToYqlIssues(const NYdb::NIssue::TIssues& sdkIssues); + +NYdb::NIssue::TIssue ToSdkIssue(const NYql::TIssue& yqlIssue); +NYdb::NIssue::TIssues ToSdkIssues(const NYql::TIssues& yqlIssues); + +} diff --git a/ydb/public/sdk/cpp/adapters/issue/ya.make b/ydb/public/sdk/cpp/adapters/issue/ya.make new file mode 100644 index 000000000000..898ca9c7cdca --- /dev/null +++ b/ydb/public/sdk/cpp/adapters/issue/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +PEERDIR( + yql/essentials/public/issue + ydb/library/yql/public/ydb_issue + ydb/public/sdk/cpp/src/library/issue +) + +SRCS( + issue.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/adapters/ya.make b/ydb/public/sdk/cpp/adapters/ya.make new file mode 100644 index 000000000000..fa575c59d873 --- /dev/null +++ b/ydb/public/sdk/cpp/adapters/ya.make @@ -0,0 +1,3 @@ +RECURSE( + issue +) diff --git a/ydb/public/sdk/cpp/client/README.md b/ydb/public/sdk/cpp/client/README.md new file mode 100644 index 000000000000..b694f675e052 --- /dev/null +++ b/ydb/public/sdk/cpp/client/README.md @@ -0,0 +1,3 @@ +# YDB SDK 2.6.2 + +It's deprecated version of C++ YDB Driver, DON'T USE IT IN NEW CODE. diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h index 4e3e544375d2..565d942e3c04 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { template std::unique_ptr StartGrpcServer(const TString& address, TService& service) { diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp index d797ecce5ad4..c4b88a5f5cb5 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp @@ -1,6 +1,6 @@ #include "scripting.h" -namespace NYdb::NScripting { +namespace NYdb::inline V2::NScripting { grpc::Status TMockSlyDbProxy::ExecuteYql( grpc::ServerContext* context, diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h index 6de9e62ed3bd..45d4a6aced80 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h @@ -2,7 +2,7 @@ #include -namespace NYdb::NScripting { +namespace NYdb::inline V2::NScripting { class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service { diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp index 03db2dd68f96..1690e31acfc9 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp @@ -1,6 +1,6 @@ #include "view.h" -namespace NYdb::NView { +namespace NYdb::inline V2::NView { grpc::Status TViewDummyService::DescribeView( grpc::ServerContext* context, diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h index 704573dbb2d1..27b7c21a9f48 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h @@ -2,7 +2,7 @@ #include -namespace NYdb::NView { +namespace NYdb::inline V2::NView { constexpr const char* DummyQueryText = "select 42"; diff --git a/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp b/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp index 6f9bc139f566..21fc04abc5f1 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp +++ b/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp @@ -6,7 +6,7 @@ #include using namespace NYdb; -using namespace NYdb::NScripting; +using namespace NYdb::V2::NScripting; Y_UNIT_TEST_SUITE(ResponseHeaders) { Y_UNIT_TEST(PassHeader) { diff --git a/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp b/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp index e5a87e0ee27b..32d6a2b911d8 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp +++ b/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp @@ -6,7 +6,7 @@ #include using namespace NYdb; -using namespace NYdb::NView; +using namespace NYdb::V2::NView; Y_UNIT_TEST_SUITE(ViewClient) { Y_UNIT_TEST(Basic) { diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp index 643af15fd9c5..3c3333b99e2b 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp @@ -3,7 +3,7 @@ #include #include -namespace NYdb::NDynamicConfig { +namespace NYdb::inline V2::NDynamicConfig { class TDynamicConfigClient::TImpl : public TClientImplCommon { public: diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h index 1c02d5427448..82848396df9a 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h @@ -14,7 +14,7 @@ #include -namespace NYdb::NDynamicConfig { +namespace NYdb::inline V2::NDynamicConfig { struct TGetConfigResult : public TStatus { TGetConfigResult( diff --git a/ydb/public/sdk/cpp/client/draft/ydb_replication.cpp b/ydb/public/sdk/cpp/client/draft/ydb_replication.cpp index 71e792ea4199..2b0411e7af36 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_replication.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_replication.cpp @@ -13,7 +13,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NReplication { TConnectionParams::TConnectionParams(const Ydb::Replication::ConnectionParams& params) { diff --git a/ydb/public/sdk/cpp/client/draft/ydb_replication.h b/ydb/public/sdk/cpp/client/draft/ydb_replication.h index 5148368be9c6..524a28e3295a 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_replication.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_replication.h @@ -15,7 +15,7 @@ namespace Ydb::Replication { class DescribeReplicationResult_Stats; } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; } @@ -23,14 +23,14 @@ namespace NYql { class TIssues; } -namespace NYdb::NReplication { +namespace NYdb::inline V2::NReplication { class TDescribeReplicationResult; using TAsyncDescribeReplicationResult = NThreading::TFuture; struct TDescribeReplicationSettings: public TOperationRequestSettings { using TSelf = TDescribeReplicationSettings; - FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeStats, false); }; struct TStaticCredentials { diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp index a11043224b25..62424128fa54 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp @@ -9,7 +9,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NScripting { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h index 324c96597d6b..953062e5e8e0 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h @@ -4,7 +4,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NScripting { class TExecuteYqlResult : public TStatus { @@ -102,8 +102,8 @@ using TAsyncExplainYqlResult = NThreading::TFuture; //////////////////////////////////////////////////////////////////////////////// struct TExecuteYqlRequestSettings : public TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(Ydb::Query::Syntax, Syntax, Ydb::Query::SYNTAX_YQL_V1); - FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); + FLUENT_SETTING_DEFAULT_DEPRECATED(Ydb::Query::Syntax, Syntax, Ydb::Query::SYNTAX_YQL_V1); + FLUENT_SETTING_DEFAULT_DEPRECATED(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); }; enum class ExplainYqlRequestMode { @@ -113,7 +113,7 @@ enum class ExplainYqlRequestMode { }; struct TExplainYqlRequestSettings : public TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); + FLUENT_SETTING_DEFAULT_DEPRECATED(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/draft/ydb_view.cpp b/ydb/public/sdk/cpp/client/draft/ydb_view.cpp index d58616d15e54..e235711721d4 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_view.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_view.cpp @@ -9,7 +9,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NView { TViewDescription::TViewDescription(const Ydb::View::DescribeViewResult& desc) diff --git a/ydb/public/sdk/cpp/client/draft/ydb_view.h b/ydb/public/sdk/cpp/client/draft/ydb_view.h index 6f9908e90414..bd138eecadc8 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_view.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_view.h @@ -7,7 +7,7 @@ namespace Ydb::View { class DescribeViewResult; } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; } @@ -15,7 +15,7 @@ namespace NYql { class TIssues; } -namespace NYdb::NView { +namespace NYdb::inline V2::NView { class TDescribeViewResult; using TAsyncDescribeViewResult = NThreading::TFuture; diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp index ec7b0f0db68d..779264a4ae0b 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp @@ -1,6 +1,6 @@ #include "pull_client.h" -namespace NSolomonStatExtension { +namespace NSolomonStatExtension::inline V2 { TSolomonStatPullExtension::TParams::TParams(const TString& host , ui16 port diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h index 8d410c2fa2eb..952dbf842918 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h @@ -11,7 +11,7 @@ #include -namespace NSolomonStatExtension { +namespace NSolomonStatExtension::inline V2 { class TSolomonStatPullExtension: public NYdb::IExtension { public: diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h index 24f5f30069d8..df7eced726af 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h @@ -6,7 +6,7 @@ #include -namespace NSolomonStatExtension { +namespace NSolomonStatExtension::inline V2 { template class TMetricRegistryConnector: public NYdb::IExtension { diff --git a/ydb/public/sdk/cpp/client/helpers/helpers.cpp b/ydb/public/sdk/cpp/client/helpers/helpers.cpp index 933e23d7c883..307578edca15 100644 --- a/ydb/public/sdk/cpp/client/helpers/helpers.cpp +++ b/ydb/public/sdk/cpp/client/helpers/helpers.cpp @@ -7,7 +7,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { TDriverConfig CreateFromEnvironment(const TStringType& connectionString) { TDriverConfig driverConfig; diff --git a/ydb/public/sdk/cpp/client/helpers/helpers.h b/ydb/public/sdk/cpp/client/helpers/helpers.h index a5b84f538e71..d693f6017097 100644 --- a/ydb/public/sdk/cpp/client/helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/helpers/helpers.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { //! Checks the following environment variables and creates TDriverConfig with the first appeared: //! YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS= — service account key file, diff --git a/ydb/public/sdk/cpp/client/iam/common/iam.cpp b/ydb/public/sdk/cpp/client/iam/common/iam.cpp index 79b3ed62c6ba..313d5dccab23 100644 --- a/ydb/public/sdk/cpp/client/iam/common/iam.cpp +++ b/ydb/public/sdk/cpp/client/iam/common/iam.cpp @@ -7,7 +7,7 @@ using namespace NYdbGrpc; using namespace yandex::cloud::iam::v1; -namespace NYdb { +namespace NYdb::inline V2 { class TIAMCredentialsProvider : public ICredentialsProvider { public: diff --git a/ydb/public/sdk/cpp/client/iam/common/iam.h b/ydb/public/sdk/cpp/client/iam/common/iam.h index 42bbf89f906c..d52412309375 100644 --- a/ydb/public/sdk/cpp/client/iam/common/iam.h +++ b/ydb/public/sdk/cpp/client/iam/common/iam.h @@ -14,7 +14,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NIam { constexpr TStringBuf DEFAULT_ENDPOINT = "iam.api.cloud.yandex.net"; diff --git a/ydb/public/sdk/cpp/client/iam/iam.cpp b/ydb/public/sdk/cpp/client/iam/iam.cpp index 02626df8fa59..276d8a3c9d4d 100644 --- a/ydb/public/sdk/cpp/client/iam/iam.cpp +++ b/ydb/public/sdk/cpp/client/iam/iam.cpp @@ -5,7 +5,7 @@ using namespace yandex::cloud::iam::v1; -namespace NYdb { +namespace NYdb::inline V2 { TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactory(const TIamJwtFilename& params) { TIamJwtParams jwtParams = { params, ReadJwtKeyFile(params.JwtFilename) }; diff --git a/ydb/public/sdk/cpp/client/iam_private/iam.cpp b/ydb/public/sdk/cpp/client/iam_private/iam.cpp index 8eff5511a765..fce393f340c4 100644 --- a/ydb/public/sdk/cpp/client/iam_private/iam.cpp +++ b/ydb/public/sdk/cpp/client/iam_private/iam.cpp @@ -5,7 +5,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { TCredentialsProviderFactoryPtr CreateIamJwtCredentialsProviderFactoryImplPrivate(TIamJwtParams&& jwtParams) { return std::make_shared -namespace NYdb { +namespace NYdb::inline V2 { /// Acquire an IAM token using a JSON Web Token (JWT) file name. TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactoryPrivate(const TIamJwtFilename& params); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp index 9c4eaf3fccf8..2488d03f1f7e 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { using std::string; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h index 4c0fa61df328..91217718c703 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { struct TEndpointRecord { std::string Endpoint; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make index 9272547a3052..f84f483ebf76 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( library/cpp/monlib/metrics ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/impl/ydb_stats ) END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.cpp index d8e120053f3a..ada62fb5b1e0 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.cpp @@ -12,7 +12,7 @@ #include #endif -namespace NYdb { +namespace NYdb::inline V2 { namespace { ui32 GetProcessId() { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.h index 799b481a73e5..66dd5f00e691 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/client_pid.h @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TStringType GetClientPIDHeaderValue(); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.cpp index 6781852b53a0..1a6232f907c9 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TStringType GetStrFromEnv(const char* envVarName, const TStringType& defaultValue) { auto envVarPointer = getenv(envVarName); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.h index 5e699db413fb..dd512bbf293e 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TStringType GetStrFromEnv(const char* envVarName, const TStringType& defaultValue = ""); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.cpp index f44bd8b75fba..a9ac509eca8b 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.cpp @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { TConnectionInfo ParseConnectionString(const TString& connectionString) { if (connectionString.length() == 0) { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h index cbcf9e156906..1c974a409c71 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { struct TConnectionInfo { TStringType Endpoint = ""; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ssl_credentials.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ssl_credentials.h index 016a01dcdd4a..ed1c13dce061 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ssl_credentials.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ssl_credentials.h @@ -2,7 +2,7 @@ #include "type_switcher.h" -namespace NYdb { +namespace NYdb::inline V2 { struct TSslCredentials { bool IsEnabled = false; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.cpp index 8187ef879218..a9fedcfe5257 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { bool StringStartsWith(const TStringType& line, const TStringType& pattern) { return std::equal(pattern.begin(), pattern.end(), line.begin()); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.h index 2220cd8a1d25..f1de6028d57c 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/string_helpers.h @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { // C++17 support for external users bool StringStartsWith(const TStringType& line, const TStringType& pattern); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h index 4cbbbafc3adf..13818006e415 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h @@ -9,7 +9,7 @@ #include #endif -namespace NYdb { +namespace NYdb::inline V2 { #ifndef EXTERNAL_BUILD using TStringType = TString; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h index 03dda5d51e1d..d065d6dd155b 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h @@ -12,7 +12,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { // Other callbacks using TSimpleCb = std::function; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp index 4ae32981e8e6..4abfa9647019 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr credentialsProvider) : CredentialsProvider_(std::move(credentialsProvider)) diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h index e9adbbbb92bb..4f4bc998ecf7 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h @@ -6,7 +6,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin { public: diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp index 02fda2ec4c49..3ce66c3b8917 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp @@ -1,7 +1,7 @@ #define INCLUDE_YDB_INTERNAL_H #include "endpoint_pool.h" -namespace NYdb { +namespace NYdb::inline V2 { using std::string; using std::vector; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h index f29df631d35b..3a81f74d090f 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h @@ -11,7 +11,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { struct TListEndpointsResult { Ydb::Discovery::ListEndpointsResult Result; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp index 21de637853d4..6da20b32afa4 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp @@ -11,7 +11,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h index 0246f7166b1a..63e7a7de8f35 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class ICredentialsProvider; class ICredentialsProviderFactory; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h index 3aa91d560d6e..cc110fa84dd8 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr ui64 TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle // before TCP starts sending keepalive probes, seconds diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp index cbf945252f33..9c7576ce8124 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp @@ -4,7 +4,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h index da3cbd04444d..6fdd33e5001f 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h @@ -13,7 +13,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { using NYdbGrpc::IQueueClientContext; using NYdbGrpc::IQueueClientEvent; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index 7eec444eb513..38cdf626b8bc 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { bool IsTokenCorrect(const TStringType& in) { for (char c : in) { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h index 703ae7181c05..3f04242174ee 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -17,7 +17,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10); constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h index 978426a7a4d7..8de14a2d423d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class IConnectionsParams { public: diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h index 166cd0bcd00f..ce566f0c9c65 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h @@ -16,7 +16,7 @@ namespace NMonitoring { class TMetricRegistry; } -namespace NYdb { +namespace NYdb::inline V2 { class TDbDriverState; struct TListEndpointsResult; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp index 104b3056a189..8772f7f7b159 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { using std::string; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h index 96f1e9c50983..9e288e7bf0d7 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h @@ -9,7 +9,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { //////////////////////////////////////////////////////////////////////////////// ui64 GetNodeIdFromSession(const TStringType& sessionId); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/ya.make index ac4ebdf6e06e..48ffee251ba3 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/kqp_session_common/ya.make @@ -6,7 +6,7 @@ SRCS( PEERDIR( library/cpp/threading/future - ydb/public/lib/operation_id/protos + ydb/public/lib/operation_id ydb/public/sdk/cpp/client/impl/ydb_endpoints ) diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp index 78cb0538a102..f793451d771d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp @@ -4,7 +4,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { static const TStringBuf LogPrioritiesStrings[] = { " :EMERG: ", diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h index dc1ede5d290c..0c798b9bba00 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TLogFormatter GetPrefixLogFormatter(const TString& prefix); TStringType GetDatabaseLogPrefix(const TStringType& database); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp index bfbf465529a4..8e10cce8e677 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp @@ -2,7 +2,7 @@ #include "make.h" -namespace NYdb { +namespace NYdb::inline V2 { void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { protoValue.set_seconds(duration.Seconds()); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h index 51432ec1e8ee..77f655af6939 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h @@ -7,7 +7,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp index cd10393e02a8..b3755018ba87 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { using std::string; @@ -17,7 +17,7 @@ TPlainStatus::TPlainStatus( TStringType msg; if (grpcStatus.InternalError) { Status = EStatus::CLIENT_INTERNAL_ERROR; - if (grpcStatus.Msg) { + if (!grpcStatus.Msg.empty()) { msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg; } else { msg = "Unknown internal client error"; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h index f2a30892de43..9ddb09a573d0 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h @@ -10,7 +10,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { struct TPlainStatus { EStatus Status; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.cpp index 801a20fa22d5..1e5384da7d68 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.cpp @@ -3,7 +3,7 @@ #include #include -namespace NYdb::NRetry { +namespace NYdb::inline V2::NRetry { constexpr ui32 MAX_BACKOFF_DURATION_MS = TDuration::Hours(1).MilliSeconds(); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.h index 71cd45f48e6c..f9766bc53e20 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry.h @@ -14,11 +14,11 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class IClientImplCommon; } -namespace NYdb::NRetry { +namespace NYdb::inline V2::NRetry { ui32 CalcBackoffTime(const TBackoffSettings& settings, ui32 retryNumber); void Backoff(const NRetry::TBackoffSettings& settings, ui32 retryNumber); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_async.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_async.h index 931d3382c522..af86079be25f 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_async.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_async.h @@ -4,7 +4,7 @@ #include -namespace NYdb::NRetry::Async { +namespace NYdb::inline V2::NRetry::Async { template class TRetryContext : public TThrRefBase, public TRetryContextBase { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_sync.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_sync.h index 27a0a3c902c0..b59f719e6a89 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_sync.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/retry/retry_sync.h @@ -6,7 +6,7 @@ #include -namespace NYdb::NRetry::Sync { +namespace NYdb::inline V2::NRetry::Sync { template class TRetryContext : public TRetryContextBase { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h index 17d0edf089aa..dce21839ced4 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h @@ -4,7 +4,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { struct TRpcRequestSettings { TStringType TraceId; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/scheme_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/scheme_helpers/helpers.h index bd43f2b21d96..3c543e864b71 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/scheme_helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/scheme_helpers/helpers.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { template inline void PermissionToSchemeEntry(const TFrom& from, TVector* to) { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_client/session_client.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_client/session_client.h index 49f26b28d0eb..7e681503dab0 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_client/session_client.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_client/session_client.h @@ -4,7 +4,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TKqpSessionCommon; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.cpp index e9abd136fd99..a0604244585f 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.cpp @@ -9,7 +9,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NSessionPool { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.h index bbd59ef2ea36..ae67ae8e9238 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool/session_pool.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TOperation; namespace NSessionPool { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h index e93129470247..c6189b62ef54 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h @@ -8,7 +8,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TStatsExtractor: public NSdkStats::IStatApi { public: diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h index c6fa2c2679a0..020d9fb568d7 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h @@ -5,7 +5,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(TMaybe mode) { if (mode) { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h index 8917c83f25da..1f72542ad4a3 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h @@ -6,7 +6,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { inline std::unique_ptr CreateThreadPool(size_t threads) { std::unique_ptr queue; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp index 637dfde518a0..c47204df4ab2 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) { if (t1.type_case() != t2.type_case()) { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h index 9b6fd6a275f3..9d65e99fc4bf 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h @@ -4,7 +4,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp index 916ce1239f2d..32c143efff0d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NSdkStats { using std::string; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h index d2c50cb8d737..bca808f1f554 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h @@ -12,7 +12,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NSdkStats { diff --git a/ydb/public/sdk/cpp/client/resources/ydb_ca.cpp b/ydb/public/sdk/cpp/client/resources/ydb_ca.cpp index 5b6815c34a29..a73ae15ca560 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_ca.cpp +++ b/ydb/public/sdk/cpp/client/resources/ydb_ca.cpp @@ -2,7 +2,7 @@ #include "ydb_ca.h" -namespace NYdb { +namespace NYdb::inline V2 { TString GetRootCertificate() { return NResource::Find("ydb_root_ca.pem"); diff --git a/ydb/public/sdk/cpp/client/resources/ydb_ca.h b/ydb/public/sdk/cpp/client/resources/ydb_ca.h index 99e3cdb558e7..dad5ad1ef4ef 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_ca.h +++ b/ydb/public/sdk/cpp/client/resources/ydb_ca.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TString GetRootCertificate(); diff --git a/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp b/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp index df0e328fc9e5..a4f028380ee5 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp +++ b/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp @@ -2,7 +2,7 @@ #include "ydb_resources.h" -namespace NYdb { +namespace NYdb::inline V2 { const char* YDB_AUTH_TICKET_HEADER = "x-ydb-auth-ticket"; const char* YDB_DATABASE_HEADER = "x-ydb-database"; diff --git a/ydb/public/sdk/cpp/client/resources/ydb_resources.h b/ydb/public/sdk/cpp/client/resources/ydb_resources.h index 41307e30940b..e53e095b18cc 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_resources.h +++ b/ydb/public/sdk/cpp/client/resources/ydb_resources.h @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { extern const char* YDB_AUTH_TICKET_HEADER; extern const char* YDB_DATABASE_HEADER; diff --git a/ydb/public/sdk/cpp/client/ya.make b/ydb/public/sdk/cpp/client/ya.make new file mode 100644 index 000000000000..52fcdd89e8ad --- /dev/null +++ b/ydb/public/sdk/cpp/client/ya.make @@ -0,0 +1,56 @@ +RECURSE( + draft + draft/ut + extensions + helpers + impl/ydb_endpoints + impl/ydb_endpoints/ut + impl/ydb_internal/common + impl/ydb_internal/db_driver_state + impl/ydb_internal/grpc_connections + impl/ydb_internal/logger + impl/ydb_internal/make_request + impl/ydb_internal/plain_status + impl/ydb_internal/thread_pool + impl/ydb_internal/value_helpers + impl/ydb_stats + resources + ydb_common_client + ydb_common_client/impl + ydb_coordination + ydb_coordination/ut + ydb_datastreams + ydb_debug + ydb_discovery + ydb_driver + ydb_driver/ut + ydb_export + ydb_extension + ydb_federated_topic + ydb_federated_topic/impl + ydb_import + ydb_operation + ydb_params + ydb_persqueue_core + ydb_persqueue_public + ydb_persqueue_public/impl + ydb_proto + ydb_query + ydb_rate_limiter + ydb_result + ydb_result/ut + ydb_scheme + ydb_table + ydb_table/impl + ydb_table/query_stats + ydb_topic + ydb_topic/codecs + ydb_topic/impl + ydb_types + ydb_types/credentials + ydb_types/exceptions + ydb_types/fatal_error_handlers + ydb_types/operation + ydb_types/status + ydb_value +) diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 24556fc65b41..86b17e990770 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -11,7 +11,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { template class TClientImplCommon diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/iface.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/iface.h index 2f0e31059962..012ef2c79e7d 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/iface.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/iface.h @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class IClientImplCommon { public: diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp index 9baa0c1328dd..1bb01e11f061 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp +++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TCommonClientSettings& TCommonClientSettings::AuthToken(const TMaybe& token) { return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.GetRef())); diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h index e73f504e8d85..5cb2e4c76151 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { using TCertificateAndPrivateKey = std::pair; @@ -23,17 +23,17 @@ struct TCommonClientSettings { //! always better to keep instance of client. //! Allows to override current database for client - FLUENT_SETTING_OPTIONAL(TStringType, Database); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TStringType, Database); //! Allows to override current discovery endpoint - FLUENT_SETTING_OPTIONAL(TStringType, DiscoveryEndpoint); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TStringType, DiscoveryEndpoint); //! Allows to override current token for client TSelf& AuthToken(const TMaybe& token); //! Allows to override current credentials provider - FLUENT_SETTING_OPTIONAL(std::shared_ptr, CredentialsProviderFactory); + FLUENT_SETTING_OPTIONAL_DEPRECATED(std::shared_ptr, CredentialsProviderFactory); //! Allows to override discovery mode - FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EDiscoveryMode, DiscoveryMode); //! Allows to override current Ssl credentials - FLUENT_SETTING_OPTIONAL(TSslCredentials, SslCredentials); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TSslCredentials, SslCredentials); }; template diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp index 3aa5e64e19ff..1f1ed2ce46ef 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp @@ -11,7 +11,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NCoordination { using NThreading::TFuture; diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h index 01e1c2034202..af4adbdd70de 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h @@ -12,7 +12,7 @@ namespace Coordination { } } -namespace NYdb { +namespace NYdb::inline V2 { namespace NScheme { struct TPermissions; @@ -180,15 +180,15 @@ template struct TNodeSettings : public TOperationRequestSettings { using TSelf = TDerived; - FLUENT_SETTING_OPTIONAL(TDuration, SelfCheckPeriod); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, SelfCheckPeriod); - FLUENT_SETTING_OPTIONAL(TDuration, SessionGracePeriod); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, SessionGracePeriod); - FLUENT_SETTING_DEFAULT(EConsistencyMode, ReadConsistencyMode, EConsistencyMode::UNSET); + FLUENT_SETTING_DEFAULT_DEPRECATED(EConsistencyMode, ReadConsistencyMode, EConsistencyMode::UNSET); - FLUENT_SETTING_DEFAULT(EConsistencyMode, AttachConsistencyMode, EConsistencyMode::UNSET); + FLUENT_SETTING_DEFAULT_DEPRECATED(EConsistencyMode, AttachConsistencyMode, EConsistencyMode::UNSET); - FLUENT_SETTING_DEFAULT(ERateLimiterCountersMode, RateLimiterCountersMode, ERateLimiterCountersMode::UNSET); + FLUENT_SETTING_DEFAULT_DEPRECATED(ERateLimiterCountersMode, RateLimiterCountersMode, ERateLimiterCountersMode::UNSET); }; struct TCreateNodeSettings : public TNodeSettings { }; @@ -227,21 +227,21 @@ struct TSessionSettings : public TRequestSettings { using TStateCallback = std::function; using TStoppedCallback = std::function; - FLUENT_SETTING(TString, Description); + FLUENT_SETTING_DEPRECATED(TString, Description); - FLUENT_SETTING(TStateCallback, OnStateChanged); + FLUENT_SETTING_DEPRECATED(TStateCallback, OnStateChanged); - FLUENT_SETTING(TStoppedCallback, OnStopped); + FLUENT_SETTING_DEPRECATED(TStoppedCallback, OnStopped); - FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, Timeout, TDuration::Seconds(5)); - FLUENT_SETTING_DEFAULT(TDuration, ReconnectBackoffDelay, TDuration::MilliSeconds(250)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ReconnectBackoffDelay, TDuration::MilliSeconds(250)); - FLUENT_SETTING_DEFAULT(double, ReconnectBackoffMultiplier, 2.0); + FLUENT_SETTING_DEFAULT_DEPRECATED(double, ReconnectBackoffMultiplier, 2.0); - FLUENT_SETTING_DEFAULT(double, ReconnectSessionTimeoutMultiplier, 2.0); + FLUENT_SETTING_DEFAULT_DEPRECATED(double, ReconnectSessionTimeoutMultiplier, 2.0); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Zero()); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Zero()); }; //////////////////////////////////////////////////////////////////////////////// @@ -250,19 +250,19 @@ struct TAcquireSemaphoreSettings { using TSelf = TAcquireSemaphoreSettings; using TAcceptedCallback = std::function; - FLUENT_SETTING(TString, Data); + FLUENT_SETTING_DEPRECATED(TString, Data); - FLUENT_SETTING(TAcceptedCallback, OnAccepted); + FLUENT_SETTING_DEPRECATED(TAcceptedCallback, OnAccepted); - FLUENT_SETTING_DEFAULT(ui64, Count, 0); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, Count, 0); - FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Max()); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, Timeout, TDuration::Max()); - FLUENT_SETTING_FLAG(Ephemeral); + FLUENT_SETTING_FLAG_DEPRECATED(Ephemeral); - FLUENT_SETTING_FLAG_ALIAS(Shared, Count, ui64(1)); + FLUENT_SETTING_FLAG_ALIAS_DEPRECATED(Shared, Count, ui64(1)); - FLUENT_SETTING_FLAG_ALIAS(Exclusive, Count, ui64(-1)); + FLUENT_SETTING_FLAG_ALIAS_DEPRECATED(Exclusive, Count, ui64(-1)); }; //////////////////////////////////////////////////////////////////////////////// @@ -271,15 +271,15 @@ struct TDescribeSemaphoreSettings { using TSelf = TDescribeSemaphoreSettings; using TChangedCallback = std::function; - FLUENT_SETTING(TChangedCallback, OnChanged); + FLUENT_SETTING_DEPRECATED(TChangedCallback, OnChanged); - FLUENT_SETTING_FLAG(WatchData); + FLUENT_SETTING_FLAG_DEPRECATED(WatchData); - FLUENT_SETTING_FLAG(WatchOwners); + FLUENT_SETTING_FLAG_DEPRECATED(WatchOwners); - FLUENT_SETTING_FLAG(IncludeOwners); + FLUENT_SETTING_FLAG_DEPRECATED(IncludeOwners); - FLUENT_SETTING_FLAG(IncludeWaiters); + FLUENT_SETTING_FLAG_DEPRECATED(IncludeWaiters); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination_ut.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination_ut.cpp index ab27f11cd0d6..caaf84f6dafa 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination_ut.cpp @@ -11,7 +11,7 @@ #include using namespace NYdb; -using namespace NYdb::NCoordination; +using namespace NYdb::V2::NCoordination; namespace { diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/proto_accessor.cpp index adc3a5d286ef..2933c28723fe 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/proto_accessor.cpp @@ -2,7 +2,7 @@ #include "coordination.h" -namespace NYdb { +namespace NYdb::inline V2 { const Ydb::Coordination::DescribeNodeResult& TProtoAccessor::GetProto(const NCoordination::TNodeDescription& nodeDescription) { return nodeDescription.GetProto(); diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp index a2933f19256e..4d803159cdde 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp @@ -9,7 +9,7 @@ #include -namespace NYdb::NDataStreams::V1 { +namespace NYdb::inline V2::NDataStreams::V1 { TPartitioningSettingsBuilder TCreateStreamSettings::BeginConfigurePartitioningSettings() { return { *this }; diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h index b51bfea0307a..6111036d2321 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h @@ -4,7 +4,7 @@ #include -namespace NYdb::NDataStreams::V1 { +namespace NYdb::inline V2::NDataStreams::V1 { template class TProtoResultWrapper : public NYdb::TStatus { @@ -170,13 +170,13 @@ namespace NYdb::NDataStreams::V1 { }; struct TCreateStreamSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, ShardCount); - FLUENT_SETTING_OPTIONAL(ui32, RetentionPeriodHours); - FLUENT_SETTING_OPTIONAL(ui32, RetentionStorageMegabytes); - FLUENT_SETTING(ui64, WriteQuotaKbPerSec); - FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + FLUENT_SETTING_DEPRECATED(ui32, ShardCount); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, RetentionPeriodHours); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, RetentionStorageMegabytes); + FLUENT_SETTING_DEPRECATED(ui64, WriteQuotaKbPerSec); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EStreamMode, StreamMode); - FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TPartitioningSettings, PartitioningSettings); TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); }; template @@ -252,53 +252,53 @@ namespace NYdb::NDataStreams::V1 { }; struct TListStreamsSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, Limit); - FLUENT_SETTING(TString, ExclusiveStartStreamName); - FLUENT_SETTING_DEFAULT(bool, Recurse, true); + FLUENT_SETTING_DEPRECATED(ui32, Limit); + FLUENT_SETTING_DEPRECATED(TString, ExclusiveStartStreamName); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Recurse, true); }; struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, EnforceConsumerDeletion, false); }; struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, Limit); - FLUENT_SETTING(TString, ExclusiveStartShardId); + FLUENT_SETTING_DEPRECATED(ui32, Limit); + FLUENT_SETTING_DEPRECATED(TString, ExclusiveStartShardId); }; struct TListShardsSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(TString, ExclusiveStartShardId); - FLUENT_SETTING(ui32, MaxResults); - FLUENT_SETTING(TString, NextToken); - FLUENT_SETTING(ui64, StreamCreationTimestamp); + FLUENT_SETTING_DEPRECATED(TString, ExclusiveStartShardId); + FLUENT_SETTING_DEPRECATED(ui32, MaxResults); + FLUENT_SETTING_DEPRECATED(TString, NextToken); + FLUENT_SETTING_DEPRECATED(ui64, StreamCreationTimestamp); }; struct TGetRecordsSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(ui32, Limit, 10000); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, Limit, 10000); }; struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(TString, StartingSequenceNumber); - FLUENT_SETTING(ui64, Timestamp); + FLUENT_SETTING_DEPRECATED(TString, StartingSequenceNumber); + FLUENT_SETTING_DEPRECATED(ui64, Timestamp); }; struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings {}; struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings {}; struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings {}; struct TDecreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, RetentionPeriodHours); + FLUENT_SETTING_DEPRECATED(ui32, RetentionPeriodHours); }; struct TIncreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, RetentionPeriodHours); + FLUENT_SETTING_DEPRECATED(ui32, RetentionPeriodHours); }; struct TUpdateShardCountSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, TargetShardCount); + FLUENT_SETTING_DEPRECATED(ui32, TargetShardCount); }; struct TUpdateStreamModeSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(EStreamMode, StreamMode, ESM_PROVISIONED); + FLUENT_SETTING_DEFAULT_DEPRECATED(EStreamMode, StreamMode, ESM_PROVISIONED); }; struct TUpdateStreamSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, TargetShardCount); - FLUENT_SETTING_OPTIONAL(ui32, RetentionPeriodHours); - FLUENT_SETTING_OPTIONAL(ui32, RetentionStorageMegabytes); - FLUENT_SETTING(ui64, WriteQuotaKbPerSec); - FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + FLUENT_SETTING_DEPRECATED(ui32, TargetShardCount); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, RetentionPeriodHours); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, RetentionStorageMegabytes); + FLUENT_SETTING_DEPRECATED(ui64, WriteQuotaKbPerSec); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EStreamMode, StreamMode); - FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TPartitioningSettings, PartitioningSettings); TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); }; struct TPutRecordSettings : public NYdb::TOperationRequestSettings {}; @@ -307,8 +307,8 @@ namespace NYdb::NDataStreams::V1 { struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings {}; struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings {}; struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING(ui32, MaxResults); - FLUENT_SETTING(TString, NextToken); + FLUENT_SETTING_DEPRECATED(ui32, MaxResults); + FLUENT_SETTING_DEPRECATED(TString, NextToken); }; struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings {}; struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings {}; diff --git a/ydb/public/sdk/cpp/client/ydb_debug/client.cpp b/ydb/public/sdk/cpp/client/ydb_debug/client.cpp index 0749645dd847..78be214fe345 100644 --- a/ydb/public/sdk/cpp/client/ydb_debug/client.cpp +++ b/ydb/public/sdk/cpp/client/ydb_debug/client.cpp @@ -10,7 +10,7 @@ #include -namespace NYdb::NDebug { +namespace NYdb::inline V2::NDebug { using namespace Ydb; diff --git a/ydb/public/sdk/cpp/client/ydb_debug/client.h b/ydb/public/sdk/cpp/client/ydb_debug/client.h index 9e0b4581a7a3..92f27997320a 100644 --- a/ydb/public/sdk/cpp/client/ydb_debug/client.h +++ b/ydb/public/sdk/cpp/client/ydb_debug/client.h @@ -2,7 +2,7 @@ #include -namespace NYdb::NDebug { +namespace NYdb::inline V2::NDebug { //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp index 0fc8604592ac..88a152b3ed1d 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NDiscovery { TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& proto) diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h index 24241cf904c8..584298d8554f 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h @@ -12,7 +12,7 @@ namespace Discovery { } // namespace Discovery } // namespace Ydb -namespace NYdb { +namespace NYdb::inline V2 { namespace NDiscovery { //////////////////////////////////////////////////////////////////////////////// @@ -20,7 +20,7 @@ namespace NDiscovery { class TListEndpointsSettings : public TSimpleRequestSettings {}; struct TWhoAmISettings : public TSimpleRequestSettings { - FLUENT_SETTING_DEFAULT(bool, WithGroups, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithGroups, false); }; struct TNodeLocation { @@ -40,14 +40,14 @@ struct TNodeLocation { }; struct TNodeRegistrationSettings : public TSimpleRequestSettings { - FLUENT_SETTING(TString, Host); - FLUENT_SETTING(ui32, Port); - FLUENT_SETTING(TString, ResolveHost); - FLUENT_SETTING(TString, Address); - FLUENT_SETTING(TNodeLocation, Location); - FLUENT_SETTING(TString, DomainPath); - FLUENT_SETTING_DEFAULT(bool, FixedNodeId, false); - FLUENT_SETTING(TString, Path); + FLUENT_SETTING_DEPRECATED(TString, Host); + FLUENT_SETTING_DEPRECATED(ui32, Port); + FLUENT_SETTING_DEPRECATED(TString, ResolveHost); + FLUENT_SETTING_DEPRECATED(TString, Address); + FLUENT_SETTING_DEPRECATED(TNodeLocation, Location); + FLUENT_SETTING_DEPRECATED(TString, DomainPath); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, FixedNodeId, false); + FLUENT_SETTING_DEPRECATED(TString, Path); }; struct TEndpointInfo { diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp index 2b23fecedcd8..a7e299b2da0d 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp @@ -13,7 +13,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { using NYdbGrpc::TGRpcClientLow; using NYdbGrpc::TServiceConnection; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.h b/ydb/public/sdk/cpp/client/ydb_driver/driver.h index f4589bd367ab..fd8bb8c95af6 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.h +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include #include @@ -11,7 +13,7 @@ //////////////////////////////////////////////////////////////////////////////// -namespace NYdb { +namespace NYdb::inline V2 { class TDriver; class TGRpcConnectionsImpl; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/fwd.h b/ydb/public/sdk/cpp/client/ydb_driver/fwd.h new file mode 100644 index 000000000000..636d953cd76a --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_driver/fwd.h @@ -0,0 +1,8 @@ +#pragma once + +namespace NYdb::inline V2 { + +class TDriver; +class TDriverConfig; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp index d9c609da968f..3d2ef52f726c 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp +++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp @@ -15,7 +15,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NExport { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.h b/ydb/public/sdk/cpp/client/ydb_export/export.h index ed63e9864766..b7eb2957473e 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.h +++ b/ydb/public/sdk/cpp/client/ydb_export/export.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NExport { /// Common @@ -34,13 +34,13 @@ struct TExportToYtSettings : public TOperationRequestSettings #undef INCLUDE_YDB_INTERNAL_H -namespace NYdb { +namespace NYdb::inline V2 { void IExtension::SelfRegister(TDriver driver) { CreateInternalInterface(driver)->RegisterExtension(this); diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.h b/ydb/public/sdk/cpp/client/ydb_extension/extension.h index 2a00b7875f2c..9938513ef81c 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/extension.h +++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.h @@ -12,7 +12,7 @@ class ListEndpointsResult; } } -namespace NYdb { +namespace NYdb::inline V2 { class IExtensionApi { public: diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/federated_topic.h b/ydb/public/sdk/cpp/client/ydb_federated_topic/federated_topic.h index 1db59f14a28a..3829b02f05ee 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/federated_topic.h +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/federated_topic.h @@ -8,7 +8,7 @@ #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { using NTopic::TPrintable; using TDbInfo = Ydb::FederationDiscovery::DatabaseInfo; @@ -252,11 +252,11 @@ struct TFederatedWriteSessionSettings : public NTopic::TWriteSessionSettings { //! Preferred database //! If specified database is unavailable, session will write to other database. - FLUENT_SETTING_OPTIONAL(TString, PreferredDatabase); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PreferredDatabase); //! Write to other databases if there are problems with connection //! to the preferred one. - FLUENT_SETTING_DEFAULT(bool, AllowFallback, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowFallback, true); TFederatedWriteSessionSettings() = default; TFederatedWriteSessionSettings(const TFederatedWriteSessionSettings&) = default; @@ -324,77 +324,77 @@ struct TFederatedReadSessionSettings: public NTopic::TReadSessionSettings { //! Data size limit for the DataReceivedHandler handler. //! The data size may exceed this limit. - FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, Max()); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxMessagesBytes, Max()); //! Function to handle data events. //! If this handler is set, data events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, DataReceivedHandler); + FLUENT_SETTING_DEPRECATED(std::function, DataReceivedHandler); //! Function to handle commit ack events. //! If this handler is set, commit ack events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, CommitOffsetAcknowledgementHandler); //! Function to handle start partition session events. //! If this handler is set, create partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, StartPartitionSessionHandler); //! Function to handle stop partition session events. //! If this handler is set, destroy partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, StopPartitionSessionHandler); //! Function to handle end partition session events. //! If this handler is set, end partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, EndPartitionSessionHandler); //! Function to handle partition session status events. //! If this handler is set, partition session status events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, PartitionSessionStatusHandler); //! Function to handle partition session closed events. //! If this handler is set, partition session closed events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, PartitionSessionClosedHandler); //! Function to handle session closed events. //! If this handler is set, close session events will be handled by handler //! and then sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(NTopic::TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING_DEPRECATED(NTopic::TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, //! this handler (if specified) will be used. //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. - FLUENT_SETTING(std::function, CommonHandler); + FLUENT_SETTING_DEPRECATED(std::function, CommonHandler); //! Executor for handlers. //! If not set, default single threaded executor will be used. //! Shared between subsessions - FLUENT_SETTING(NTopic::IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING_DEPRECATED(NTopic::IExecutor::TPtr, HandlersExecutor); }; //! Federated event handlers. //! See description in TFederatedEventHandlers class. - FLUENT_SETTING(TFederatedEventHandlers, FederatedEventHandlers); + FLUENT_SETTING_DEPRECATED(TFederatedEventHandlers, FederatedEventHandlers); //! Read policy settings @@ -486,16 +486,16 @@ struct TFederatedTopicClientSettings : public TCommonClientSettingsBase #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { NTopic::TTopicClientSettings FromFederated(const TFederatedTopicClientSettings& settings); diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session.h b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session.h index 1db9d23ca413..46c118b02388 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session.h @@ -5,7 +5,7 @@ #include #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { class TEventFederator { public: diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session_event.cpp b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session_event.cpp index dfe5f8616997..14536257b72c 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session_event.cpp +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_read_session_event.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Printable specializations -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { using namespace NFederatedTopic; @@ -136,7 +136,7 @@ void TPrintable::DebugString(TStringBuilder& ret, bool print } -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // NFederatedTopic::TReadSessionEvent::TDataReceivedEvent diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic.cpp b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic.cpp index 266aeb953976..00aebe198e39 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic.cpp +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic.cpp @@ -1,7 +1,7 @@ #include #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { // TFederatedReadSessionSettings // Read policy settings diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.cpp b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.cpp index 5071a31a6477..cda82416d584 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.cpp @@ -3,7 +3,7 @@ #include "federated_read_session.h" #include "federated_write_session.h" -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { std::shared_ptr TFederatedTopicClient::TImpl::CreateReadSession(const TFederatedReadSessionSettings& settings) { diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.h b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.h index 58320dd23f8e..ca29eafe1749 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_topic_impl.h @@ -12,7 +12,7 @@ #include #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { class TFederatedTopicClient::TImpl { public: diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.cpp b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.cpp index 532decb2d109..6f221bc5202c 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.cpp @@ -11,7 +11,7 @@ #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { constexpr TDuration UPDATE_FEDERATION_STATE_DELAY = TDuration::Seconds(10); diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.h b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.h index 7194a541f005..3eed3cc9b9f7 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federated_write_session.h @@ -6,7 +6,7 @@ #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { std::pair, EStatus> SelectDatabaseByHashImpl( NTopic::TFederatedWriteSessionSettings const& settings, diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.cpp b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.cpp index 0de268d16dd2..a77e3761890c 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.cpp +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { constexpr TDuration REDISCOVERY_DELAY = TDuration::Seconds(30); diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.h b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.h index ffa3ca8cfa52..011ca9aaef08 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.h +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/impl/federation_observer.h @@ -17,7 +17,7 @@ #include #include -namespace NYdb::NFederatedTopic { +namespace NYdb::inline V2::NFederatedTopic { struct TFederatedDbState { public: diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/ya.make b/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/ya.make deleted file mode 100644 index 2c65325016f8..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/ya.make +++ /dev/null @@ -1,8 +0,0 @@ -LIBRARY() - -SRCS( - fds_mock.h -) - -END() - diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/ya.make deleted file mode 100644 index ca2fed5a38f6..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/ya.make +++ /dev/null @@ -1,37 +0,0 @@ -UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_federated_topic) - -IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( - library/cpp/testing/gmock_in_unittest - ydb/core/testlib/default - ydb/public/lib/json_value - ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_persqueue_public/include - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - - ydb/public/sdk/cpp/client/ydb_topic/codecs - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_topic/impl - - ydb/public/sdk/cpp/client/ydb_federated_topic - ydb/public/sdk/cpp/client/ydb_federated_topic/impl - ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock -) - -YQL_LAST_ABI_VERSION() - -SRCS( - basic_usage_ut.cpp -) - -END() diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/ya.make b/ydb/public/sdk/cpp/client/ydb_federated_topic/ya.make index 217088f2e0f2..fac1f1a50682 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_federated_topic/ya.make @@ -12,7 +12,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp index 47fc97e906d1..4a02c928a306 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp +++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp @@ -10,7 +10,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NImport { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h index 162e973e0e32..23247edf56c0 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.h +++ b/ydb/public/sdk/cpp/client/ydb_import/import.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NImport { /// Common @@ -38,11 +38,11 @@ struct TImportFromS3Settings : public TOperationRequestSettings { using TSelf = TImportYdbDumpDataSettings; - FLUENT_SETTING_VECTOR(TString, Columns); + FLUENT_SETTING_VECTOR_DEPRECATED(TString, Columns); using TOperationRequestSettings::TOperationRequestSettings; }; diff --git a/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.cpp b/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.cpp index 701eb2453408..6aeb1c6a31e2 100644 --- a/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.cpp +++ b/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.cpp @@ -9,7 +9,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NMonitoring { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.h b/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.h index 834e2bcbac8f..f17070fef3f5 100644 --- a/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.h +++ b/ydb/public/sdk/cpp/client/ydb_monitoring/monitoring.h @@ -8,7 +8,7 @@ namespace Monitoring { } } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; @@ -27,9 +27,9 @@ enum class EStatusFlag { }; struct TSelfCheckSettings : public TOperationRequestSettings{ - FLUENT_SETTING_OPTIONAL(bool, ReturnVerboseStatus); - FLUENT_SETTING_OPTIONAL(EStatusFlag, MinimumStatus); - FLUENT_SETTING_OPTIONAL(ui32, MaximumLevel); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, ReturnVerboseStatus); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EStatusFlag, MinimumStatus); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, MaximumLevel); }; class TSelfCheckResult : public TStatus { diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp index 6c0930ca7053..65d37c9f9971 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp @@ -15,7 +15,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NOperation { constexpr TDuration OPERATION_CLIENT_TIMEOUT = TDuration::Seconds(5); @@ -133,21 +133,21 @@ TOperationClient::TOperationClient(const TDriver& driver, const TCommonClientSet template TFuture TOperationClient::Get(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); + request.set_id(id.ToString()); return Impl_->Get(std::move(request)); } TAsyncStatus TOperationClient::Cancel(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); + request.set_id(id.ToString()); return Impl_->Cancel(std::move(request)); } TAsyncStatus TOperationClient::Forget(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); + request.set_id(id.ToString()); return Impl_->Forget(std::move(request)); } diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.h b/ydb/public/sdk/cpp/client/ydb_operation/operation.h index 794741331fc4..6540d4f57624 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NOperation { template diff --git a/ydb/public/sdk/cpp/client/ydb_params/fwd.h b/ydb/public/sdk/cpp/client/ydb_params/fwd.h new file mode 100644 index 000000000000..f1abe4330786 --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_params/fwd.h @@ -0,0 +1,9 @@ +#pragma once + +namespace NYdb::inline V2 { + +class TParams; +class TParamValueBuilder; +class TParamsBuilder; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp index 9e8c5e1e7d72..5f2bcd578a37 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp @@ -6,7 +6,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TParams::TImpl::TImpl(::google::protobuf::Map&& paramsMap) { ParamsMap_.swap(paramsMap); diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.h b/ydb/public/sdk/cpp/client/ydb_params/impl.h index 9e38d2463dde..4b2db57338d9 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.h +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.h @@ -2,7 +2,7 @@ #include "params.h" -namespace NYdb { +namespace NYdb::inline V2 { class TParams::TImpl { public: diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.cpp b/ydb/public/sdk/cpp/client/ydb_params/params.cpp index 8b066096e4be..b1d4f327285a 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/params.cpp @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.h b/ydb/public/sdk/cpp/client/ydb_params/params.h index 5d29822f1805..b0fcfe9691b1 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.h +++ b/ydb/public/sdk/cpp/client/ydb_params/params.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include @@ -8,7 +10,7 @@ namespace Ydb { class TypedValue; } -namespace NYdb { +namespace NYdb::inline V2 { namespace NScripting { class TScriptingClient; @@ -42,7 +44,7 @@ class TParams { friend class NExperimental::TStreamQueryClient; friend class NQuery::TExecQueryImpl; friend class NQuery::TQueryClient; - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; public: bool Empty() const; diff --git a/ydb/public/sdk/cpp/client/ydb_params/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ya.make index 32f82f820b53..d72c8d0d9d0c 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_params/ya.make @@ -12,7 +12,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h deleted file mode 100644 index 6e19834687d5..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h deleted file mode 100644 index 916bc5e459e5..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ut_utils.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ut_utils.h deleted file mode 100644 index a314c8863e0a..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ut_utils.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make deleted file mode 100644 index 5bd70168ac34..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -LIBRARY() - -SRCS( - data_plane_helpers.h - test_server.h - ut_utils.h -) - -PEERDIR( - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils -) - -END() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make deleted file mode 100644 index befc840da75d..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make +++ /dev/null @@ -1,41 +0,0 @@ -UNITTEST() - -IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( - library/cpp/testing/gmock_in_unittest - ydb/core/testlib/default - ydb/public/lib/json_value - ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_persqueue_public/impl - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_topic/codecs - ydb/public/sdk/cpp/client/ydb_topic/impl -) - -YQL_LAST_ABI_VERSION() - -SRCDIR( - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut - ydb/public/sdk/cpp/client/ydb_persqueue_public -) - -SRCS( - common_ut.cpp - read_session_ut.cpp - basic_usage_ut.cpp - compress_executor_ut.cpp - compression_ut.cpp - retry_policy_ut.cpp -) - -END() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make index 31542832f38e..08231494a336 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make @@ -10,7 +10,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/aliases.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/aliases.h index 448c4595931c..44652936ed4e 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/aliases.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/aliases.h @@ -5,7 +5,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { // codecs using NTopic::ICodec; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.cpp index bdbbb6a44c71..aa9900434806 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { ERetryErrorClass GetRetryErrorClass(EStatus status) { switch (status) { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.h index e0308e24aa66..a8d1f33cdc4a 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/common.h @@ -4,7 +4,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { ERetryErrorClass GetRetryErrorClass(EStatus status); ERetryErrorClass GetRetryErrorClassV2(EStatus status); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue.cpp index 1cac026d12ce..dcd75785fe75 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue.cpp @@ -9,7 +9,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { class TCommonCodecsProvider { public: diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.cpp index a3ec1032c141..ba554ce104f2 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.cpp @@ -3,7 +3,7 @@ #include "read_session.h" #include "write_session.h" -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { std::shared_ptr TPersQueueClient::TImpl::CreateReadSession(const TReadSessionSettings& settings) { TMaybe maybeSettings; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.h index 519c3df9ef5f..6168093e9cf6 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/persqueue_impl.h @@ -11,7 +11,7 @@ #undef INCLUDE_YDB_INTERNAL_H -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { class TPersQueueClient::TImpl : public TClientImplCommon { public: diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.cpp index 7267b795e70f..fa8d5a70df42 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.cpp @@ -8,7 +8,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { static const TString DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.h index 7e2f78d116e2..2f009e1fcf9c 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session.h @@ -6,7 +6,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { // High level class that manages several read session impls. // Each one of them works with single cluster. diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session_messages.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session_messages.cpp index fe6e7adeadc2..c3400c420560 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session_messages.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/read_session_messages.cpp @@ -4,7 +4,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation( ui64 offset, diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.cpp index ae8d3864e738..19cfb096b6e4 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.cpp @@ -8,7 +8,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TWriteSession diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.h index 0dd73f0987f6..2f2b8288f47f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session.h @@ -8,7 +8,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { namespace NTests { class TSimpleWriteSessionTestAdapter; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.cpp index 44868795a4c1..68a9495f1bb9 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.cpp @@ -9,7 +9,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.h index 0c612083e481..0971aab5da44 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/impl/write_session_impl.h @@ -7,7 +7,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TWriteSessionEventsQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/aliases.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/aliases.h index 2df49c31eb78..acba0308e027 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/aliases.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/aliases.h @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { // codecs using NTopic::ECodec; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/client.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/client.h index b85d58265a8c..901acae794b5 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/client.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/client.h @@ -7,19 +7,19 @@ #include #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { struct TPersQueueClientSettings : public TCommonClientSettingsBase { using TSelf = TPersQueueClientSettings; //! Default executor for compression tasks. - FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); + FLUENT_SETTING_DEFAULT_DEPRECATED(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); //! Default executor for callbacks. - FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); + FLUENT_SETTING_DEFAULT_DEPRECATED(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); //! Manages cluster discovery mode. - FLUENT_SETTING_DEFAULT(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); + FLUENT_SETTING_DEFAULT_DEPRECATED(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); }; // PersQueue client. diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/control_plane.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/control_plane.h index a6a487e30af7..9cd970cb1692 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/control_plane.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/control_plane.h @@ -11,11 +11,11 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; } -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { enum class EFormat { BASE = 1, @@ -176,14 +176,14 @@ const TVector& GetDefaultCodecs(); struct TReadRuleSettings { TReadRuleSettings() {} using TSelf = TReadRuleSettings; - FLUENT_SETTING(TString, ConsumerName); - FLUENT_SETTING_DEFAULT(bool, Important, false); - FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); - FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) - FLUENT_SETTING_DEFAULT(TVector, SupportedCodecs, GetDefaultCodecs()); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Important, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(TInstant, StartingMessageTimestamp, TInstant::Zero()); + FLUENT_SETTING_DEFAULT_DEPRECATED(EFormat, SupportedFormat, EFormat::BASE) + FLUENT_SETTING_DEFAULT_DEPRECATED(TVector, SupportedCodecs, GetDefaultCodecs()); - FLUENT_SETTING_DEFAULT(ui32, Version, 0); - FLUENT_SETTING(TString, ServiceType); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, Version, 0); + FLUENT_SETTING_DEPRECATED(TString, ServiceType); TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) { ConsumerName_ = settings.ConsumerName(); @@ -209,12 +209,12 @@ struct TTopicSettings : public TOperationRequestSettings { struct TRemoteMirrorRuleSettings { TRemoteMirrorRuleSettings() {} using TSelf = TRemoteMirrorRuleSettings; - FLUENT_SETTING(TString, Endpoint); - FLUENT_SETTING(TString, TopicPath); - FLUENT_SETTING(TString, ConsumerName); - FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); - FLUENT_SETTING(TCredentials, Credentials); - FLUENT_SETTING(TString, Database); + FLUENT_SETTING_DEPRECATED(TString, Endpoint); + FLUENT_SETTING_DEPRECATED(TString, TopicPath); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); + FLUENT_SETTING_DEFAULT_DEPRECATED(TInstant, StartingMessageTimestamp, TInstant::Zero()); + FLUENT_SETTING_DEPRECATED(TCredentials, Credentials); + FLUENT_SETTING_DEPRECATED(TString, Database); TRemoteMirrorRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TRemoteMirrorRule& settings) { Endpoint_ = settings.Endpoint(); @@ -230,28 +230,28 @@ struct TTopicSettings : public TOperationRequestSettings { using TSelf = TDerived; - FLUENT_SETTING_DEFAULT(ui32, PartitionsCount, 1); - FLUENT_SETTING_DEFAULT(TDuration, RetentionPeriod, TDuration::Hours(18)); - FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) - FLUENT_SETTING_DEFAULT(TVector, SupportedCodecs, GetDefaultCodecs()); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, PartitionsCount, 1); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, RetentionPeriod, TDuration::Hours(18)); + FLUENT_SETTING_DEFAULT_DEPRECATED(EFormat, SupportedFormat, EFormat::BASE) + FLUENT_SETTING_DEFAULT_DEPRECATED(TVector, SupportedCodecs, GetDefaultCodecs()); - FLUENT_SETTING_DEFAULT(ui64, MaxPartitionStorageSize, 0); - FLUENT_SETTING_DEFAULT(ui64, MaxPartitionWriteSpeed, 2_MB); - FLUENT_SETTING_DEFAULT(ui64, MaxPartitionWriteBurst, 2_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, MaxPartitionStorageSize, 0); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, MaxPartitionWriteSpeed, 2_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, MaxPartitionWriteBurst, 2_MB); - FLUENT_SETTING_DEFAULT(bool, ClientWriteDisabled, false); - FLUENT_SETTING_DEFAULT(bool, AllowUnauthenticatedWrite, false); - FLUENT_SETTING_DEFAULT(bool, AllowUnauthenticatedRead, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, ClientWriteDisabled, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowUnauthenticatedWrite, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowUnauthenticatedRead, false); - FLUENT_SETTING_OPTIONAL(ui32, PartitionsPerTablet); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, PartitionsPerTablet); - FLUENT_SETTING_OPTIONAL(ui32, AbcId); - FLUENT_SETTING_OPTIONAL(TString, AbcSlug); - FLUENT_SETTING_OPTIONAL(TString, FederationAccount); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, AbcId); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, AbcSlug); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, FederationAccount); - //TODO: FLUENT_SETTING_VECTOR - FLUENT_SETTING_DEFAULT(TVector, ReadRules, {}); - FLUENT_SETTING_OPTIONAL(TRemoteMirrorRuleSettings, RemoteMirrorRule); + //TODO: FLUENT_SETTING_VECTOR_DEPRECATED + FLUENT_SETTING_DEFAULT_DEPRECATED(TVector, ReadRules, {}); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TRemoteMirrorRuleSettings, RemoteMirrorRule); TSelf& SetSettings(const TDescribeTopicResult::TTopicSettings& settings) { @@ -316,12 +316,12 @@ struct TDescribeTopicSettings : public TOperationRequestSettings { - FLUENT_SETTING(TReadRuleSettings, ReadRule); + FLUENT_SETTING_DEPRECATED(TReadRuleSettings, ReadRule); }; // Settings for remove read rule request struct TRemoveReadRuleSettings : public TOperationRequestSettings { - FLUENT_SETTING(TString, ConsumerName); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); }; } // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_events.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_events.h index 35edb8749ba8..c380edeeb29c 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_events.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_events.h @@ -4,7 +4,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { //! Partition stream. struct TPartitionStream : public TThrRefBase { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_session.h index a92d1b2862b0..c6b85491def9 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/read_session.h @@ -11,7 +11,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { //! Read settings for single topic. struct TTopicReadSettings { @@ -28,14 +28,14 @@ struct TTopicReadSettings { TTopicReadSettings& operator=(TTopicReadSettings&&) = default; //! Path of topic to read. - FLUENT_SETTING(TString, Path); + FLUENT_SETTING_DEPRECATED(TString, Path); //! Start reading from this timestamp. - FLUENT_SETTING_OPTIONAL(TInstant, StartingMessageTimestamp); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, StartingMessageTimestamp); //! Partition groups to read. //! 1-based. - FLUENT_SETTING_VECTOR(ui64, PartitionGroupIds); + FLUENT_SETTING_VECTOR_DEPRECATED(ui64, PartitionGroupIds); }; //! Settings for read session. @@ -65,66 +65,66 @@ struct TReadSessionSettings : public TRequestSettings { //! Data size limit for the DataReceivedHandler handler. //! The data size may exceed this limit. - FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, Max()); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxMessagesBytes, Max()); //! Function to handle data events. //! If this handler is set, data events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, DataReceivedHandler); + FLUENT_SETTING_DEPRECATED(std::function, DataReceivedHandler); //! Function to handle commit ack events. //! If this handler is set, commit ack events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, CommitAcknowledgementHandler); + FLUENT_SETTING_DEPRECATED(std::function, CommitAcknowledgementHandler); //! Function to handle create partition stream events. //! If this handler is set, create partition stream events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, CreatePartitionStreamHandler); + FLUENT_SETTING_DEPRECATED(std::function, CreatePartitionStreamHandler); //! Function to handle destroy partition stream events. //! If this handler is set, destroy partition stream events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, DestroyPartitionStreamHandler); + FLUENT_SETTING_DEPRECATED(std::function, DestroyPartitionStreamHandler); //! Function to handle partition stream status events. //! If this handler is set, partition stream status events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, PartitionStreamStatusHandler); + FLUENT_SETTING_DEPRECATED(std::function, PartitionStreamStatusHandler); //! Function to handle partition stream closed events. //! If this handler is set, partition stream closed events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, PartitionStreamClosedHandler); + FLUENT_SETTING_DEPRECATED(std::function, PartitionStreamClosedHandler); //! Function to handle session closed events. //! If this handler is set, close session events will be handled by handler //! and then sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING_DEPRECATED(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, //! this handler (if specified) will be used. //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. - FLUENT_SETTING(std::function, CommonHandler); + FLUENT_SETTING_DEPRECATED(std::function, CommonHandler); //! Executor for handlers. //! If not set, default single threaded executor will be used. - FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, HandlersExecutor); }; //! Consumer. - FLUENT_SETTING(TString, ConsumerName); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); //! Topics. - FLUENT_SETTING_VECTOR(TTopicReadSettings, Topics); + FLUENT_SETTING_VECTOR_DEPRECATED(TTopicReadSettings, Topics); //! Default variant. //! Read topic instance specified in "Topics" from all clusters. @@ -147,41 +147,41 @@ struct TReadSessionSettings : public TRequestSettings { //! Disable Clusters discovery. ReadMirrored/ReadOriginal/ReadAll will not have any effect //! if this option is true. - FLUENT_SETTING_DEFAULT(bool, DisableClusterDiscovery, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, DisableClusterDiscovery, false); //! Maximum memory usage for read session. - FLUENT_SETTING_DEFAULT(size_t, MaxMemoryUsageBytes, 100_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxMemoryUsageBytes, 100_MB); //! Max message time lag. All messages older that now - MaxTimeLag will be ignored. - FLUENT_SETTING_OPTIONAL(TDuration, MaxTimeLag); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, MaxTimeLag); //! Start reading from this timestamp. - FLUENT_SETTING_OPTIONAL(TInstant, StartingMessageTimestamp); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, StartingMessageTimestamp); //! Policy for reconnections. //! IRetryPolicy::GetDefaultPolicy() if null (not set). - FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + FLUENT_SETTING_DEPRECATED(IRetryPolicy::TPtr, RetryPolicy); //! Event handlers. //! See description in TEventHandlers class. - FLUENT_SETTING(TEventHandlers, EventHandlers); + FLUENT_SETTING_DEPRECATED(TEventHandlers, EventHandlers); //! Decompress messages - FLUENT_SETTING_DEFAULT(bool, Decompress, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Decompress, true); //! Executor for decompression tasks. //! If not set, default executor will be used. - FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, DecompressionExecutor); //! Counters. //! If counters are not provided explicitly, //! they will be created inside session (without link with parent counters). - FLUENT_SETTING(TReaderCounters::TPtr, Counters); + FLUENT_SETTING_DEPRECATED(TReaderCounters::TPtr, Counters); //! Read only original topic instance, don't read mirrored. //! //! It's better to control this setting via ReadAll()/ReadMirrored()/ReadOriginal() helpers. - FLUENT_SETTING_DEFAULT(bool, ReadOnlyOriginal, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, ReadOnlyOriginal, true); //! Read topics from specified clusters. //! @@ -193,15 +193,15 @@ struct TReadSessionSettings : public TRequestSettings { //! Use ReadOriginal() function for this variant. //! 3. If ReadOnlyOriginal is false and one cluster is specified read will be done from all topic instances (mirrored and original) in one cluster. //! Use ReadMirrored() function for this variant. - FLUENT_SETTING_VECTOR(TString, Clusters); + FLUENT_SETTING_VECTOR_DEPRECATED(TString, Clusters); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Seconds(30)); //! Experimental option - FLUENT_SETTING_OPTIONAL(bool, RangesMode); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, RangesMode); //! Log. - FLUENT_SETTING_OPTIONAL(TLog, Log); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TLog, Log); }; class IReadSession { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_events.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_events.h index 34adf0788e3c..0fecf6bc7f8b 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_events.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_events.h @@ -6,7 +6,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { struct TWriteStat : public TThrRefBase { TDuration WriteTime; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_session.h index d1706eb15aa3..9a265701b8af 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_session.h @@ -8,7 +8,7 @@ #include -namespace NYdb::NPersQueue { +namespace NYdb::inline V2::NPersQueue { enum class EClusterDiscoveryMode { Auto = 0, // enables cluster discovery only for hostname "logbroker.yandex.net" and "logbroker-prestable.yandex.net" @@ -32,38 +32,38 @@ struct TWriteSessionSettings : public TRequestSettings { TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; //! Path of topic to write. - FLUENT_SETTING(TString, Path); + FLUENT_SETTING_DEPRECATED(TString, Path); //! MessageGroupId (aka SourceId) to use. - FLUENT_SETTING(TString, MessageGroupId); + FLUENT_SETTING_DEPRECATED(TString, MessageGroupId); //! Write to an exact partition group. Generally server assigns partition group automatically. //! Using this option is not recommended unless you know for sure why you need it. - FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, PartitionGroupId); //! Preferred LB cluster. Used for multi-cluster installation. //! If specified cluster is unavailable, session will write to other cluster. - FLUENT_SETTING_OPTIONAL(TString, PreferredCluster); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PreferredCluster); //! Write to other clusters if there are problems with connection //! to the first one. - FLUENT_SETTING_DEFAULT(bool, AllowFallbackToOtherClusters, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowFallbackToOtherClusters, true); //! codec and level to use for data compression prior to write. - FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); - FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4); + FLUENT_SETTING_DEFAULT_DEPRECATED(ECodec, Codec, ECodec::GZIP); + FLUENT_SETTING_DEFAULT_DEPRECATED(i32, CompressionLevel, 4); //! Writer will not accept new messages if memory usage exceeds this limit. //! Memory usage consists of raw data pending compression and compressed messages being sent. - FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, MaxMemoryUsage, 20_MB); //! Maximum messages accepted by writer but not written (with confirmation from server). //! Writer will not accept new messages after reaching the limit. - FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MaxInflightCount, 100000); //! Retry policy enables automatic retries for non-fatal errors. //! IRetryPolicy::GetDefaultPolicy() if null (not set). - FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + FLUENT_SETTING_DEPRECATED(IRetryPolicy::TPtr, RetryPolicy); //! User metadata that may be attached to write session. TWriteSessionSettings& AppendSessionMeta(const TString& key, const TString& value) { @@ -79,16 +79,16 @@ struct TWriteSessionSettings : public TRequestSettings { //! Greatly increases performance for small messages. //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients //! sending small messages at high rate). - FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); - FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, BatchFlushInterval); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, BatchFlushSizeBytes); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Seconds(30)); - FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TWriterCounters::TPtr, Counters); //! Executor for compression tasks. //! If not set, default executor will be used. - FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, CompressionExecutor); struct TEventHandlers { using TSelf = TEventHandlers; @@ -98,27 +98,27 @@ struct TWriteSessionSettings : public TRequestSettings { //! Function to handle Acks events. //! If this handler is set, write ack events will be handled by handler, //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TWriteAckHandler, AcksHandler); + FLUENT_SETTING_DEPRECATED(TWriteAckHandler, AcksHandler); //! Function to handle ReadyToAccept event. //! If this handler is set, write these events will be handled by handler, //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHandler); + FLUENT_SETTING_DEPRECATED(TReadyToAcceptHandler, ReadyToAcceptHandler); //! Function to handle close session events. //! If this handler is set, close session events will be handled by handler //! and then sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING_DEPRECATED(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, //! this handler (if specified) will be used. //! If this handler is not specified, event can be received with TWriteSession::GetEvent() method. - FLUENT_SETTING(std::function, CommonHandler); + FLUENT_SETTING_DEPRECATED(std::function, CommonHandler); //! Executor for handlers. //! If not set, default single threaded executor will be used. - FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, HandlersExecutor); [[deprecated("Typo in name. Use ReadyToAcceptHandler instead.")]] TSelf& ReadyToAcceptHander(const TReadyToAcceptHandler& value) { @@ -127,13 +127,13 @@ struct TWriteSessionSettings : public TRequestSettings { }; //! Event handlers. - FLUENT_SETTING(TEventHandlers, EventHandlers); + FLUENT_SETTING_DEPRECATED(TEventHandlers, EventHandlers); //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception. - FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, ValidateSeqNo, true); //! Manages cluster discovery mode. - FLUENT_SETTING_DEFAULT(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); + FLUENT_SETTING_DEFAULT_DEPRECATED(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); }; //! Simple write session. Does not need event handlers. Does not provide Events, ContinuationTokens, write Acks. diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_public/ya.make index 4af769994e10..253c58c437c4 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/ya.make @@ -22,7 +22,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index f51b8101600e..5f82e9236c14 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { const Ydb::Type& TProtoAccessor::GetProto(const TType& type) { return type.GetProto(); diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h index 7c609bb01176..afcfd3cba750 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h @@ -21,7 +21,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class TResultSet; class TValue; diff --git a/ydb/public/sdk/cpp/client/ydb_proto/ya.make b/ydb/public/sdk/cpp/client/ydb_proto/ya.make index c753c6e08cf4..34dbefecea9f 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_proto/ya.make @@ -8,7 +8,7 @@ PEERDIR( ydb/public/api/grpc ydb/public/api/grpc/draft ydb/public/api/protos - ydb/public/lib/operation_id/protos + ydb/public/sdk/cpp/src/library/operation_id ydb/public/sdk/cpp/client/ydb_params ydb/public/sdk/cpp/client/ydb_value yql/essentials/public/issue/protos diff --git a/ydb/public/sdk/cpp/client/ydb_query/client.cpp b/ydb/public/sdk/cpp/client/ydb_query/client.cpp index 716cacdfc3b9..e606fa7b8167 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/client.cpp +++ b/ydb/public/sdk/cpp/client/ydb_query/client.cpp @@ -16,7 +16,7 @@ #include #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { using TRetryContextResultAsync = NRetry::Async::TRetryContext; using TRetryContextAsync = NRetry::Async::TRetryContext; @@ -130,7 +130,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncFetchScriptResultsResult FetchScriptResults(const NKikimr::NOperationId::TOperationId& operationId, int64_t resultSetIndex, const TFetchScriptResultsSettings& settings) { auto request = MakeRequest(); - request.set_operation_id(NKikimr::NOperationId::ProtoToString(operationId)); + request.set_operation_id(operationId.ToString()); request.set_result_set_index(resultSetIndex); return FetchScriptResultsImpl(std::move(request), settings); } diff --git a/ydb/public/sdk/cpp/client/ydb_query/client.h b/ydb/public/sdk/cpp/client/ydb_query/client.h index a459c4b3981a..5a37cdb9b497 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/client.h +++ b/ydb/public/sdk/cpp/client/ydb_query/client.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include "query.h" #include "tx.h" @@ -11,7 +13,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; namespace NRetry::Async { @@ -24,7 +26,7 @@ namespace NYdb { } // namespace NRetry::Sync } -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { struct TCreateSessionSettings : public TSimpleRequestSettings { TCreateSessionSettings(); @@ -38,20 +40,20 @@ struct TSessionPoolSettings { using TSelf = TSessionPoolSettings; // Max number of sessions client can get from session pool - FLUENT_SETTING_DEFAULT(ui32, MaxActiveSessions, 50); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MaxActiveSessions, 50); // Max time session to be in idle state before closing - FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); // Min number of session in session pool. // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. - FLUENT_SETTING_DEFAULT(ui32, MinPoolSize, 10); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MinPoolSize, 10); }; struct TClientSettings : public TCommonClientSettingsBase { using TSessionPoolSettings = TSessionPoolSettings; using TSelf = TClientSettings; - FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); + FLUENT_SETTING_DEPRECATED(TSessionPoolSettings, SessionPoolSettings); }; // ! WARNING: Experimental API diff --git a/ydb/public/sdk/cpp/client/ydb_query/fwd.h b/ydb/public/sdk/cpp/client/ydb_query/fwd.h new file mode 100644 index 000000000000..4a103bb515fb --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_query/fwd.h @@ -0,0 +1,37 @@ +#pragma once + +namespace NYdb::inline V2::NQuery { + +struct TClientSettings; +struct TSessionPoolSettings; +struct TCreateSessionSettings; +struct TExecuteQuerySettings; +struct TBeginTxSettings; +struct TCommitTxSettings; +struct TRollbackTxSettings; +struct TExecuteScriptSettings; +struct TFetchScriptResultsSettings; +struct TTxOnlineSettings; +struct TTxSettings; + +class TQueryClient; +class TSession; + +class TCreateSessionResult; +class TBeginTransactionResult; +class TExecuteQueryResult; +class TCommitTransactionResult; +class TFetchScriptResultsResult; + +class TExecuteQueryPart; +class TExecuteQueryIterator; + +class TTransaction; +struct TTxControl; + +class TQueryContent; +class TResultSetMeta; +class TScriptExecutionOperation; +class TExecStats; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.cpp b/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.cpp index 807be98c2e48..dd633abba432 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.cpp @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { // Custom lock primitive to protect session from destroying // during async read execution. diff --git a/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.h b/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.h index 40d6379df2a8..d4000a33ab85 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.h +++ b/ydb/public/sdk/cpp/client/ydb_query/impl/client_session.h @@ -5,7 +5,7 @@ #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { class TSafeTSessionImplHolder; diff --git a/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.cpp b/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.cpp index 897a7ee8a3c8..971712a474d5 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.cpp +++ b/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.cpp @@ -13,7 +13,7 @@ #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.h b/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.h index e455ac1cdc97..df8c9597fe88 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.h +++ b/ydb/public/sdk/cpp/client/ydb_query/impl/exec_query.h @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { class TExecQueryImpl { public: diff --git a/ydb/public/sdk/cpp/client/ydb_query/query.cpp b/ydb/public/sdk/cpp/client/ydb_query/query.cpp index 4f30e763eba9..ee3608a58cb1 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/query.cpp +++ b/ydb/public/sdk/cpp/client/ydb_query/query.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { std::optional ParseStatsMode(std::string_view statsMode) { if (statsMode == "unspecified") { diff --git a/ydb/public/sdk/cpp/client/ydb_query/query.h b/ydb/public/sdk/cpp/client/ydb_query/query.h index 3ea49527ae77..3736def97dce 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/query.h +++ b/ydb/public/sdk/cpp/client/ydb_query/query.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include #include @@ -9,7 +11,7 @@ #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { enum class ESyntax { Unspecified = 0, @@ -68,12 +70,12 @@ class TExecuteQueryIterator : public TStatus { using TAsyncExecuteQueryIterator = NThreading::TFuture; struct TExecuteQuerySettings : public TRequestSettings { - FLUENT_SETTING_OPTIONAL(ui32, OutputChunkMaxSize); - FLUENT_SETTING_DEFAULT(ESyntax, Syntax, ESyntax::YqlV1); - FLUENT_SETTING_DEFAULT(EExecMode, ExecMode, EExecMode::Execute); - FLUENT_SETTING_DEFAULT(EStatsMode, StatsMode, EStatsMode::None); - FLUENT_SETTING_OPTIONAL(bool, ConcurrentResultSets); - FLUENT_SETTING(TString, ResourcePool); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, OutputChunkMaxSize); + FLUENT_SETTING_DEFAULT_DEPRECATED(ESyntax, Syntax, ESyntax::YqlV1); + FLUENT_SETTING_DEFAULT_DEPRECATED(EExecMode, ExecMode, EExecMode::Execute); + FLUENT_SETTING_DEFAULT_DEPRECATED(EStatsMode, StatsMode, EStatsMode::None); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, ConcurrentResultSets); + FLUENT_SETTING_DEPRECATED(TString, ResourcePool); }; struct TBeginTxSettings : public TRequestSettings {}; @@ -93,11 +95,11 @@ using TAsyncBeginTransactionResult = NThreading::TFuture; struct TExecuteScriptSettings : public TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(ESyntax, Syntax, ESyntax::YqlV1); - FLUENT_SETTING_DEFAULT(EExecMode, ExecMode, EExecMode::Execute); - FLUENT_SETTING_DEFAULT(EStatsMode, StatsMode, EStatsMode::None); - FLUENT_SETTING(TDuration, ResultsTtl); - FLUENT_SETTING(TString, ResourcePool); + FLUENT_SETTING_DEFAULT_DEPRECATED(ESyntax, Syntax, ESyntax::YqlV1); + FLUENT_SETTING_DEFAULT_DEPRECATED(EExecMode, ExecMode, EExecMode::Execute); + FLUENT_SETTING_DEFAULT_DEPRECATED(EStatsMode, StatsMode, EStatsMode::None); + FLUENT_SETTING_DEPRECATED(TDuration, ResultsTtl); + FLUENT_SETTING_DEPRECATED(TString, ResourcePool); }; class TQueryContent { @@ -152,8 +154,8 @@ class TScriptExecutionOperation : public TOperation { }; struct TFetchScriptResultsSettings : public TRequestSettings { - FLUENT_SETTING(TString, FetchToken); - FLUENT_SETTING_DEFAULT(ui64, RowsLimit, 1000); + FLUENT_SETTING_DEPRECATED(TString, FetchToken); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, RowsLimit, 1000); }; class TFetchScriptResultsResult : public TStatus { diff --git a/ydb/public/sdk/cpp/client/ydb_query/stats.cpp b/ydb/public/sdk/cpp/client/ydb_query/stats.cpp index c007547d4e84..6b5c594f9332 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/stats.cpp +++ b/ydb/public/sdk/cpp/client/ydb_query/stats.cpp @@ -6,7 +6,7 @@ #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { class TExecStats::TImpl { public: diff --git a/ydb/public/sdk/cpp/client/ydb_query/stats.h b/ydb/public/sdk/cpp/client/ydb_query/stats.h index 15c8a15a5134..a789da5a55c4 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/stats.h +++ b/ydb/public/sdk/cpp/client/ydb_query/stats.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include #include @@ -12,14 +14,14 @@ namespace Ydb::TableStats { class QueryStats; } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; } -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { class TExecStats { - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; public: TExecStats() = default; diff --git a/ydb/public/sdk/cpp/client/ydb_query/tx.h b/ydb/public/sdk/cpp/client/ydb_query/tx.h index d6e291f316c3..34145bec4193 100644 --- a/ydb/public/sdk/cpp/client/ydb_query/tx.h +++ b/ydb/public/sdk/cpp/client/ydb_query/tx.h @@ -1,13 +1,15 @@ #pragma once +#include "fwd.h" + #include -namespace NYdb::NQuery { +namespace NYdb::inline V2::NQuery { struct TTxOnlineSettings { using TSelf = TTxOnlineSettings; - FLUENT_SETTING_DEFAULT(bool, AllowInconsistentReads, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowInconsistentReads, false); TTxOnlineSettings() {} }; @@ -69,7 +71,7 @@ struct TTxSettings { TS_SNAPSHOT_RW, }; - FLUENT_SETTING(TTxOnlineSettings, OnlineSettings); + FLUENT_SETTING_DEPRECATED(TTxOnlineSettings, OnlineSettings); ETransactionMode GetMode() const { return Mode_; @@ -98,7 +100,7 @@ struct TTxControl { const TMaybe TxId_; const TMaybe TxSettings_; - FLUENT_SETTING_FLAG(CommitTx); + FLUENT_SETTING_FLAG_DEPRECATED(CommitTx); bool HasTx() const { return TxId_.Defined() || TxSettings_.Defined(); } diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp index aa5efd3d82b4..d8ced7307958 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NRateLimiter { +namespace NYdb::inline V2::NRateLimiter { TListResourcesResult::TListResourcesResult(TStatus status, TVector paths) : TStatus(std::move(status)) diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h index a65936900c3d..6c5dae519cee 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h @@ -9,7 +9,7 @@ class DescribeResourceResult; class HierarchicalDrrSettings; } // namespace Ydb::RateLimiter -namespace NYdb::NRateLimiter { +namespace NYdb::inline V2::NRateLimiter { // Settings for hierarchical deficit round robin (HDRR) algorithm. template @@ -19,20 +19,20 @@ struct THierarchicalDrrSettings : public TOperationRequestSettings { // Resource consumption speed limit. // Value is required for root resource. // Must be nonnegative. - FLUENT_SETTING_OPTIONAL(double, MaxUnitsPerSecond); + FLUENT_SETTING_OPTIONAL_DEPRECATED(double, MaxUnitsPerSecond); // Maximum burst size of resource consumption across the whole cluster // divided by max_units_per_second. // Default value is 1. // This means that maximum burst size might be equal to max_units_per_second. // Must be nonnegative. - FLUENT_SETTING_OPTIONAL(double, MaxBurstSizeCoefficient); + FLUENT_SETTING_OPTIONAL_DEPRECATED(double, MaxBurstSizeCoefficient); // Prefetch in local bucket up to PrefetchCoefficient*MaxUnitsPerSecond units (full size). // Default value is inherited from parent or 0.2 for root. // Disables prefetching if any negative value is set // (It is useful to avoid bursts in case of large number of local buckets). - FLUENT_SETTING_OPTIONAL(double, PrefetchCoefficient); + FLUENT_SETTING_OPTIONAL_DEPRECATED(double, PrefetchCoefficient); void DisablePrefetching() { PrefetchCoefficient_ = -1.0; @@ -41,7 +41,7 @@ struct THierarchicalDrrSettings : public TOperationRequestSettings { // Prefetching starts if there is less than PrefetchWatermark fraction of full local bucket left. // Default value is inherited from parent or 0.75 for root. // Must be nonnegative and less than or equal to 1. - FLUENT_SETTING_OPTIONAL(double, PrefetchWatermark); + FLUENT_SETTING_OPTIONAL_DEPRECATED(double, PrefetchWatermark); }; // Settings for create resource request. @@ -60,7 +60,7 @@ struct TListResourcesSettings : public TOperationRequestSettings { using TSelf = TAcquireResourceSettings; - FLUENT_SETTING_OPTIONAL(ui64, Amount); - FLUENT_SETTING_FLAG(IsUsedAmount); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, Amount); + FLUENT_SETTING_FLAG_DEPRECATED(IsUsedAmount); }; using TAsyncListResourcesResult = NThreading::TFuture; diff --git a/ydb/public/sdk/cpp/client/ydb_result/fwd.h b/ydb/public/sdk/cpp/client/ydb_result/fwd.h new file mode 100644 index 000000000000..92e1606ccdfc --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_result/fwd.h @@ -0,0 +1,8 @@ +#pragma once + +namespace NYdb::inline V2 { + +class TResultSet; +class TResultSetParser; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp index 3ff390d4ced2..cccb0629c58c 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) { return resultSet.GetProto(); diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.cpp b/ydb/public/sdk/cpp/client/ydb_result/result.cpp index e8ba1a73191e..e2feb79d01e6 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/result.cpp @@ -10,7 +10,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { TString TColumn::ToString() const { TString result; diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.h b/ydb/public/sdk/cpp/client/ydb_result/result.h index 3f3b1b355f88..d9ac8ff9f552 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.h +++ b/ydb/public/sdk/cpp/client/ydb_result/result.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include @@ -9,7 +11,7 @@ namespace Ydb { class ResultSet; } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; @@ -31,7 +33,7 @@ bool operator!=(const TColumn& col1, const TColumn& col2); //! Collection of rows, represents result of query or part of the result in case of stream operations class TResultSet { friend class TResultSetParser; - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; public: TResultSet(const Ydb::ResultSet& proto); TResultSet(Ydb::ResultSet&& proto); diff --git a/ydb/public/sdk/cpp/client/ydb_retry/retry.h b/ydb/public/sdk/cpp/client/ydb_retry/retry.h index ff4b2cbe92ca..28aaf70a3dfb 100644 --- a/ydb/public/sdk/cpp/client/ydb_retry/retry.h +++ b/ydb/public/sdk/cpp/client/ydb_retry/retry.h @@ -3,28 +3,28 @@ #include #include -namespace NYdb::NRetry { +namespace NYdb::inline V2::NRetry { struct TBackoffSettings { using TSelf = TBackoffSettings; - FLUENT_SETTING_DEFAULT(TDuration, SlotDuration, TDuration::Seconds(1)); - FLUENT_SETTING_DEFAULT(ui32, Ceiling, 6); - FLUENT_SETTING_DEFAULT(double, UncertainRatio, 0.5); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, SlotDuration, TDuration::Seconds(1)); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, Ceiling, 6); + FLUENT_SETTING_DEFAULT_DEPRECATED(double, UncertainRatio, 0.5); }; struct TRetryOperationSettings { using TSelf = TRetryOperationSettings; - FLUENT_SETTING_DEFAULT(ui32, MaxRetries, 10); - FLUENT_SETTING_DEFAULT(bool, RetryNotFound, true); - FLUENT_SETTING_DEFAULT(TDuration, GetSessionClientTimeout, TDuration::Seconds(5)); - FLUENT_SETTING_DEFAULT(TDuration, MaxTimeout, TDuration::Max()); - FLUENT_SETTING_DEFAULT(TBackoffSettings, FastBackoffSettings, DefaultFastBackoffSettings()); - FLUENT_SETTING_DEFAULT(TBackoffSettings, SlowBackoffSettings, DefaultSlowBackoffSettings()); - FLUENT_SETTING_FLAG(Idempotent); - FLUENT_SETTING_FLAG(Verbose); - FLUENT_SETTING_FLAG(RetryUndefined); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MaxRetries, 10); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, RetryNotFound, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, GetSessionClientTimeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, MaxTimeout, TDuration::Max()); + FLUENT_SETTING_DEFAULT_DEPRECATED(TBackoffSettings, FastBackoffSettings, DefaultFastBackoffSettings()); + FLUENT_SETTING_DEFAULT_DEPRECATED(TBackoffSettings, SlowBackoffSettings, DefaultSlowBackoffSettings()); + FLUENT_SETTING_FLAG_DEPRECATED(Idempotent); + FLUENT_SETTING_FLAG_DEPRECATED(Verbose); + FLUENT_SETTING_FLAG_DEPRECATED(RetryUndefined); static TBackoffSettings DefaultFastBackoffSettings() { return TBackoffSettings() diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp index 5ef2708eb339..808d129c7f4d 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp @@ -11,7 +11,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NScheme { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h index 1a31e3e1db7d..73deb2ce14f1 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h @@ -11,7 +11,7 @@ namespace Ydb { } } -namespace NYdb { +namespace NYdb::inline V2 { namespace NScheme { //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.cpp b/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.cpp index 5848f4532d29..ee5b7a3a8d3d 100644 --- a/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.cpp +++ b/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.cpp @@ -5,7 +5,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NSchemeShard { /// YT diff --git a/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.h b/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.h index 9bab848c5283..68879064841e 100644 --- a/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.h +++ b/ydb/public/sdk/cpp/client/ydb_ss_tasks/task.h @@ -5,7 +5,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NSchemeShard { class TBackgroundProcessesResponse: public TOperation { diff --git a/ydb/public/sdk/cpp/client/ydb_table/fwd.h b/ydb/public/sdk/cpp/client/ydb_table/fwd.h new file mode 100644 index 000000000000..49495ce38dd8 --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_table/fwd.h @@ -0,0 +1,116 @@ +#pragma once + +namespace NYdb::inline V2::NTable { + +class TKeyBound; +class TKeyRange; + +struct TTableColumn; +struct TAlterTableColumn; + +struct TPartitionStats; + +struct TSequenceDescription; +class TChangefeedDescription; +class TIndexDescription; +class TColumnFamilyDescription; +class TTableDescription; + +struct TExplicitPartitions; + +struct TRenameIndex; + +struct TGlobalIndexSettings; +struct TVectorIndexSettings; +struct TKMeansTreeSettings; +struct TCreateSessionSettings; +struct TSessionPoolSettings; +struct TClientSettings; +struct TBulkUpsertSettings; +struct TReadRowsSettings; +struct TStreamExecScanQuerySettings; +struct TTxOnlineSettings; +struct TCreateTableSettings; +struct TDropTableSettings; +struct TAlterTableSettings; +struct TCopyTableSettings; +struct TCopyTablesSettings; +struct TRenameTablesSettings; +struct TDescribeTableSettings; +struct TExplainDataQuerySettings; +struct TPrepareDataQuerySettings; +struct TExecDataQuerySettings; +struct TExecSchemeQuerySettings; +struct TBeginTxSettings; +struct TCommitTxSettings; +struct TRollbackTxSettings; +struct TCloseSessionSettings; +struct TKeepAliveSettings; +struct TReadTableSettings; + +class TPartitioningSettings; +class TDateTypeColumnModeSettings; +class TValueSinceUnixEpochModeSettings; +class TTtlTierSettings; +class TTtlSettings; +class TAlterTtlSettings; +class TStorageSettings; +class TReadReplicasSettings; +class TTxSettings; + +struct TColumnFamilyPolicy; +struct TStoragePolicy; +struct TPartitioningPolicy; +struct TReplicationPolicy; + +class TBuildIndexOperation; + +class TTtlDeleteAction; +class TTtlEvictToExternalStorageAction; + +class TStorageSettingsBuilder; +class TPartitioningSettingsBuilder; +class TColumnFamilyBuilder; +class TTableStorageSettingsBuilder; +class TTableColumnFamilyBuilder; +class TTablePartitioningSettingsBuilder; +class TTableBuilder; +class TAlterStorageSettingsBuilder; +class TAlterColumnFamilyBuilder; +class TAlterTtlSettingsBuilder; +class TAlterAttributesBuilder; +class TAlterPartitioningSettingsBuilder; + +class TPrepareQueryResult; +class TExplainQueryResult; +class TDescribeTableResult; +class TDataQueryResult; +class TBeginTransactionResult; +class TCommitTransactionResult; +class TCreateSessionResult; +class TKeepAliveResult; +class TBulkUpsertResult; +class TReadRowsResult; + +template +class TSimpleStreamPart; + +class TScanQueryPart; + +class TTablePartIterator; +class TScanQueryPartIterator; + +class TReadTableSnapshot; + +class TCopyItem; +class TRenameItem; + +class TDataQuery; + +class TTransaction; +class TTxControl; + +class TSession; +class TTableClient; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp index 772e7c4aa368..f901eb83dddd 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { TSession::TImpl::TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize, bool isOwnedBySessionPool) diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h index 03f802d2cb25..9b7d0bbd3cb6 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h @@ -13,7 +13,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp index e23013efeeea..ded44d288971 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { TString GetQueryHash(const TString& text) { diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h index 2ebbd5296e9b..caa66f39728f 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { TString EncodeQuery(const TString& text, bool reversible); diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp index 11a8d4d0b176..cc54950d2519 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.h b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.h index 7ac87e3f38df..6879eca2f855 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.h @@ -12,7 +12,7 @@ #include "request_migrator.h" -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp index 21fe00039b44..0d8c902e34a5 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp @@ -4,7 +4,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NMath { diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h index b2b146047b70..b47f72dd7b86 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h @@ -11,7 +11,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NMath { diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp index 3e6dfa8d1f2f..04fa0286ce92 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp @@ -1,6 +1,6 @@ #include "table_client.h" -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.h b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.h index da681f0d959a..04e168d20f31 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.h @@ -19,7 +19,7 @@ #include "readers.h" -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { //How ofter run host scan to perform session balancing diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.cpp index 5f769d01000a..d8fcbd6e39d8 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.cpp @@ -1,7 +1,7 @@ #include "transaction.h" #include "table_client.h" -namespace NYdb::NTable { +namespace NYdb::inline V2::NTable { TTransaction::TImpl::TImpl(const TSession& session, const TString& txId) : Session_(session) diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.h b/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.h index 21cb097eebd7..114080421e2d 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/transaction.h @@ -2,7 +2,7 @@ #include -namespace NYdb::NTable { +namespace NYdb::inline V2::NTable { class TTransaction::TImpl { public: diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make index 51f737262807..62932cccc370 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make @@ -12,7 +12,7 @@ SRCS( PEERDIR( library/cpp/threading/future ydb/public/api/protos - ydb/public/lib/operation_id/protos + ydb/public/lib/operation_id ydb/public/sdk/cpp/client/impl/ydb_endpoints ydb/public/sdk/cpp/client/impl/ydb_internal/session_pool ydb/public/sdk/cpp/client/ydb_table/query_stats diff --git a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp index d1eb4f7a6632..e97097393463 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp @@ -2,7 +2,7 @@ #include "table.h" -namespace NYdb { +namespace NYdb::inline V2 { const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) { return queryStats.GetProto(); diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp index f2d37a91ace0..9a1cb196d4bb 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp @@ -1,6 +1,6 @@ #include "stats.h" -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { std::optional ParseQueryStatsMode(std::string_view statsMode) diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h index 29a7c87fa1f8..8c8d40a52863 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h @@ -19,7 +19,7 @@ namespace Ydb { } } -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 8c108bf83ded..95b4f0bd2beb 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -33,7 +33,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index a6d2f68dca9a..3c042105de16 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include "table_enum.h" #include @@ -39,7 +41,7 @@ class EvictionToExternalStorageSettings; } // namespace Table } // namespace Ydb -namespace NYdb { +namespace NYdb::inline V2 { namespace NRetry::Async { template @@ -186,7 +188,7 @@ class TPartitioningSettings { struct TExplicitPartitions { using TSelf = TExplicitPartitions; - FLUENT_SETTING_VECTOR(TValue, SplitPoints); + FLUENT_SETTING_VECTOR_DEPRECATED(TValue, SplitPoints); static TExplicitPartitions FromProto(const Ydb::Table::ExplicitPartitions& proto); void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; @@ -262,7 +264,7 @@ struct TKMeansTreeSettings { //! Represents index description class TIndexDescription { - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; public: TIndexDescription( @@ -339,7 +341,7 @@ class TBuildIndexOperation : public TOperation { //! Represents changefeed description class TChangefeedDescription { - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; public: class TInitialScanProgress { @@ -649,7 +651,7 @@ enum class EStoreType { //! Represents table description class TTableDescription { friend class TTableBuilder; - friend class NYdb::TProtoAccessor; + friend class NYdb::V2::TProtoAccessor; using EUnit = TValueSinceUnixEpochModeSettings::EUnit; @@ -1103,22 +1105,22 @@ struct TSessionPoolSettings { using TSelf = TSessionPoolSettings; // Max number of sessions client can get from session pool - FLUENT_SETTING_DEFAULT(ui32, MaxActiveSessions, 50); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MaxActiveSessions, 50); // Max number of attempt to create session inside session pool // to handle OVERLOADED error - FLUENT_SETTING_DEFAULT(ui32, RetryLimit, 5); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, RetryLimit, 5); // Max time session to be in idle state in session pool before // keep alive start to touch it - FLUENT_SETTING_DEFAULT(TDuration, KeepAliveIdleThreshold, TDuration::Minutes(5)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, KeepAliveIdleThreshold, TDuration::Minutes(5)); // Max time session to be in idle state before closing - FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); // Min number of session in session pool. // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. - FLUENT_SETTING_DEFAULT(ui32, MinPoolSize, 10); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MinPoolSize, 10); }; struct TClientSettings : public TCommonClientSettingsBase { @@ -1132,9 +1134,9 @@ struct TClientSettings : public TCommonClientSettingsBase { // as it doesn't require client-server synchronization and can recompile // query on demand without client interaction. // The recommended value is False. - FLUENT_SETTING_DEFAULT(bool, UseQueryCache, false); - FLUENT_SETTING_DEFAULT(ui32, QueryCacheSize, 1000); - FLUENT_SETTING_DEFAULT(bool, KeepDataQueryText, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, UseQueryCache, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, QueryCacheSize, 1000); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, KeepDataQueryText, true); // Min allowed session variation coefficient (%) to start session balancing. // Variation coefficient is a ratio of the standard deviation sigma to the mean @@ -1143,19 +1145,19 @@ struct TClientSettings : public TCommonClientSettingsBase { // - add new host ([90, 100, 110, 0] sessions per host). Cv will be 77% // Balancing is will be performed if calculated cv greater than MinSessionCV // Zero - disable this feature - FLUENT_SETTING_DEFAULT(ui32, MinSessionCV, 20); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MinSessionCV, 20); // Allow migrate requests between session during session balancing - FLUENT_SETTING_DEFAULT(bool, AllowRequestMigration, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowRequestMigration, true); // Settings of session pool - FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); + FLUENT_SETTING_DEPRECATED(TSessionPoolSettings, SessionPoolSettings); }; struct TBulkUpsertSettings : public TOperationRequestSettings { // Format setting proto serialized into string. If not set format defaults are used. // I.e. it's Ydb.Table.CsvSettings for CSV. - FLUENT_SETTING_DEFAULT(TString, FormatSettings, ""); + FLUENT_SETTING_DEFAULT_DEPRECATED(TString, FormatSettings, ""); }; struct TReadRowsSettings : public TOperationRequestSettings { @@ -1163,13 +1165,13 @@ struct TReadRowsSettings : public TOperationRequestSettings { struct TStreamExecScanQuerySettings : public TRequestSettings { // Return query plan without actual query execution - FLUENT_SETTING_DEFAULT(bool, Explain, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Explain, false); // Collect runtime statistics with a given detalization mode - FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); + FLUENT_SETTING_DEFAULT_DEPRECATED(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); // Collect full query compilation diagnostics - FLUENT_SETTING_DEFAULT(bool, CollectFullDiagnostics, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, CollectFullDiagnostics, false); }; class TSession; @@ -1289,7 +1291,7 @@ struct TTxOnlineSettings { TTxOnlineSettings() {} - FLUENT_SETTING_DEFAULT(bool, AllowInconsistentReads, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AllowInconsistentReads, false); }; class TTxSettings { @@ -1353,7 +1355,7 @@ class TTxSettings { TS_SNAPSHOT_RW, }; - FLUENT_SETTING(TTxOnlineSettings, OnlineSettings); + FLUENT_SETTING_DEPRECATED(TTxOnlineSettings, OnlineSettings); private: TTxSettings(ETransactionMode mode) @@ -1375,55 +1377,55 @@ enum class EAutoPartitioningPolicy { struct TColumnFamilyPolicy { using TSelf = TColumnFamilyPolicy; - FLUENT_SETTING_OPTIONAL(TString, Name); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Name); - FLUENT_SETTING_OPTIONAL(TString, Data); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Data); - FLUENT_SETTING_OPTIONAL(TString, External); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, External); - FLUENT_SETTING_OPTIONAL(bool, KeepInMemory); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, KeepInMemory); - FLUENT_SETTING_OPTIONAL(bool, Compressed); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, Compressed); }; struct TStoragePolicy { using TSelf = TStoragePolicy; - FLUENT_SETTING_OPTIONAL(TString, PresetName); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PresetName); - FLUENT_SETTING_OPTIONAL(TString, SysLog); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, SysLog); - FLUENT_SETTING_OPTIONAL(TString, Log); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Log); - FLUENT_SETTING_OPTIONAL(TString, Data); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Data); - FLUENT_SETTING_OPTIONAL(TString, External); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, External); - FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies); + FLUENT_SETTING_VECTOR_DEPRECATED(TColumnFamilyPolicy, ColumnFamilies); }; struct TPartitioningPolicy { using TSelf = TPartitioningPolicy; - FLUENT_SETTING_OPTIONAL(TString, PresetName); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PresetName); - FLUENT_SETTING_OPTIONAL(EAutoPartitioningPolicy, AutoPartitioning); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EAutoPartitioningPolicy, AutoPartitioning); - FLUENT_SETTING_OPTIONAL(ui64, UniformPartitions); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, UniformPartitions); - FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TExplicitPartitions, ExplicitPartitions); }; struct TReplicationPolicy { using TSelf = TReplicationPolicy; - FLUENT_SETTING_OPTIONAL(TString, PresetName); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PresetName); - FLUENT_SETTING_OPTIONAL(ui32, ReplicasCount); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, ReplicasCount); - FLUENT_SETTING_OPTIONAL(bool, CreatePerAvailabilityZone); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, CreatePerAvailabilityZone); - FLUENT_SETTING_OPTIONAL(bool, AllowPromotion); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, AllowPromotion); }; //////////////////////////////////////////////////////////////////////////////// @@ -1431,17 +1433,17 @@ struct TReplicationPolicy { struct TCreateTableSettings : public TOperationRequestSettings { using TSelf = TCreateTableSettings; - FLUENT_SETTING_OPTIONAL(TString, PresetName); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, PresetName); - FLUENT_SETTING_OPTIONAL(TString, ExecutionPolicy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, ExecutionPolicy); - FLUENT_SETTING_OPTIONAL(TString, CompactionPolicy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, CompactionPolicy); - FLUENT_SETTING_OPTIONAL(TPartitioningPolicy, PartitioningPolicy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TPartitioningPolicy, PartitioningPolicy); - FLUENT_SETTING_OPTIONAL(TStoragePolicy, StoragePolicy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TStoragePolicy, StoragePolicy); - FLUENT_SETTING_OPTIONAL(TReplicationPolicy, ReplicationPolicy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TReplicationPolicy, ReplicationPolicy); }; //////////////////////////////////////////////////////////////////////////////// @@ -1608,42 +1610,42 @@ struct TAlterTableSettings : public TOperationRequestSettings& value); const TMaybe& GetAlterTtlSettings() const; - FLUENT_SETTING(TAlterAttributes, AlterAttributes); + FLUENT_SETTING_DEPRECATED(TAlterAttributes, AlterAttributes); - FLUENT_SETTING(TString, SetCompactionPolicy); + FLUENT_SETTING_DEPRECATED(TString, SetCompactionPolicy); - FLUENT_SETTING_OPTIONAL(TPartitioningSettings, AlterPartitioningSettings); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TPartitioningSettings, AlterPartitioningSettings); - FLUENT_SETTING_OPTIONAL(bool, SetKeyBloomFilter); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, SetKeyBloomFilter); - FLUENT_SETTING_OPTIONAL(TReadReplicasSettings, SetReadReplicasSettings); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TReadReplicasSettings, SetReadReplicasSettings); TSelf& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { SetReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); return *this; @@ -1707,23 +1709,23 @@ struct TCopyTablesSettings : public TOperationRequestSettings {}; struct TDescribeTableSettings : public TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(bool, WithKeyShardBoundary, false); - FLUENT_SETTING_DEFAULT(bool, WithTableStatistics, false); - FLUENT_SETTING_DEFAULT(bool, WithPartitionStatistics, false); - FLUENT_SETTING_DEFAULT(bool, WithSetVal, false); - FLUENT_SETTING_DEFAULT(bool, WithShardNodesInfo, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithKeyShardBoundary, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithTableStatistics, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithPartitionStatistics, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithSetVal, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithShardNodesInfo, false); }; struct TExplainDataQuerySettings : public TOperationRequestSettings { - FLUENT_SETTING_DEFAULT(bool, WithCollectFullDiagnostics, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, WithCollectFullDiagnostics, false); }; struct TPrepareDataQuerySettings : public TOperationRequestSettings {}; struct TExecDataQuerySettings : public TOperationRequestSettings { - FLUENT_SETTING_OPTIONAL(bool, KeepInQueryCache); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, KeepInQueryCache); - FLUENT_SETTING_OPTIONAL(ECollectQueryStatsMode, CollectQueryStats); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ECollectQueryStatsMode, CollectQueryStats); }; struct TExecSchemeQuerySettings : public TOperationRequestSettings {}; @@ -1731,7 +1733,7 @@ struct TExecSchemeQuerySettings : public TOperationRequestSettings {}; struct TCommitTxSettings : public TOperationRequestSettings { - FLUENT_SETTING_OPTIONAL(ECollectQueryStatsMode, CollectQueryStats); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ECollectQueryStatsMode, CollectQueryStats); }; struct TRollbackTxSettings : public TOperationRequestSettings {}; @@ -1744,23 +1746,23 @@ struct TReadTableSettings : public TRequestSettings { using TSelf = TReadTableSettings; - FLUENT_SETTING_OPTIONAL(TKeyBound, From); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TKeyBound, From); - FLUENT_SETTING_OPTIONAL(TKeyBound, To); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TKeyBound, To); - FLUENT_SETTING_VECTOR(TString, Columns); + FLUENT_SETTING_VECTOR_DEPRECATED(TString, Columns); - FLUENT_SETTING_FLAG(Ordered); + FLUENT_SETTING_FLAG_DEPRECATED(Ordered); - FLUENT_SETTING_OPTIONAL(ui64, RowLimit); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, RowLimit); - FLUENT_SETTING_OPTIONAL(bool, UseSnapshot); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, UseSnapshot); - FLUENT_SETTING_OPTIONAL(ui64, BatchLimitBytes); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, BatchLimitBytes); - FLUENT_SETTING_OPTIONAL(ui64, BatchLimitRows); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, BatchLimitRows); - FLUENT_SETTING_OPTIONAL(bool, ReturnNotNullAsOptional); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, ReturnNotNullAsOptional); }; using TPrecommitTransactionCallback = std::function; @@ -1938,7 +1940,7 @@ class TTxControl { return TTxControl(settings); } - FLUENT_SETTING_FLAG(CommitTx); + FLUENT_SETTING_FLAG_DEPRECATED(CommitTx); private: TTxControl(const TTransaction& tx); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h index 1660706f57a7..af87ba3cabb4 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h @@ -1,8 +1,10 @@ #pragma once +#include "fwd.h" + #include -namespace NYdb { +namespace NYdb::inline V2 { namespace NTable { //! Column family compression codec diff --git a/ydb/public/sdk/cpp/client/ydb_topic/common/callback_context.h b/ydb/public/sdk/cpp/client/ydb_topic/common/callback_context.h index ed3e3aeb165d..fe1e6d3ddfb7 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/common/callback_context.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/common/callback_context.h @@ -10,7 +10,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { template class TContextOwner; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.cpp b/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.cpp index ad5369a24652..8f2f69c34005 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.cpp @@ -1,6 +1,6 @@ #include "executor_impl.h" -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { void IAsyncExecutor::Post(TFunction&& f) { PostImpl(std::move(f)); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.h b/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.h index b1b0ba562b92..a55c69a38e64 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/common/executor_impl.h @@ -9,7 +9,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class IAsyncExecutor : public IExecutor { private: diff --git a/ydb/public/sdk/cpp/client/ydb_topic/common/retry_policy.cpp b/ydb/public/sdk/cpp/client/ydb_topic/common/retry_policy.cpp index 62a5822aba2e..4dfeecba90e6 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/common/retry_policy.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/common/retry_policy.cpp @@ -1,7 +1,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { IRetryPolicy::TPtr IRetryPolicy::GetDefaultPolicy() { static IRetryPolicy::TPtr policy = GetExponentialBackoffPolicy(); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/common.cpp index b7becebc39f4..7639fe1d17c1 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/common.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/common.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { ERetryErrorClass GetRetryErrorClass(EStatus status) { switch (status) { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/common.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/common.h index 3f40636b1e97..97f70b9c0a3a 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/common.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/common.h @@ -11,7 +11,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { ERetryErrorClass GetRetryErrorClass(EStatus status); ERetryErrorClass GetRetryErrorClassV2(EStatus status); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/counters_logger.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/counters_logger.h index 9c7e4fb04a0f..ff0f4b36413e 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/counters_logger.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/counters_logger.h @@ -8,7 +8,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { template class TSingleClusterReadSessionImpl; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/deferred_commit.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/deferred_commit.cpp index d00ca6cc4ed3..d1db75dec61d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/deferred_commit.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/deferred_commit.cpp @@ -4,7 +4,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/event_handlers.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/event_handlers.cpp index f15f18f3d58c..229b561ff321 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/event_handlers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/event_handlers.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.cpp index 2f74e1cc7fb6..4c2785ae0ff0 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.cpp @@ -1,6 +1,6 @@ #include "offsets_collector.h" -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { TVector TOffsetsCollector::GetOffsets() const { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.h index 849922b21018..2cccfc08833e 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/offsets_collector.h @@ -14,7 +14,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class TOffsetsCollector { public: diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.cpp index 3916bb0818aa..8a6c76b5bfb8 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.cpp @@ -7,7 +7,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { static const TString DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.h index 79c9136eaf2e..08e56408f6d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session.h @@ -6,7 +6,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class TReadSession : public IReadSession { public: diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_event.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_event.cpp index 6737c517f55b..516a28fe4ad7 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_event.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_event.cpp @@ -3,7 +3,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Aliases for event types diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_impl.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_impl.h index 9e35ae0ac2d4..1197cda816d9 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/read_session_impl.h @@ -33,7 +33,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { template using TClientMessage = std::conditional_t #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { static const bool RangesMode = !GetEnv("PQ_OFFSET_RANGES_MODE").empty(); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic.cpp index 240cf3382621..b252d0b5be01 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic.cpp @@ -9,7 +9,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class TCommonCodecsProvider { public: diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.cpp index 35b5cfdf1691..3ff49665d663 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.cpp @@ -3,7 +3,7 @@ #include "read_session.h" #include "write_session.h" -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { std::shared_ptr TTopicClient::TImpl::CreateReadSession(const TReadSessionSettings& settings) { TMaybe maybeSettings; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.h index 785d11379030..f7d82deb2ff4 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/topic_impl.h @@ -13,7 +13,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { struct TOffsetsRange { ui64 Start; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.cpp index 95b938c86ed7..ec5d8f1e2d8a 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.cpp @@ -1,7 +1,7 @@ #include "transaction.h" #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { TTransactionId MakeTransactionId(const NTable::TTransaction& tx) { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.h index ab37b5bafe84..ce87ed93ac60 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/transaction.h @@ -4,13 +4,13 @@ #include -namespace NYdb::NTable { +namespace NYdb::inline V2::NTable { class TTransaction; } -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { using TTransactionId = std::pair; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.cpp index a8e170a9074d..e02eb327efec 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.cpp @@ -2,7 +2,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TWriteSession diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.h index 15273c4e7698..e02330d2359d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session.h @@ -8,7 +8,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TWriteSession diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.cpp index 46b774419a6b..96df38467b63 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.cpp @@ -12,7 +12,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); // Error code from file ydb/public/api/protos/persqueue_error_codes_v1.proto diff --git a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.h b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.h index c091c17e5eb6..80644e9ccfc9 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/impl/write_session_impl.h @@ -9,7 +9,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class TWriteSessionEventsQueue: public TBaseSessionEventsQueue { using TParent = TBaseSessionEventsQueue; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/client.h b/ydb/public/sdk/cpp/client/ydb_topic/include/client.h index 5ea7e8744b83..b7d227db066e 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/client.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/client.h @@ -4,16 +4,16 @@ #include "read_session.h" #include "write_session.h" -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { struct TTopicClientSettings : public TCommonClientSettingsBase { using TSelf = TTopicClientSettings; //! Default executor for compression tasks. - FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); + FLUENT_SETTING_DEFAULT_DEPRECATED(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); //! Default executor for callbacks. - FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); + FLUENT_SETTING_DEFAULT_DEPRECATED(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); }; // Topic client. diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/codecs.h b/ydb/public/sdk/cpp/client/ydb_topic/include/codecs.h index 7197bb349e48..84652bd2ae71 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/codecs.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/codecs.h @@ -12,7 +12,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { enum class ECodec : ui32 { RAW = 1, diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/control_plane.h b/ydb/public/sdk/cpp/client/ydb_topic/include/control_plane.h index 6044340845c9..d8bf90b8cf72 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/control_plane.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/control_plane.h @@ -12,7 +12,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TProtoAccessor; namespace NScheme { @@ -20,7 +20,7 @@ namespace NYdb { } } -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { enum class EMeteringMode : ui32 { Unspecified = 0, @@ -203,10 +203,10 @@ struct TAlterAutoPartitioningSettings { public: TAlterAutoPartitioningSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} - FLUENT_SETTING_OPTIONAL(EAutoPartitioningStrategy, Strategy); - FLUENT_SETTING_OPTIONAL(TDuration, StabilizationWindow); - FLUENT_SETTING_OPTIONAL(ui64, DownUtilizationPercent); - FLUENT_SETTING_OPTIONAL(ui64, UpUtilizationPercent); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EAutoPartitioningStrategy, Strategy); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, StabilizationWindow); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, DownUtilizationPercent); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, UpUtilizationPercent); TAlterPartitioningSettings& EndAlterAutoPartitioningSettings() { return Parent_; }; @@ -246,8 +246,8 @@ struct TAlterPartitioningSettings { public: TAlterPartitioningSettings(TAlterTopicSettings& parent): Parent_(parent) {} - FLUENT_SETTING_OPTIONAL(ui64, MinActivePartitions); - FLUENT_SETTING_OPTIONAL(ui64, MaxActivePartitions); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, MinActivePartitions); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, MaxActivePartitions); TAlterTopicSettings& EndAlterTopicPartitioningSettings() { return Parent_; }; @@ -440,13 +440,13 @@ struct TConsumerSettings { TConsumerSettings(TSettings& parent): Parent_(parent) {} TConsumerSettings(TSettings& parent, const TString& name) : ConsumerName_(name), Parent_(parent) {} - FLUENT_SETTING(TString, ConsumerName); - FLUENT_SETTING_DEFAULT(bool, Important, false); - FLUENT_SETTING_DEFAULT(TInstant, ReadFrom, TInstant::Zero()); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Important, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(TInstant, ReadFrom, TInstant::Zero()); - FLUENT_SETTING_VECTOR(ECodec, SupportedCodecs); + FLUENT_SETTING_VECTOR_DEPRECATED(ECodec, SupportedCodecs); - FLUENT_SETTING(TAttributes, Attributes); + FLUENT_SETTING_DEPRECATED(TAttributes, Attributes); TConsumerSettings& AddAttribute(const TString& key, const TString& value) { Attributes_[key] = value; @@ -492,13 +492,13 @@ struct TAlterConsumerSettings { TAlterConsumerSettings(TAlterTopicSettings& parent): Parent_(parent) {} TAlterConsumerSettings(TAlterTopicSettings& parent, const TString& name) : ConsumerName_(name), Parent_(parent) {} - FLUENT_SETTING(TString, ConsumerName); - FLUENT_SETTING_OPTIONAL(bool, SetImportant); - FLUENT_SETTING_OPTIONAL(TInstant, SetReadFrom); + FLUENT_SETTING_DEPRECATED(TString, ConsumerName); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, SetImportant); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, SetReadFrom); - FLUENT_SETTING_OPTIONAL_VECTOR(ECodec, SetSupportedCodecs); + FLUENT_SETTING_OPTIONAL_VECTOR_DEPRECATED(ECodec, SetSupportedCodecs); - FLUENT_SETTING(TAlterAttributes, AlterAttributes); + FLUENT_SETTING_DEPRECATED(TAlterAttributes, AlterAttributes); TAlterConsumerAttributesBuilder BeginAlterAttributes() { return TAlterConsumerAttributesBuilder(*this); @@ -526,21 +526,21 @@ struct TCreateTopicSettings : public TOperationRequestSettings; - FLUENT_SETTING(TPartitioningSettings, PartitioningSettings); + FLUENT_SETTING_DEPRECATED(TPartitioningSettings, PartitioningSettings); - FLUENT_SETTING_DEFAULT(TDuration, RetentionPeriod, TDuration::Hours(24)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, RetentionPeriod, TDuration::Hours(24)); - FLUENT_SETTING_VECTOR(ECodec, SupportedCodecs); + FLUENT_SETTING_VECTOR_DEPRECATED(ECodec, SupportedCodecs); - FLUENT_SETTING_DEFAULT(ui64, RetentionStorageMb, 0); - FLUENT_SETTING_DEFAULT(EMeteringMode, MeteringMode, EMeteringMode::Unspecified); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, RetentionStorageMb, 0); + FLUENT_SETTING_DEFAULT_DEPRECATED(EMeteringMode, MeteringMode, EMeteringMode::Unspecified); - FLUENT_SETTING_DEFAULT(ui64, PartitionWriteSpeedBytesPerSecond, 0); - FLUENT_SETTING_DEFAULT(ui64, PartitionWriteBurstBytes, 0); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, PartitionWriteSpeedBytesPerSecond, 0); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, PartitionWriteBurstBytes, 0); - FLUENT_SETTING_VECTOR(TConsumerSettings, Consumers); + FLUENT_SETTING_VECTOR_DEPRECATED(TConsumerSettings, Consumers); - FLUENT_SETTING(TAttributes, Attributes); + FLUENT_SETTING_DEPRECATED(TAttributes, Attributes); TCreateTopicSettings& SetSupportedCodecs(TVector&& codecs) { SupportedCodecs_ = std::move(codecs); @@ -651,22 +651,22 @@ struct TAlterTopicSettings : public TOperationRequestSettings; - FLUENT_SETTING_OPTIONAL(TDuration, SetRetentionPeriod); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, SetRetentionPeriod); - FLUENT_SETTING_OPTIONAL_VECTOR(ECodec, SetSupportedCodecs); + FLUENT_SETTING_OPTIONAL_VECTOR_DEPRECATED(ECodec, SetSupportedCodecs); - FLUENT_SETTING_OPTIONAL(ui64, SetRetentionStorageMb); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, SetRetentionStorageMb); - FLUENT_SETTING_OPTIONAL(ui64, SetPartitionWriteSpeedBytesPerSecond); - FLUENT_SETTING_OPTIONAL(ui64, SetPartitionWriteBurstBytes); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, SetPartitionWriteSpeedBytesPerSecond); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, SetPartitionWriteBurstBytes); - FLUENT_SETTING_OPTIONAL(EMeteringMode, SetMeteringMode); + FLUENT_SETTING_OPTIONAL_DEPRECATED(EMeteringMode, SetMeteringMode); - FLUENT_SETTING_VECTOR(TConsumerSettings, AddConsumers); - FLUENT_SETTING_VECTOR(TString, DropConsumers); - FLUENT_SETTING_VECTOR(TAlterConsumerSettings, AlterConsumers); + FLUENT_SETTING_VECTOR_DEPRECATED(TConsumerSettings, AddConsumers); + FLUENT_SETTING_VECTOR_DEPRECATED(TString, DropConsumers); + FLUENT_SETTING_VECTOR_DEPRECATED(TAlterConsumerSettings, AlterConsumers); - FLUENT_SETTING(TAlterAttributes, AlterAttributes); + FLUENT_SETTING_DEPRECATED(TAlterAttributes, AlterAttributes); TAlterTopicAttributesBuilder BeginAlterAttributes() { return TAlterTopicAttributesBuilder(*this); @@ -731,27 +731,27 @@ struct TDropTopicSettings : public TOperationRequestSettings struct TDescribeTopicSettings : public TOperationRequestSettings { using TSelf = TDescribeTopicSettings; - FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeStats, false); - FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeLocation, false); }; // Settings for describe consumer request. struct TDescribeConsumerSettings : public TOperationRequestSettings { using TSelf = TDescribeConsumerSettings; - FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeStats, false); - FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeLocation, false); }; // Settings for describe partition request. struct TDescribePartitionSettings: public TOperationRequestSettings { using TSelf = TDescribePartitionSettings; - FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeStats, false); - FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, IncludeLocation, false); }; // Settings for commit offset request. diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/counters.h b/ydb/public/sdk/cpp/client/ydb_topic/include/counters.h index 8d44dc131c27..cf142eed5672 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/counters.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/counters.h @@ -2,7 +2,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { using TCounterPtr = ::NMonitoring::TDynamicCounters::TCounterPtr; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/errors.h b/ydb/public/sdk/cpp/client/ydb_topic/include/errors.h index 048314462e2c..3877edc5b297 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/errors.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/errors.h @@ -5,7 +5,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { ERetryErrorClass GetRetryErrorClass(EStatus status); ERetryErrorClass GetRetryErrorClassV2(EStatus status); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/events_common.h b/ydb/public/sdk/cpp/client/ydb_topic/include/events_common.h index b6c531687a34..9d17395005f1 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/events_common.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/events_common.h @@ -7,7 +7,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { template class TPrintable { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/executor.h b/ydb/public/sdk/cpp/client/ydb_topic/include/executor.h index 9a591fa2af95..a48b6243708c 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/executor.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/executor.h @@ -4,7 +4,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { class IExecutor: public TThrRefBase { public: diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/read_events.h b/ydb/public/sdk/cpp/client/ydb_topic/include/read_events.h index 807865333990..da33e33e258d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/read_events.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/read_events.h @@ -6,7 +6,7 @@ #include #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //! Partition session. struct TPartitionSession: public TThrRefBase, public TPrintable { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/read_session.h b/ydb/public/sdk/cpp/client/ydb_topic/include/read_session.h index 3acd973fda53..e635e0493110 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/read_session.h @@ -13,11 +13,11 @@ #include -namespace NYdb::NTable { +namespace NYdb::inline V2::NTable { class TTransaction; } -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //! Read settings for single topic. struct TTopicReadSettings { @@ -34,17 +34,17 @@ struct TTopicReadSettings { TTopicReadSettings& operator=(TTopicReadSettings&&) = default; //! Path of topic to read. - FLUENT_SETTING(TString, Path); + FLUENT_SETTING_DEPRECATED(TString, Path); //! Start reading from this timestamp. - FLUENT_SETTING_OPTIONAL(TInstant, ReadFromTimestamp); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, ReadFromTimestamp); //! Partition ids to read. //! 0-based. - FLUENT_SETTING_VECTOR(ui64, PartitionIds); + FLUENT_SETTING_VECTOR_DEPRECATED(ui64, PartitionIds); //! Max message time lag. All messages older that now - MaxLag will be ignored. - FLUENT_SETTING_OPTIONAL(TDuration, MaxLag); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, MaxLag); }; //! Settings for read session. @@ -75,71 +75,71 @@ struct TReadSessionSettings: public TRequestSettings { //! Data size limit for the DataReceivedHandler handler. //! The data size may exceed this limit. - FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, Max()); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxMessagesBytes, Max()); //! Function to handle data events. //! If this handler is set, data events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, DataReceivedHandler); + FLUENT_SETTING_DEPRECATED(std::function, DataReceivedHandler); //! Function to handle commit ack events. //! If this handler is set, commit ack events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, CommitOffsetAcknowledgementHandler); //! Function to handle start partition session events. //! If this handler is set, create partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, StartPartitionSessionHandler); //! Function to handle stop partition session events. //! If this handler is set, destroy partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, StopPartitionSessionHandler); //! Function to handle end partition session events. //! If this handler is set, end partition session events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, EndPartitionSessionHandler); //! Function to handle partition session status events. //! If this handler is set, partition session status events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, PartitionSessionStatusHandler); //! Function to handle partition session closed events. //! If this handler is set, partition session closed events will be handled by handler, //! otherwise sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(std::function, + FLUENT_SETTING_DEPRECATED(std::function, PartitionSessionClosedHandler); //! Function to handle session closed events. //! If this handler is set, close session events will be handled by handler //! and then sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING_DEPRECATED(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, //! this handler (if specified) will be used. //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. - FLUENT_SETTING(std::function, CommonHandler); + FLUENT_SETTING_DEPRECATED(std::function, CommonHandler); //! Executor for handlers. //! If not set, default single threaded executor will be used. - FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, HandlersExecutor); }; @@ -160,53 +160,53 @@ struct TReadSessionSettings: public TRequestSettings { } //! Topics. - FLUENT_SETTING_VECTOR(TTopicReadSettings, Topics); + FLUENT_SETTING_VECTOR_DEPRECATED(TTopicReadSettings, Topics); //! Maximum memory usage for read session. - FLUENT_SETTING_DEFAULT(size_t, MaxMemoryUsageBytes, 100_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxMemoryUsageBytes, 100_MB); //! Max message time lag. All messages older that now - MaxLag will be ignored. - FLUENT_SETTING_OPTIONAL(TDuration, MaxLag); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, MaxLag); //! Start reading from this timestamp. - FLUENT_SETTING_OPTIONAL(TInstant, ReadFromTimestamp); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, ReadFromTimestamp); //! Policy for reconnections. //! IRetryPolicy::GetDefaultPolicy() if null (not set). - FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + FLUENT_SETTING_DEPRECATED(IRetryPolicy::TPtr, RetryPolicy); //! Event handlers. //! See description in TEventHandlers class. - FLUENT_SETTING(TEventHandlers, EventHandlers); + FLUENT_SETTING_DEPRECATED(TEventHandlers, EventHandlers); //! Decompress messages - FLUENT_SETTING_DEFAULT(bool, Decompress, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Decompress, true); //! Executor for decompression tasks. //! If not set, default executor will be used. - FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, DecompressionExecutor); //! Counters. //! If counters are not provided explicitly, //! they will be created inside session (without link with parent counters). - FLUENT_SETTING(TReaderCounters::TPtr, Counters); + FLUENT_SETTING_DEPRECATED(TReaderCounters::TPtr, Counters); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Seconds(30)); //! AutoPartitioningSupport. - FLUENT_SETTING_DEFAULT(bool, AutoPartitioningSupport, false); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, AutoPartitioningSupport, false); //! Log. - FLUENT_SETTING_OPTIONAL(TLog, Log); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TLog, Log); }; struct TReadSessionGetEventSettings : public TCommonClientSettingsBase { using TSelf = TReadSessionGetEventSettings; - FLUENT_SETTING_DEFAULT(bool, Block, false); - FLUENT_SETTING_OPTIONAL(size_t, MaxEventsCount); - FLUENT_SETTING_DEFAULT(size_t, MaxByteSize, std::numeric_limits::max()); - FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, Block, false); + FLUENT_SETTING_OPTIONAL_DEPRECATED(size_t, MaxEventsCount); + FLUENT_SETTING_DEFAULT_DEPRECATED(size_t, MaxByteSize, std::numeric_limits::max()); + FLUENT_SETTING_OPTIONAL_DEPRECATED(std::reference_wrapper, Tx); }; class IReadSession { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/retry_policy.h b/ydb/public/sdk/cpp/client/ydb_topic/include/retry_policy.h index 39ab4de0f7f8..6fb09d127d2d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/retry_policy.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/retry_policy.h @@ -6,7 +6,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { //! Retry policy. diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/write_events.h b/ydb/public/sdk/cpp/client/ydb_topic/include/write_events.h index 9ac070beeb98..45d907f37b2d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/write_events.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/write_events.h @@ -7,7 +7,7 @@ #include -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { struct TWriteStat : public TThrRefBase { TDuration WriteTime; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/include/write_session.h b/ydb/public/sdk/cpp/client/ydb_topic/include/write_session.h index e2b3301b2b81..a5ebd6949de7 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/include/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/include/write_session.h @@ -11,11 +11,11 @@ #include -namespace NYdb::NTable { +namespace NYdb::inline V2::NTable { class TTransaction; } -namespace NYdb::NTopic { +namespace NYdb::inline V2::NTopic { using TTransaction = NTable::TTransaction; @@ -36,22 +36,22 @@ struct TWriteSessionSettings : public TRequestSettings { TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; //! Path of topic to write. - FLUENT_SETTING(TString, Path); + FLUENT_SETTING_DEPRECATED(TString, Path); //! ProducerId (aka SourceId) to use. - FLUENT_SETTING(TString, ProducerId); + FLUENT_SETTING_DEPRECATED(TString, ProducerId); //! MessageGroupId to use. - FLUENT_SETTING(TString, MessageGroupId); + FLUENT_SETTING_DEPRECATED(TString, MessageGroupId); //! Explicitly enables or disables deduplication for this write session. //! If ProducerId option is defined deduplication will always be enabled. //! If ProducerId option is empty, but deduplication is enable, a random ProducerId is generated. - FLUENT_SETTING_OPTIONAL(bool, DeduplicationEnabled); + FLUENT_SETTING_OPTIONAL_DEPRECATED(bool, DeduplicationEnabled); //! Write to an exact partition. Generally server assigns partition automatically by message_group_id. //! Using this option is not recommended unless you know for sure why you need it. - FLUENT_SETTING_OPTIONAL(ui32, PartitionId); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui32, PartitionId); //! Direct write to the partition host. //! If both PartitionId and DirectWriteToPartition are set, write session goes directly to the partition host. @@ -59,23 +59,23 @@ struct TWriteSessionSettings : public TRequestSettings { //! 1. Get a partition ID. //! 2. Find out the location of the partition by its ID. //! 3. Connect directly to the partition host. - FLUENT_SETTING_DEFAULT(bool, DirectWriteToPartition, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, DirectWriteToPartition, true); //! codec and level to use for data compression prior to write. - FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); - FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4); + FLUENT_SETTING_DEFAULT_DEPRECATED(ECodec, Codec, ECodec::GZIP); + FLUENT_SETTING_DEFAULT_DEPRECATED(i32, CompressionLevel, 4); //! Writer will not accept new messages if memory usage exceeds this limit. //! Memory usage consists of raw data pending compression and compressed messages being sent. - FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20_MB); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui64, MaxMemoryUsage, 20_MB); //! Maximum messages accepted by writer but not written (with confirmation from server). //! Writer will not accept new messages after reaching the limit. - FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000); + FLUENT_SETTING_DEFAULT_DEPRECATED(ui32, MaxInflightCount, 100000); //! Retry policy enables automatic retries for non-fatal errors. //! IRetryPolicy::GetDefaultPolicy() if null (not set). - FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + FLUENT_SETTING_DEPRECATED(IRetryPolicy::TPtr, RetryPolicy); //! User metadata that may be attached to write session. TWriteSessionSettings& AppendSessionMeta(const TString& key, const TString& value) { @@ -91,16 +91,16 @@ struct TWriteSessionSettings : public TRequestSettings { //! Greatly increases performance for small messages. //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients //! sending small messages at high rate). - FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); - FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TDuration, BatchFlushInterval); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, BatchFlushSizeBytes); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Seconds(30)); - FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TWriterCounters::TPtr, Counters); //! Executor for compression tasks. //! If not set, default executor will be used. - FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, CompressionExecutor); struct TEventHandlers { using TSelf = TEventHandlers; @@ -110,17 +110,17 @@ struct TWriteSessionSettings : public TRequestSettings { //! Function to handle Acks events. //! If this handler is set, write ack events will be handled by handler, //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TWriteAckHandler, AcksHandler); + FLUENT_SETTING_DEPRECATED(TWriteAckHandler, AcksHandler); //! Function to handle ReadyToAccept event. //! If this handler is set, write these events will be handled by handler, //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHandler); + FLUENT_SETTING_DEPRECATED(TReadyToAcceptHandler, ReadyToAcceptHandler); //! Function to handle close session events. //! If this handler is set, close session events will be handled by handler //! and then sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING_DEPRECATED(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, @@ -134,7 +134,7 @@ struct TWriteSessionSettings : public TRequestSettings { //! Executor for handlers. //! If not set, default single threaded executor will be used. - FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING_DEPRECATED(IExecutor::TPtr, HandlersExecutor); [[deprecated("Typo in name. Use ReadyToAcceptHandler instead.")]] TSelf& ReadyToAcceptHander(const TReadyToAcceptHandler& value) { @@ -143,10 +143,10 @@ struct TWriteSessionSettings : public TRequestSettings { }; //! Event handlers. - FLUENT_SETTING(TEventHandlers, EventHandlers); + FLUENT_SETTING_DEPRECATED(TEventHandlers, EventHandlers); //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception. - FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, ValidateSeqNo, true); }; //! Contains the message to write and all the options. @@ -183,16 +183,16 @@ struct TWriteMessage { //! Message SeqNo, optional. If not provided SDK core will calculate SeqNo automatically. //! NOTICE: Either all messages within one write session must have SeqNo provided or none of them. - FLUENT_SETTING_OPTIONAL(ui64, SeqNo); + FLUENT_SETTING_OPTIONAL_DEPRECATED(ui64, SeqNo); //! Message creation timestamp. If not provided, Now() will be used. - FLUENT_SETTING_OPTIONAL(TInstant, CreateTimestamp); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TInstant, CreateTimestamp); //! Message metadata. Limited to 4096 characters overall (all keys and values combined). - FLUENT_SETTING(TMessageMeta, MessageMeta); + FLUENT_SETTING_DEPRECATED(TMessageMeta, MessageMeta); //! Transaction id - FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); + FLUENT_SETTING_OPTIONAL_DEPRECATED(std::reference_wrapper, Tx); TTransaction* GetTxPtr() const { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_topic/proto_accessor.cpp index 9ae37fc3d04d..cb88654e816f 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/proto_accessor.cpp @@ -1,6 +1,6 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { const Ydb::Topic::DescribeTopicResult& TProtoAccessor::GetProto(const NTopic::TTopicDescription& topicDescription) { return topicDescription.GetProto(); } diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ya.make b/ydb/public/sdk/cpp/client/ydb_topic/ya.make index d626e2ce43c7..4f32d3aa7b96 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_topic/ya.make @@ -22,7 +22,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h b/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h index 91b65f7181dc..93146d67b442 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h +++ b/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { using TPeriodicCb = std::function; // !!!Experimental!!! diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp index 3d0ff1b3d977..efd9fa7d6db0 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp @@ -1,7 +1,7 @@ #include "credentials.h" #include -namespace NYdb { +namespace NYdb::inline V2 { class TInsecureCredentialsProvider : public ICredentialsProvider { public: diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h index a2274d1a8c73..2ad90334dece 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h @@ -1,10 +1,12 @@ #pragma once +#include + #include #include -namespace NYdb { +namespace NYdb::inline V2 { class ICredentialsProvider { public: diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/login/login.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/login/login.cpp index 48528e59717c..fba7c095599c 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/login/login.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/login/login.cpp @@ -8,7 +8,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class TLoginCredentialsProvider : public ICredentialsProvider { public: diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.cpp index aeeb5327b33b..fdab6b80fa8e 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.cpp @@ -20,7 +20,7 @@ #define INV_ARG "Invalid argument for " PROV_ERR #define EXCH_ERR "Exchange token error in " PROV_ERR -namespace NYdb { +namespace NYdb::inline V2 { namespace { diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.h index 947dbe1bfe13..24e53d57d649 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/credentials.h @@ -1,11 +1,14 @@ #pragma once + +#include + #include #include #include #include -namespace NYdb { +namespace NYdb::inline V2 { class ITokenSource { public: @@ -24,8 +27,8 @@ class ITokenSource { std::shared_ptr CreateFixedTokenSource(const TString& token, const TString& tokenType); -#define FLUENT_SETTING_VECTOR_OR_SINGLE(type, name) \ - FLUENT_SETTING_VECTOR(type, name); \ +#define FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(type, name) \ + FLUENT_SETTING_VECTOR_DEPRECATED(type, name); \ TSelf& name(const type& value) { \ name##_.resize(1); \ name##_[0] = value; \ @@ -35,22 +38,22 @@ std::shared_ptr CreateFixedTokenSource(const TString& token, const struct TOauth2TokenExchangeParams { using TSelf = TOauth2TokenExchangeParams; - FLUENT_SETTING(TString, TokenEndpoint); + FLUENT_SETTING_DEPRECATED(TString, TokenEndpoint); - FLUENT_SETTING_DEFAULT(TString, GrantType, "urn:ietf:params:oauth:grant-type:token-exchange"); + FLUENT_SETTING_DEFAULT_DEPRECATED(TString, GrantType, "urn:ietf:params:oauth:grant-type:token-exchange"); - FLUENT_SETTING_VECTOR_OR_SINGLE(TString, Resource); - FLUENT_SETTING_VECTOR_OR_SINGLE(TString, Audience); - FLUENT_SETTING_VECTOR_OR_SINGLE(TString, Scope); + FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(TString, Resource); + FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(TString, Audience); + FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(TString, Scope); - FLUENT_SETTING_DEFAULT(TString, RequestedTokenType, "urn:ietf:params:oauth:token-type:access_token"); + FLUENT_SETTING_DEFAULT_DEPRECATED(TString, RequestedTokenType, "urn:ietf:params:oauth:token-type:access_token"); - FLUENT_SETTING(std::shared_ptr, SubjectTokenSource); - FLUENT_SETTING(std::shared_ptr, ActorTokenSource); + FLUENT_SETTING_DEPRECATED(std::shared_ptr, SubjectTokenSource); + FLUENT_SETTING_DEPRECATED(std::shared_ptr, ActorTokenSource); - FLUENT_SETTING_DEFAULT(TDuration, SocketTimeout, TDuration::Seconds(5)); - FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); - FLUENT_SETTING_DEFAULT(TDuration, SyncUpdateTimeout, TDuration::Seconds(20)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, SocketTimeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, SyncUpdateTimeout, TDuration::Seconds(20)); }; // Creates OAuth 2.0 token exchange credentials provider factory that exchanges token using standard protocol diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.cpp index 04923943326f..67453831e0b3 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.cpp @@ -12,7 +12,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { namespace { @@ -147,15 +147,15 @@ bool IsAsciiEqualUpper(const TString& jsonParam, const TStringBuf& constantInUpp struct TFixedTokenSourceParamsForParsing { using TSelf = TFixedTokenSourceParamsForParsing; - FLUENT_SETTING(TString, Token); - FLUENT_SETTING(TString, TokenType); + FLUENT_SETTING_DEPRECATED(TString, Token); + FLUENT_SETTING_DEPRECATED(TString, TokenType); }; // special struct to apply macros struct TJwtTokenSourceParamsForParsing : public TJwtTokenSourceParams { - FLUENT_SETTING(TString, AlgStr); - FLUENT_SETTING(TString, PrivateKeyStr); - FLUENT_SETTING(TString, TtlStr); + FLUENT_SETTING_DEPRECATED(TString, AlgStr); + FLUENT_SETTING_DEPRECATED(TString, PrivateKeyStr); + FLUENT_SETTING_DEPRECATED(TString, TtlStr); }; std::shared_ptr ParseFixedTokenSource(const NJson::TJsonValue& cfg) { diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.h index 9184c83ba13b..98217abbcce0 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/from_file.h @@ -1,11 +1,14 @@ #pragma once + +#include + #include #include #include -namespace NYdb { +namespace NYdb::inline V2 { // Lists supported algorithms for creation of OAuth 2.0 token exchange provider via config file std::vector GetSupportedOauth2TokenExchangeJwtAlgorithms(); diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.cpp index 833851bce8ef..9de13d36257c 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.cpp @@ -7,7 +7,7 @@ #define INV_ARG "Invalid argument for JWT token source: " -namespace NYdb { +namespace NYdb::inline V2 { static const TString TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"; diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.h index 22c29d50f2e6..1b243d088bcd 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/jwt_token_source.h @@ -1,18 +1,21 @@ #pragma once + +#include + #include "credentials.h" #include #include #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr TDuration DEFAULT_JWT_TOKEN_TTL = TDuration::Hours(1); struct TJwtTokenSourceParams { using TSelf = TJwtTokenSourceParams; - FLUENT_SETTING(TString, KeyId); + FLUENT_SETTING_DEPRECATED(TString, KeyId); template TSelf& SigningAlgorithm(T&&... args) { @@ -21,12 +24,12 @@ struct TJwtTokenSourceParams { } // JWT Claims - FLUENT_SETTING(TString, Issuer); - FLUENT_SETTING(TString, Subject); - FLUENT_SETTING(TString, Id); - FLUENT_SETTING_VECTOR_OR_SINGLE(TString, Audience); + FLUENT_SETTING_DEPRECATED(TString, Issuer); + FLUENT_SETTING_DEPRECATED(TString, Subject); + FLUENT_SETTING_DEPRECATED(TString, Id); + FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(TString, Audience); - FLUENT_SETTING_DEFAULT(TDuration, TokenTtl, DEFAULT_JWT_TOKEN_TTL); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, TokenTtl, DEFAULT_JWT_TOKEN_TTL); // Helpers diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/ut/jwt_check_helper.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/ut/jwt_check_helper.h index 551822575f82..0569a381c019 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/ut/jwt_check_helper.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/oauth2_token_exchange/ut/jwt_check_helper.h @@ -39,13 +39,13 @@ struct TJwtCheck { } THolder Alg_ = MakeHolder>(TestRSAPublicKeyContent); - FLUENT_SETTING_OPTIONAL(TString, KeyId); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, KeyId); - FLUENT_SETTING_OPTIONAL(TString, Issuer); - FLUENT_SETTING_OPTIONAL(TString, Subject); - FLUENT_SETTING_OPTIONAL(TString, Id); - FLUENT_SETTING_VECTOR_OR_SINGLE(TString, Audience); - FLUENT_SETTING_DEFAULT(TDuration, TokenTtl, TDuration::Hours(1)); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Issuer); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Subject); + FLUENT_SETTING_OPTIONAL_DEPRECATED(TString, Id); + FLUENT_SETTING_VECTOR_OR_SINGLE_DEPRECATED(TString, Audience); + FLUENT_SETTING_DEFAULT_DEPRECATED(TDuration, TokenTtl, TDuration::Hours(1)); void Check(const TString& token) { jwt::decoded_jwt decoded(token); diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp index f103a88ecc5e..dc1ab97e57b4 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp @@ -1,6 +1,6 @@ #include "exceptions.h" -namespace NYdb { +namespace NYdb::inline V2 { TYdbException::TYdbException(const TString& reason) { Append(reason); diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h index 38c94b168a1c..caabf50d0cd5 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h @@ -1,8 +1,10 @@ #pragma once +#include + #include -namespace NYdb { +namespace NYdb::inline V2 { class TYdbException : public yexception { public: diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp index 69b94f9b40d7..aa3cb1bbfe16 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp @@ -1,7 +1,7 @@ #include "handlers.h" #include -namespace NYdb { +namespace NYdb::inline V2 { void ThrowFatalError(const TString& str) { throw TContractViolation(str); diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h index 6af54a52f25b..25176c3e71b1 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h @@ -4,7 +4,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { void ThrowFatalError(const TString& str); diff --git a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h index cda28a75cff9..881814ca8ad5 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h +++ b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h @@ -2,48 +2,48 @@ #include -#define FLUENT_SETTING(type, name) \ +#define FLUENT_SETTING_DEPRECATED(type, name) \ type name##_; \ TSelf& name(const type& value) { \ name##_ = value; \ return static_cast(*this); \ } -#define FLUENT_SETTING_OPTIONAL(type, name) \ +#define FLUENT_SETTING_OPTIONAL_DEPRECATED(type, name) \ TMaybe name##_; \ TSelf& name(const TMaybe& value) { \ name##_ = value; \ return static_cast(*this); \ } -#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \ +#define FLUENT_SETTING_DEFAULT_DEPRECATED(type, name, defaultValue) \ type name##_ = defaultValue; \ TSelf& name(const type& value) { \ name##_ = value; \ return static_cast(*this); \ } -#define FLUENT_SETTING_FLAG(name) \ +#define FLUENT_SETTING_FLAG_DEPRECATED(name) \ bool name##_ = false; \ TSelf& name(bool value = true) { \ name##_ = value; \ return static_cast(*this); \ } -#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \ +#define FLUENT_SETTING_FLAG_ALIAS_DEPRECATED(name, other, value) \ TSelf& name() { \ other##_ = value; \ return static_cast(*this); \ } -#define FLUENT_SETTING_VECTOR(type, name) \ +#define FLUENT_SETTING_VECTOR_DEPRECATED(type, name) \ TVector name##_; \ TSelf& Append##name(const type& value) { \ name##_.push_back(value); \ return static_cast(*this); \ } -#define FLUENT_SETTING_OPTIONAL_VECTOR(type, name) \ +#define FLUENT_SETTING_OPTIONAL_VECTOR_DEPRECATED(type, name) \ TMaybe> name##_; \ TSelf& Append##name(const type& value) { \ if (!name##_) name##_ = TVector{}; \ diff --git a/ydb/public/sdk/cpp/client/ydb_types/fwd.h b/ydb/public/sdk/cpp/client/ydb_types/fwd.h new file mode 100644 index 000000000000..f01974b1fb00 --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_types/fwd.h @@ -0,0 +1,34 @@ +#pragma once + +namespace NYdb::inline V2 { + +template +struct TRequestSettings; + +template +struct TSimpleRequestSettings; + +template +struct TOperationRequestSettings; + +template +struct TS3Settings; + +class TStatus; +class TStreamPartStatus; + +class TOperation; + +class TYdbException; +class TContractViolation; + +class ICredentialsProvider; +class ICredentialsProviderFactory; + +class ITokenSource; + +namespace NStatusHelpers { +class TYdbErrorException; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp index 51cb186d0ebb..541baddb7a59 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp @@ -3,7 +3,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { class TOperation::TImpl { diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h index 52f2070eeefe..9c21c9fa9618 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -16,7 +18,7 @@ class Operation; } // namespace Operations } // namespace Ydb -namespace NYdb { +namespace NYdb::inline V2 { class TStatus; diff --git a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h index 5cf862fb6e6c..2a5f0f370eb0 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include "fluent_settings_helpers.h" #include @@ -7,17 +9,17 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { template struct TRequestSettings { using TSelf = TDerived; using THeader = std::vector>; - FLUENT_SETTING(TString, TraceId); - FLUENT_SETTING(TString, RequestType); - FLUENT_SETTING(THeader, Header); - FLUENT_SETTING(TDuration, ClientTimeout); + FLUENT_SETTING_DEPRECATED(TString, TraceId); + FLUENT_SETTING_DEPRECATED(TString, RequestType); + FLUENT_SETTING_DEPRECATED(THeader, Header); + FLUENT_SETTING_DEPRECATED(TDuration, ClientTimeout); TRequestSettings() = default; @@ -47,11 +49,11 @@ struct TOperationRequestSettings : public TSimpleRequestSettings { using TSelf = TDerived; /* Cancel/timeout operation settings available from 18-8 YDB server version */ - FLUENT_SETTING(TDuration, OperationTimeout); - FLUENT_SETTING(TDuration, CancelAfter); - FLUENT_SETTING(TDuration, ForgetAfter); - FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); - FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); + FLUENT_SETTING_DEPRECATED(TDuration, OperationTimeout); + FLUENT_SETTING_DEPRECATED(TDuration, CancelAfter); + FLUENT_SETTING_DEPRECATED(TDuration, ForgetAfter); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, UseClientTimeoutForOperation, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, ReportCostInfo, false); TOperationRequestSettings() = default; diff --git a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h index 7c5b80a69df2..417740cb816d 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h @@ -1,8 +1,10 @@ #pragma once +#include "fwd.h" + #include "fluent_settings_helpers.h" -namespace NYdb { +namespace NYdb::inline V2 { enum class ES3Scheme { HTTP = 1 /* "http" */, @@ -13,13 +15,13 @@ template struct TS3Settings { using TSelf = TDerived; - FLUENT_SETTING(TString, Endpoint); - FLUENT_SETTING_DEFAULT(ES3Scheme, Scheme, ES3Scheme::HTTPS); - FLUENT_SETTING(TString, Bucket); - FLUENT_SETTING(TString, AccessKey); - FLUENT_SETTING(TString, SecretKey); + FLUENT_SETTING_DEPRECATED(TString, Endpoint); + FLUENT_SETTING_DEFAULT_DEPRECATED(ES3Scheme, Scheme, ES3Scheme::HTTPS); + FLUENT_SETTING_DEPRECATED(TString, Bucket); + FLUENT_SETTING_DEPRECATED(TString, AccessKey); + FLUENT_SETTING_DEPRECATED(TString, SecretKey); // true by default for backward compatibility - FLUENT_SETTING_DEFAULT(bool, UseVirtualAddressing, true); + FLUENT_SETTING_DEFAULT_DEPRECATED(bool, UseVirtualAddressing, true); }; } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index 7be1b80a4946..b06da2e902a3 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -6,7 +6,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { class TStatus::TImpl { public: @@ -97,4 +97,26 @@ bool TStreamPartStatus::EOS() const { return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE; } +//////////////////////////////////////////////////////////////////////////////// + +namespace NStatusHelpers { + +void ThrowOnError(TStatus status, std::function onSuccess) { + if (!status.IsSuccess()) { + throw TYdbErrorException(status) << status; + } else { + onSuccess(status); + } +} + +void ThrowOnErrorOrPrintIssues(TStatus status) { + ThrowOnError(status, [](TStatus status) { + if (status.GetIssues()) { + Cerr << status; + } + }); +} + +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index c2f9b7997d44..072c4067c935 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -1,5 +1,8 @@ #pragma once +#include + +#include #include #include #include @@ -8,7 +11,7 @@ #include -namespace NYdb { +namespace NYdb::inline V2 { //! Internal status representation struct TPlainStatus; @@ -46,4 +49,27 @@ class TStreamPartStatus : public TStatus { bool EOS() const; }; +namespace NStatusHelpers { + +class TYdbErrorException : public TYdbException { +public: + TYdbErrorException(TStatus status) + : Status_(std::move(status)) + { + *this << status; + } + + friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { + return out << e.Status_; + } + +private: + TStatus Status_; +}; + +void ThrowOnError(TStatus status, std::function onSuccess = [](TStatus) {}); +void ThrowOnErrorOrPrintIssues(TStatus status); + +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status_codes.h b/ydb/public/sdk/cpp/client/ydb_types/status_codes.h index 906a2bf00b0b..f489c8623e2c 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status_codes.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status_codes.h @@ -1,8 +1,10 @@ #pragma once +#include "fwd.h" + #include -namespace NYdb { +namespace NYdb::inline V2 { constexpr size_t TRANSPORT_STATUSES_FIRST = 401000; constexpr size_t TRANSPORT_STATUSES_LAST = 401999; diff --git a/ydb/public/sdk/cpp/client/ydb_types/ydb.h b/ydb/public/sdk/cpp/client/ydb_types/ydb.h index 2321d94c2a0d..8d48eb9230cd 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/ydb.h +++ b/ydb/public/sdk/cpp/client/ydb_types/ydb.h @@ -1,8 +1,10 @@ #pragma once +#include "fwd.h" + #include "status_codes.h" -namespace NYdb { +namespace NYdb::inline V2 { enum class EDiscoveryMode { //! Block in ctor (driver or client if database and/or auth token is overridden by client settings) diff --git a/ydb/public/sdk/cpp/client/ydb_value/fwd.h b/ydb/public/sdk/cpp/client/ydb_value/fwd.h new file mode 100644 index 000000000000..f9f5df1abf5c --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_value/fwd.h @@ -0,0 +1,15 @@ +#pragma once + +namespace NYdb::inline V2 { + +class TType; +class TTypeParser; +class TTypeBuilder; +class TValue; +class TValueParser; +class TValueBuilder; + +template +class TValueBuilderBase; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index aedfeccbfc5f..d57f09761dab 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -18,7 +18,7 @@ #include #include -namespace NYdb { +namespace NYdb::inline V2 { static void CheckKind(TTypeParser::ETypeKind actual, TTypeParser::ETypeKind expected, const TString& method) { diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index 5b0d04eac9fe..3cce6e39fc46 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -1,5 +1,7 @@ #pragma once +#include "fwd.h" + #include #include @@ -10,7 +12,7 @@ namespace Ydb { class Value; } -namespace NYdb { +namespace NYdb::inline V2 { class TResultSetParser; diff --git a/ydb/public/sdk/cpp/client/ydb_value/ya.make b/ydb/public/sdk/cpp/client/ydb_value/ya.make index bf206ce371dd..f1efaa26f5e0 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_value/ya.make @@ -17,7 +17,3 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/sdk/cpp/examples/CMakeLists.txt b/ydb/public/sdk/cpp/examples/CMakeLists.txt new file mode 100644 index 000000000000..97119cff5e8f --- /dev/null +++ b/ydb/public/sdk/cpp/examples/CMakeLists.txt @@ -0,0 +1,8 @@ +add_subdirectory(basic_example) +add_subdirectory(bulk_upsert_simple) +add_subdirectory(pagination) +add_subdirectory(secondary_index) +add_subdirectory(secondary_index_builtin) +add_subdirectory(topic_reader) +add_subdirectory(ttl) +add_subdirectory(vector_index) diff --git a/ydb/public/sdk/cpp/examples/basic_example/CMakeLists.txt b/ydb/public/sdk/cpp/examples/basic_example/CMakeLists.txt new file mode 100644 index 000000000000..75d2d1b2538b --- /dev/null +++ b/ydb/public/sdk/cpp/examples/basic_example/CMakeLists.txt @@ -0,0 +1,38 @@ +add_executable(basic_example) + +target_link_libraries(basic_example PUBLIC + yutil + getopt + YDB-CPP-SDK::Query + YDB-CPP-SDK::Params + YDB-CPP-SDK::Driver +) + +target_sources(basic_example PRIVATE + main.cpp + basic_example_data.cpp + basic_example.cpp +) + +vcs_info(basic_example) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(basic_example PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(basic_example PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(basic_example PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/basic_example/basic_example.cpp b/ydb/public/sdk/cpp/examples/basic_example/basic_example.cpp index 8f7aef57d52c..126f9eecd140 100644 --- a/ydb/public/sdk/cpp/examples/basic_example/basic_example.cpp +++ b/ydb/public/sdk/cpp/examples/basic_example/basic_example.cpp @@ -1,30 +1,29 @@ #include "basic_example.h" -#include +#include -#include -#include +#include + +#include using namespace NYdb; using namespace NYdb::NQuery; +using namespace NYdb::NStatusHelpers; -class TYdbErrorException : public yexception { -public: - TYdbErrorException(const TStatus& status) - : Status(status) {} - - TStatus Status; -}; - -static void ThrowOnError(const TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; +template +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return std::to_string(opt.value()); } + return "(NULL)"; } -static void PrintStatus(const TStatus& status) { - Cerr << "Status: " << status.GetStatus() << Endl; - status.GetIssues().PrintTo(Cerr); +template <> +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return opt.value(); + } + return "(NULL)"; } /////////////////////////////////////////////////////////////////////////////// @@ -32,7 +31,7 @@ static void PrintStatus(const TStatus& status) { static void CreateTables(TQueryClient client) { //! Creates sample tables with the ExecuteQuery method ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( CREATE TABLE series ( series_id Uint64, title Utf8, @@ -40,12 +39,12 @@ static void CreateTables(TQueryClient client) { release_date Uint64, PRIMARY KEY (series_id) ); - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( CREATE TABLE seasons ( series_id Uint64, season_id Uint64, @@ -54,12 +53,12 @@ static void CreateTables(TQueryClient client) { last_aired Uint64, PRIMARY KEY (series_id, season_id) ); - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( CREATE TABLE episodes ( series_id Uint64, season_id Uint64, @@ -68,7 +67,7 @@ static void CreateTables(TQueryClient client) { air_date Uint64, PRIMARY KEY (series_id, season_id, episode_id) ); - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); } @@ -77,23 +76,23 @@ static void CreateTables(TQueryClient client) { static void DropTables(TQueryClient client) { ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( DROP TABLE series; - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( DROP TABLE seasons; - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( DROP TABLE episodes; - )"); + )"; return session.ExecuteQuery(query, TTxControl::NoTx()).GetValueSync(); })); } @@ -102,7 +101,7 @@ static void DropTables(TQueryClient client) { void FillTableData(TQueryClient client) { ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( DECLARE $seriesData AS List resultSet; + std::optional resultSet; ThrowOnError(client.RetryQuerySync([&resultSet](TSession session) { - auto query = Sprintf(R"( + auto query = R"( SELECT series_id, title, CAST(release_date AS Date) AS release_date FROM series WHERE series_id = 1; - )"); + )"; auto txControl = // Begin a new transaction with SerializableRW mode @@ -184,20 +183,20 @@ void SelectSimple(TQueryClient client) { TResultSetParser parser(*resultSet); while (parser.TryNextRow()) { - Cout << "> SelectSimple:" << Endl << "Series" - << ", Id: " << parser.ColumnParser("series_id").GetOptionalUint64() - << ", Title: " << parser.ColumnParser("title").GetOptionalUtf8() + std::cout << "> SelectSimple:" << std::endl << "Series" + << ", Id: " << OptionalToString(parser.ColumnParser("series_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) << ", Release date: " << parser.ColumnParser("release_date").GetOptionalDate()->FormatLocalTime("%Y-%m-%d") - << Endl; + << std::endl; } } void UpsertSimple(TQueryClient client) { ThrowOnError(client.RetryQuerySync([](TSession session) { - auto query = Sprintf(R"( + auto query = R"( UPSERT INTO episodes (series_id, season_id, episode_id, title) VALUES (2, 6, 1, "TBD"); - )"); + )"; return session.ExecuteQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); @@ -205,11 +204,11 @@ void UpsertSimple(TQueryClient client) { } void SelectWithParams(TQueryClient client) { - TMaybe resultSet; + std::optional resultSet; ThrowOnError(client.RetryQuerySync([&resultSet](TSession session) { ui64 seriesId = 2; ui64 seasonId = 3; - auto query = Sprintf(R"( + auto query = R"( DECLARE $seriesId AS Uint64; DECLARE $seasonId AS Uint64; @@ -218,7 +217,7 @@ void SelectWithParams(TQueryClient client) { INNER JOIN series AS sr ON sa.series_id = sr.series_id WHERE sa.series_id = $seriesId AND sa.season_id = $seasonId; - )"); + )"; auto params = TParamsBuilder() .AddParam("$seriesId") @@ -243,10 +242,10 @@ void SelectWithParams(TQueryClient client) { TResultSetParser parser(*resultSet); if (parser.TryNextRow()) { - Cout << "> SelectWithParams:" << Endl << "Season" - << ", Title: " << parser.ColumnParser("season_title").GetOptionalUtf8() - << ", Series title: " << parser.ColumnParser("series_title").GetOptionalUtf8() - << Endl; + std::cout << "> SelectWithParams:" << std::endl << "Season" + << ", Title: " << OptionalToString(parser.ColumnParser("season_title").GetOptionalUtf8()) + << ", Series title: " << OptionalToString(parser.ColumnParser("series_title").GetOptionalUtf8()) + << std::endl; } } @@ -255,13 +254,13 @@ void MultiStep(TQueryClient client) { ThrowOnError(client.RetryQuerySync([&resultSet](TSession session) { ui64 seriesId = 2; ui64 seasonId = 5; - auto query1 = Sprintf(R"( + auto query1 = R"( DECLARE $seriesId AS Uint64; DECLARE $seasonId AS Uint64; SELECT first_aired AS from_date FROM seasons WHERE series_id = $seriesId AND season_id = $seasonId; - )"); + )"; auto params1 = TParamsBuilder() .AddParam("$seriesId") @@ -303,14 +302,14 @@ void MultiStep(TQueryClient client) { TInstant toDate = userFunc(fromDate); // Construct next query based on the results of client logic - auto query2 = Sprintf(R"( + auto query2 = R"( DECLARE $seriesId AS Uint64; DECLARE $fromDate AS Uint64; DECLARE $toDate AS Uint64; SELECT season_id, episode_id, title, air_date FROM episodes WHERE series_id = $seriesId AND air_date >= $fromDate AND air_date <= $toDate; - )"); + )"; auto params2 = TParamsBuilder() .AddParam("$seriesId") @@ -340,15 +339,15 @@ void MultiStep(TQueryClient client) { })); // The end of the retried lambda TResultSetParser parser(*resultSet); - Cout << "> MultiStep:" << Endl; + std::cout << "> MultiStep:" << std::endl; while (parser.TryNextRow()) { auto airDate = TInstant::Days(*parser.ColumnParser("air_date").GetOptionalUint64()); - Cout << "Episode " << parser.ColumnParser("episode_id").GetOptionalUint64() - << ", Season: " << parser.ColumnParser("season_id").GetOptionalUint64() - << ", Title: " << parser.ColumnParser("title").GetOptionalUtf8() + std::cout << "Episode " << OptionalToString(parser.ColumnParser("episode_id").GetOptionalUint64()) + << ", Season: " << OptionalToString(parser.ColumnParser("season_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) << ", Air date: " << airDate.FormatLocalTime("%a %b %d, %Y") - << Endl; + << std::endl; } } @@ -367,11 +366,11 @@ void ExplicitTcl(TQueryClient client) { // Get newly created transaction id auto tx = beginResult.GetTransaction(); - auto query = Sprintf(R"( + auto query = R"( DECLARE $airDate AS Date; UPDATE episodes SET air_date = CAST($airDate AS Uint16) WHERE title = "TBD"; - )"); + )"; auto params = TParamsBuilder() .AddParam("$airDate") @@ -394,17 +393,17 @@ void ExplicitTcl(TQueryClient client) { } void StreamQuerySelect(TQueryClient client) { - Cout << "> StreamQuery:" << Endl; + std::cout << "> StreamQuery:" << std::endl; ThrowOnError(client.RetryQuerySync([](TQueryClient client) -> TStatus { - auto query = Sprintf(R"( + auto query = R"( DECLARE $series AS List; SELECT series_id, season_id, title, CAST(first_aired AS Date) AS first_aired FROM seasons WHERE series_id IN $series ORDER BY season_id; - )"); + )"; auto paramsBuilder = TParamsBuilder(); auto& listParams = paramsBuilder @@ -446,16 +445,16 @@ void StreamQuerySelect(TQueryClient client) { auto rs = streamPart.ExtractResultSet(); TResultSetParser parser(rs); while (parser.TryNextRow()) { - Cout << "Season" - << ", SeriesId: " << parser.ColumnParser("series_id").GetOptionalUint64() - << ", SeasonId: " << parser.ColumnParser("season_id").GetOptionalUint64() - << ", Title: " << parser.ColumnParser("title").GetOptionalUtf8() + std::cout << "Season" + << ", SeriesId: " << OptionalToString(parser.ColumnParser("series_id").GetOptionalUint64()) + << ", SeasonId: " << OptionalToString(parser.ColumnParser("season_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) << ", Air date: " << parser.ColumnParser("first_aired").GetOptionalDate()->FormatLocalTime("%Y-%m-%d") - << Endl; + << std::endl; } } } - return TStatus(EStatus::SUCCESS, NYql::TIssues()); + return TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()); })); } @@ -485,8 +484,7 @@ bool Run(const TDriver& driver) { DropTables(client); } catch (const TYdbErrorException& e) { - Cerr << "Execution failed due to fatal error:" << Endl; - PrintStatus(e.Status); + std::cerr << "Execution failed due to fatal error: " << e.what() << std::endl; return false; } diff --git a/ydb/public/sdk/cpp/examples/basic_example/basic_example.h b/ydb/public/sdk/cpp/examples/basic_example/basic_example.h index 75e4b84a5c73..04260a5f2974 100644 --- a/ydb/public/sdk/cpp/examples/basic_example/basic_example.h +++ b/ydb/public/sdk/cpp/examples/basic_example/basic_example.h @@ -1,10 +1,10 @@ #pragma once -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include NYdb::TParams GetTablesDataParams(); diff --git a/ydb/public/sdk/cpp/examples/basic_example/basic_example_data.cpp b/ydb/public/sdk/cpp/examples/basic_example/basic_example_data.cpp index 6e9cecc29b98..19c639d6fe35 100644 --- a/ydb/public/sdk/cpp/examples/basic_example/basic_example_data.cpp +++ b/ydb/public/sdk/cpp/examples/basic_example/basic_example_data.cpp @@ -4,12 +4,12 @@ using namespace NYdb; using namespace NYdb::NTable; struct TSeries { - ui64 SeriesId; - TString Title; + uint64_t SeriesId; + std::string Title; TInstant ReleaseDate; - TString SeriesInfo; + std::string SeriesInfo; - TSeries(ui64 seriesId, const TString& title, const TInstant& releaseDate, const TString& seriesInfo) + TSeries(uint64_t seriesId, const std::string& title, const TInstant& releaseDate, const std::string& seriesInfo) : SeriesId(seriesId) , Title(title) , ReleaseDate(releaseDate) @@ -17,13 +17,13 @@ struct TSeries { }; struct TSeason { - ui64 SeriesId; - ui64 SeasonId; - TString Title; + uint64_t SeriesId; + uint64_t SeasonId; + std::string Title; TInstant FirstAired; TInstant LastAired; - TSeason(ui64 seriesId, ui64 seasonId, const TString& title, const TInstant& firstAired, const TInstant& lastAired) + TSeason(uint64_t seriesId, uint64_t seasonId, const std::string& title, const TInstant& firstAired, const TInstant& lastAired) : SeriesId(seriesId) , SeasonId(seasonId) , Title(title) @@ -32,13 +32,13 @@ struct TSeason { }; struct TEpisode { - ui64 SeriesId; - ui64 SeasonId; - ui64 EpisodeId; - TString Title; + uint64_t SeriesId; + uint64_t SeasonId; + uint64_t EpisodeId; + std::string Title; TInstant AirDate; - TEpisode(ui64 seriesId, ui64 seasonId, ui64 episodeId, const TString& title, const TInstant& airDate) + TEpisode(uint64_t seriesId, uint64_t seasonId, uint64_t episodeId, const std::string& title, const TInstant& airDate) : SeriesId(seriesId) , SeasonId(seasonId) , EpisodeId(episodeId) @@ -47,7 +47,7 @@ struct TEpisode { }; TParams GetTablesDataParams() { - TVector seriesData = { + std::vector seriesData = { TSeries(1, "IT Crowd", TInstant::ParseIso8601("2006-02-03"), "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by " "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry."), @@ -56,7 +56,7 @@ TParams GetTablesDataParams() { "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.") }; - TVector seasonsData = { + std::vector seasonsData = { TSeason(1, 1, "Season 1", TInstant::ParseIso8601("2006-02-03"), TInstant::ParseIso8601("2006-03-03")), TSeason(1, 2, "Season 2", TInstant::ParseIso8601("2007-08-24"), TInstant::ParseIso8601("2007-09-28")), TSeason(1, 3, "Season 3", TInstant::ParseIso8601("2008-11-21"), TInstant::ParseIso8601("2008-12-26")), @@ -68,7 +68,7 @@ TParams GetTablesDataParams() { TSeason(2, 5, "Season 5", TInstant::ParseIso8601("2018-03-25"), TInstant::ParseIso8601("2018-05-13")) }; - TVector episodesData = { + std::vector episodesData = { TEpisode(1, 1, 1, "Yesterday's Jam", TInstant::ParseIso8601("2006-02-03")), TEpisode(1, 1, 2, "Calamity Jen", TInstant::ParseIso8601("2006-02-03")), TEpisode(1, 1, 3, "Fifty-Fifty", TInstant::ParseIso8601("2006-02-10")), diff --git a/ydb/public/sdk/cpp/examples/basic_example/main.cpp b/ydb/public/sdk/cpp/examples/basic_example/main.cpp index b2a83ff33769..5bc2571762ef 100644 --- a/ydb/public/sdk/cpp/examples/basic_example/main.cpp +++ b/ydb/public/sdk/cpp/examples/basic_example/main.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include using namespace NLastGetopt; using namespace NYdb; @@ -12,12 +12,19 @@ void StopHandler(int) { exit(1); } +std::string ReadFile(const std::string& filename) { + std::ifstream input(filename); + std::stringstream data; + data << input.rdbuf(); + return data.str(); +} + int main(int argc, char** argv) { TOpts opts = TOpts::Default(); - TString endpoint; - TString database; - TString certPath; + std::string endpoint; + std::string database; + std::string certPath; opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&endpoint); @@ -34,10 +41,10 @@ int main(int argc, char** argv) { auto driverConfig = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); if (!certPath.empty()) { - TString cert = TFileInput(certPath).ReadAll(); + std::string cert = ReadFile(certPath); driverConfig.UseSecureConnection(cert); } diff --git a/ydb/public/sdk/cpp/examples/basic_example/ya.make b/ydb/public/sdk/cpp/examples/basic_example/ya.make index d503a1a7527a..e03169f3ed09 100644 --- a/ydb/public/sdk/cpp/examples/basic_example/ya.make +++ b/ydb/public/sdk/cpp/examples/basic_example/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp basic_example_data.cpp @@ -8,9 +10,9 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/ydb_params - ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/driver ) END() diff --git a/ydb/public/sdk/cpp/examples/bulk_upsert_simple/CMakeLists.txt b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/CMakeLists.txt new file mode 100644 index 000000000000..4f7c3eca7f90 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/CMakeLists.txt @@ -0,0 +1,34 @@ +add_executable(bulk_upsert_simple) + +target_link_libraries(bulk_upsert_simple PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(bulk_upsert_simple PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/bulk_upsert_simple/main.cpp +) + +vcs_info(bulk_upsert_simple) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(bulk_upsert_simple PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(bulk_upsert_simple PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(bulk_upsert_simple PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/bulk_upsert_simple/main.cpp b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/main.cpp index 5fb2740d268a..f30d16279509 100644 --- a/ydb/public/sdk/cpp/examples/bulk_upsert_simple/main.cpp +++ b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/main.cpp @@ -1,31 +1,25 @@ -#include +#include #include -#include -#include - -Y_DECLARE_OUT_SPEC(, NYdb::TStatus, stream, value) { - stream << "Status: " << value.GetStatus() << Endl; - value.GetIssues().PrintTo(stream); -} +#include constexpr size_t BATCH_SIZE = 1000; struct TLogMessage { - TString App; - TString Host; + std::string App; + std::string Host; TInstant Timestamp; - ui32 HttpCode; - TString Message; + uint32_t HttpCode; + std::string Message; }; -void GetLogBatch(ui64 logOffset, TVector& logBatch) { +void GetLogBatch(uint64_t logOffset, std::vector& logBatch) { logBatch.clear(); for (size_t i = 0; i < BATCH_SIZE; ++i) { TLogMessage message; - message.App = "App_" + ToString(logOffset % 10); - message.Host = "192.168.0." + ToString(logOffset % 11); + message.App = "App_" + std::to_string(logOffset % 10); + message.Host = "192.168.0." + std::to_string(logOffset % 11); message.Timestamp = TInstant::Now() + TDuration::MilliSeconds(i % 1000); message.HttpCode = 200; message.Message = i % 2 ? "GET / HTTP/1.1" : "GET /images/logo.png HTTP/1.1"; @@ -33,7 +27,7 @@ void GetLogBatch(ui64 logOffset, TVector& logBatch) { } } -bool WriteLogBatch(NYdb::NTable::TTableClient& tableClient, const TString& table, const TVector& logBatch, +bool WriteLogBatch(NYdb::NTable::TTableClient& tableClient, const std::string& table, const std::vector& logBatch, const NYdb::NTable::TRetryOperationSettings& retrySettings) { NYdb::TValueBuilder rows; @@ -59,14 +53,14 @@ bool WriteLogBatch(NYdb::NTable::TTableClient& tableClient, const TString& table auto status = tableClient.RetryOperationSync(bulkUpsertOperation, retrySettings); if (!status.IsSuccess()) { - Cerr << Endl << "Write failed with status: " << (const NYdb::TStatus&)status << Endl; + std::cerr << std::endl << "Write failed with status: " << ToString(status) << std::endl; return false; } return true; } -bool CreateLogTable(NYdb::NTable::TTableClient& client, const TString& table) { - Cerr << "Create table " << table << "\n"; +bool CreateLogTable(NYdb::NTable::TTableClient& client, const std::string& table) { + std::cerr << "Create table " << table << "\n"; NYdb::NTable::TRetryOperationSettings settings; auto status = client.RetryOperationSync([&table](NYdb::NTable::TSession session) { @@ -83,13 +77,13 @@ bool CreateLogTable(NYdb::NTable::TTableClient& client, const TString& table) { }, settings); if (!status.IsSuccess()) { - Cerr << "Create table failed with status: " << status << Endl; + std::cerr << "Create table failed with status: " << ToString(status) << std::endl; return false; } return true; } -bool Run(const NYdb::TDriver &driver, const TString &table, ui32 batchCount) { +bool Run(const NYdb::TDriver &driver, const std::string &table, uint32_t batchCount) { NYdb::NTable::TTableClient client(driver); if (!CreateLogTable(client, table)) { return false; @@ -100,28 +94,28 @@ bool Run(const NYdb::TDriver &driver, const TString &table, ui32 batchCount) { .Idempotent(true) .MaxRetries(20); - TVector logBatch; - for (ui32 offset = 0; offset < batchCount; ++offset) { + std::vector logBatch; + for (uint32_t offset = 0; offset < batchCount; ++offset) { GetLogBatch(offset, logBatch); if (!WriteLogBatch(client, table, logBatch, writeRetrySettings)) { return false; } - Cerr << "."; + std::cerr << "."; } - Cerr << Endl << "Done." << Endl; + std::cerr << std::endl << "Done." << std::endl; return true; } -TString JoinPath(const TString& basePath, const TString& path) { +std::string JoinPath(const std::string& basePath, const std::string& path) { if (basePath.empty()) { return path; } - TPathSplitUnix prefixPathSplit(basePath); - prefixPathSplit.AppendComponent(path); + std::filesystem::path prefixPathSplit(basePath); + prefixPathSplit /= path; - return prefixPathSplit.Reconstruct(); + return prefixPathSplit; } int main(int argc, char** argv) { @@ -129,10 +123,10 @@ int main(int argc, char** argv) { TOpts opts = TOpts::Default(); - TString endpoint; - TString database; - TString table = "bulk_upsert_example"; - ui32 count = 1000; + std::string endpoint; + std::string database; + std::string table = "bulk_upsert_example"; + uint32_t count = 1000; opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&endpoint); @@ -143,15 +137,14 @@ int main(int argc, char** argv) { opts.AddLongOption('c', "count", "count requests").Optional().RequiredArgument("NUM") .StoreResult(&count).DefaultValue(count); - TOptsParseResult res(&opts, argc, argv); - Y_UNUSED(res); + [[maybe_unused]] TOptsParseResult res(&opts, argc, argv); table = JoinPath(database, table); auto driverConfig = NYdb::TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); NYdb::TDriver driver(driverConfig); diff --git a/ydb/public/sdk/cpp/examples/bulk_upsert_simple/ya.make b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/ya.make index d2ad2b027dbd..99f546b0e4cd 100644 --- a/ydb/public/sdk/cpp/examples/bulk_upsert_simple/ya.make +++ b/ydb/public/sdk/cpp/examples/bulk_upsert_simple/ya.make @@ -1,12 +1,14 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp ) PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/examples/pagination/CMakeLists.txt b/ydb/public/sdk/cpp/examples/pagination/CMakeLists.txt new file mode 100644 index 000000000000..0936f3855855 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/pagination/CMakeLists.txt @@ -0,0 +1,36 @@ +add_executable(pagination) + +target_link_libraries(pagination PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(pagination PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/pagination/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/pagination/pagination_data.cpp + ${YDB_SDK_SOURCE_DIR}/examples/pagination/pagination.cpp +) + +vcs_info(pagination) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(pagination PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(pagination PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(pagination PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() \ No newline at end of file diff --git a/ydb/public/sdk/cpp/examples/pagination/main.cpp b/ydb/public/sdk/cpp/examples/pagination/main.cpp index 04a9bce64af8..5345f5b83fa2 100644 --- a/ydb/public/sdk/cpp/examples/pagination/main.cpp +++ b/ydb/public/sdk/cpp/examples/pagination/main.cpp @@ -1,7 +1,6 @@ #include "pagination.h" #include -#include using namespace NLastGetopt; using namespace NYdb; @@ -13,9 +12,9 @@ void StopHandler(int) { int main(int argc, char** argv) { TOpts opts = TOpts::Default(); - TString endpoint; - TString database; - TString path; + std::string endpoint; + std::string database; + std::string path; opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&endpoint); opts.AddLongOption('d', "database", "YDB database name").Required().RequiredArgument("PATH") @@ -35,7 +34,7 @@ int main(int argc, char** argv) { auto driverConfig = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); TDriver driver(driverConfig); if (!Run(driver, path)) { diff --git a/ydb/public/sdk/cpp/examples/pagination/pagination.cpp b/ydb/public/sdk/cpp/examples/pagination/pagination.cpp index a59727251157..4a1a1300db8e 100644 --- a/ydb/public/sdk/cpp/examples/pagination/pagination.cpp +++ b/ydb/public/sdk/cpp/examples/pagination/pagination.cpp @@ -1,45 +1,29 @@ #include "pagination.h" -#include -#include +#include + +#include +#include using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; -const ui32 MaxPages = 10; - -class TYdbErrorException : public yexception { -public: - TYdbErrorException(const TStatus& status) - : Status(status) {} - - TStatus Status; -}; - -static void ThrowOnError(const TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } -} - -static void PrintStatus(const TStatus& status) { - Cerr << "Status: " << status.GetStatus() << Endl; - status.GetIssues().PrintTo(Cerr); -} +const uint32_t MaxPages = 10; -static TString JoinPath(const TString& basePath, const TString& path) { +static std::string JoinPath(const std::string& basePath, const std::string& path) { if (basePath.empty()) { return path; } - TPathSplitUnix prefixPathSplit(basePath); - prefixPathSplit.AppendComponent(path); + std::filesystem::path prefixPathSplit(basePath); + prefixPathSplit /= path; - return prefixPathSplit.Reconstruct(); + return prefixPathSplit; } //! Creates sample table with CrateTable API. -static void CreateTable(TTableClient client, const TString& path) { +static void CreateTable(TTableClient client, const std::string& path) { ThrowOnError(client.RetryOperationSync([path](TSession session) { auto schoolsDesc = TTableBuilder() .AddNullableColumn("city", EPrimitiveType::Utf8) @@ -53,10 +37,10 @@ static void CreateTable(TTableClient client, const TString& path) { } //! Fills sample tables with data in single parameterized data query. -static TStatus FillTableDataTransaction(TSession& session, const TString& path) { - auto query = Sprintf(R"( +static TStatus FillTableDataTransaction(TSession& session, const std::string& path) { + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $schoolsData AS List& resultSet) +static TStatus SelectPagingTransaction(TSession& session, const std::string& path, + uint64_t pageLimit, const std::string& lastCity, uint32_t lastNumber, std::optional& resultSet) { - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $limit AS Uint64; DECLARE $lastCity AS Utf8; @@ -111,7 +95,7 @@ static TStatus SelectPagingTransaction(TSession& session, const TString& path, SELECT * FROM $union ORDER BY city, number LIMIT $limit; - )", path.c_str()); + )", path); auto params = session.GetParamsBuilder() .AddParam("$limit") @@ -137,8 +121,8 @@ static TStatus SelectPagingTransaction(TSession& session, const TString& path, return result; } -bool SelectPaging(TTableClient client, const TString& path, ui64 pageLimit, TString& lastCity, ui32& lastNumber) { - TMaybe resultSet; +bool SelectPaging(TTableClient client, const std::string& path, uint64_t pageLimit, std::string& lastCity, uint32_t& lastNumber) { + std::optional resultSet; ThrowOnError(client.RetryOperationSync([path, pageLimit, &lastCity, lastNumber, &resultSet](TSession session) { return SelectPagingTransaction(session, path, pageLimit, lastCity, lastNumber, resultSet); })); @@ -149,14 +133,14 @@ bool SelectPaging(TTableClient client, const TString& path, ui64 pageLimit, TStr return false; } do { - lastCity = parser.ColumnParser("city").GetOptionalUtf8().GetRef(); - lastNumber = parser.ColumnParser("number").GetOptionalUint32().GetRef(); - Cout << lastCity << ", Школа №" << lastNumber << ", Адрес: " << parser.ColumnParser("address").GetOptionalUtf8() << Endl; + lastCity = parser.ColumnParser("city").GetOptionalUtf8().value(); + lastNumber = parser.ColumnParser("number").GetOptionalUint32().value(); + std::cout << lastCity << ", Школа №" << lastNumber << ", Адрес: " << parser.ColumnParser("address").GetOptionalUtf8().value_or("(NULL)") << std::endl; } while (parser.TryNextRow()); return true; } -bool Run(const TDriver& driver, const TString& path) { +bool Run(const TDriver& driver, const std::string& path) { TTableClient client(driver); try { @@ -166,24 +150,23 @@ bool Run(const TDriver& driver, const TString& path) { return FillTableDataTransaction(session, path); })); - ui64 limit = 3; - TString lastCity; - ui32 lastNumber = 0; - ui32 page = 0; + uint64_t limit = 3; + std::string lastCity; + uint32_t lastNumber = 0; + uint32_t page = 0; bool pageNotEmpty = true; - Cout << "> Pagination, Limit=" << limit << Endl; + std::cout << "> Pagination, Limit=" << limit << std::endl; // show first MaxPages=10 pages: while (pageNotEmpty && page <= MaxPages) { ++page; - Cout << "> Page " << page << ":" << Endl; + std::cout << "> Page " << page << ":" << std::endl; pageNotEmpty = SelectPaging(client, path, limit, lastCity, lastNumber); } } catch (const TYdbErrorException& e) { - Cerr << "Execution failed due to fatal error:" << Endl; - PrintStatus(e.Status); + std::cerr << "Execution failed due to fatal error: " << e.what() << std::endl; return false; } diff --git a/ydb/public/sdk/cpp/examples/pagination/pagination.h b/ydb/public/sdk/cpp/examples/pagination/pagination.h index d419c216680a..99e0804f4bbe 100644 --- a/ydb/public/sdk/cpp/examples/pagination/pagination.h +++ b/ydb/public/sdk/cpp/examples/pagination/pagination.h @@ -1,8 +1,8 @@ #pragma once -#include -#include +#include +#include NYdb::TParams GetTablesDataParams(); -bool Run(const NYdb::TDriver& driver, const TString& path); +bool Run(const NYdb::TDriver& driver, const std::string& path); diff --git a/ydb/public/sdk/cpp/examples/pagination/pagination_data.cpp b/ydb/public/sdk/cpp/examples/pagination/pagination_data.cpp index 30efd41f43c6..a5287c97a107 100644 --- a/ydb/public/sdk/cpp/examples/pagination/pagination_data.cpp +++ b/ydb/public/sdk/cpp/examples/pagination/pagination_data.cpp @@ -4,18 +4,18 @@ using namespace NYdb; using namespace NYdb::NTable; struct TSchool { - TString City; - ui32 Number; - TString Address; + std::string City; + uint32_t Number; + std::string Address; - TSchool(const TString& city, ui32 number, const TString& address) + TSchool(const std::string& city, uint32_t number, const std::string& address) : City(city) , Number(number) , Address(address) {} }; TParams GetTablesDataParams() { - TVector schoolsData = { + std::vector schoolsData = { TSchool("Орлов", 1, "Ст.Халтурина, 2"), TSchool("Орлов", 2, "Свободы, 4"), TSchool("Яранск", 1, "Гоголя, 25"), diff --git a/ydb/public/sdk/cpp/examples/pagination/ya.make b/ydb/public/sdk/cpp/examples/pagination/ya.make index 94bce03191bf..d6517f9df8d4 100644 --- a/ydb/public/sdk/cpp/examples/pagination/ya.make +++ b/ydb/public/sdk/cpp/examples/pagination/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp pagination_data.cpp @@ -8,7 +10,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/examples/secondary_index/CMakeLists.txt b/ydb/public/sdk/cpp/examples/secondary_index/CMakeLists.txt new file mode 100644 index 000000000000..6030de5f7f08 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/secondary_index/CMakeLists.txt @@ -0,0 +1,41 @@ +add_executable(secondary_index) + +target_link_libraries(secondary_index PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(secondary_index PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_create.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_delete.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_drop.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_generate.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_list.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index/secondary_index_update.cpp +) + +vcs_info(secondary_index) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(secondary_index PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(secondary_index PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(secondary_index PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/secondary_index/main.cpp b/ydb/public/sdk/cpp/examples/secondary_index/main.cpp index 830926298cde..3fb520bb21ce 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/main.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/main.cpp @@ -1,18 +1,17 @@ #include "secondary_index.h" -#include - using namespace NLastGetopt; using namespace NYdb; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// int main(int argc, char** argv) { TOpts opts = TOpts::Default(); - TString endpoint; - TString database; - TString prefix; + std::string endpoint; + std::string database; + std::string prefix; opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&endpoint); @@ -31,7 +30,7 @@ int main(int argc, char** argv) { ECmd cmd = ParseCmd(*argv); if (cmd == ECmd::NONE) { - Cerr << "Unsupported command '" << *argv << "'" << Endl; + std::cerr << "Unsupported command '" << *argv << "'" << std::endl; return 1; } @@ -42,7 +41,7 @@ int main(int argc, char** argv) { auto config = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); TDriver driver(config); try { @@ -63,10 +62,10 @@ int main(int argc, char** argv) { return RunDeleteSeries(driver, prefix, argc, argv); } } catch (const TYdbErrorException& e) { - Cerr << "Execution failed: " << e << Endl; + std::cerr << "Execution failed: " << e.what() << std::endl; return 1; } - Y_UNREACHABLE(); + __builtin_unreachable(); return 1; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.cpp index bd2ec617a9af..01177689e858 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.cpp @@ -1,8 +1,8 @@ #include "secondary_index.h" -#include +#include -TString GetCmdList() { +std::string GetCmdList() { return "create_tables, drop_tables, update_views, list, generate, delete"; } @@ -28,13 +28,13 @@ ECmd ParseCmd(const char* cmd) { return ECmd::NONE; } -TString JoinPath(const TString& prefix, const TString& path) { +std::string JoinPath(const std::string& prefix, const std::string& path) { if (prefix.empty()) { return path; } - TPathSplitUnix prefixPathSplit(prefix); - prefixPathSplit.AppendComponent(path); + std::filesystem::path prefixPathSplit(prefix); + prefixPathSplit /= path; - return prefixPathSplit.Reconstruct(); + return prefixPathSplit; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.h b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.h index e60e61c8cac1..273b4edc2676 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.h +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index.h @@ -1,15 +1,11 @@ #pragma once -#include -#include +#include +#include #include -#include -#include -#include -#include -#include +#include //////////////////////////////////////////////////////////////////////////////// @@ -17,11 +13,11 @@ #define TABLE_SERIES_REV_VIEWS "series_rev_views" struct TSeries { - ui64 SeriesId; - TString Title; - TString SeriesInfo; + uint64_t SeriesId; + std::string Title; + std::string SeriesInfo; TInstant ReleaseDate; - ui64 Views; + uint64_t Views; }; //////////////////////////////////////////////////////////////////////////////// @@ -36,45 +32,18 @@ enum class ECmd { DELETE_SERIES, }; -TString GetCmdList(); +std::string GetCmdList(); ECmd ParseCmd(const char* cmd); //////////////////////////////////////////////////////////////////////////////// -TString JoinPath(const TString& prefix, const TString& path); +std::string JoinPath(const std::string& prefix, const std::string& path); //////////////////////////////////////////////////////////////////////////////// -class TYdbErrorException : public yexception { -public: - TYdbErrorException(NYdb::TStatus status) - : Status(std::move(status)) - { } - - friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { - out << "Status: " << e.Status.GetStatus(); - if (e.Status.GetIssues()) { - out << Endl; - e.Status.GetIssues().PrintTo(out); - } - return out; - } - -private: - NYdb::TStatus Status; -}; - -inline void ThrowOnError(NYdb::TStatus status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -int RunCreateTables(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); -int RunDropTables(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); -int RunUpdateViews(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); -int RunListSeries(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); -int RunGenerateSeries(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); -int RunDeleteSeries(NYdb::TDriver& driver, const TString& prefix, int argc, char** argv); +int RunCreateTables(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); +int RunDropTables(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); +int RunUpdateViews(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); +int RunListSeries(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); +int RunGenerateSeries(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); +int RunDeleteSeries(NYdb::TDriver& driver, const std::string& prefix, int argc, char** argv); diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_create.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_create.cpp index f96ca7ff08b9..ac0f556967ca 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_create.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_create.cpp @@ -3,10 +3,11 @@ using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// -static void CreateSeriesTable(TTableClient& client, const TString& prefix) { +static void CreateSeriesTable(TTableClient& client, const std::string& prefix) { ThrowOnError(client.RetryOperationSync([prefix](TSession session) { auto desc = TTableBuilder() .AddNullableColumn("series_id", EPrimitiveType::Uint64) @@ -21,7 +22,7 @@ static void CreateSeriesTable(TTableClient& client, const TString& prefix) { })); } -static void CreateSeriesIndexTable(TTableClient& client, const TString& prefix) { +static void CreateSeriesIndexTable(TTableClient& client, const std::string& prefix) { ThrowOnError(client.RetryOperationSync([prefix](TSession session) { auto desc = TTableBuilder() .AddNullableColumn("rev_views", EPrimitiveType::Uint64) @@ -33,9 +34,9 @@ static void CreateSeriesIndexTable(TTableClient& client, const TString& prefix) })); } -int RunCreateTables(TDriver& driver, const TString& prefix, int argc, char**) { +int RunCreateTables(TDriver& driver, const std::string& prefix, int argc, char**) { if (argc > 1) { - Cerr << "Unexpected arguments after create_tables" << Endl; + std::cerr << "Unexpected arguments after create_tables" << std::endl; return 1; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_delete.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_delete.cpp index 27e8695f2920..5f38f76f0adf 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_delete.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_delete.cpp @@ -3,13 +3,14 @@ using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// -static TStatus DeleteSeries(TSession& session, const TString& prefix, ui64 seriesId, ui64& deletedCount) { - auto queryText = Sprintf(R"( +static TStatus DeleteSeries(TSession& session, const std::string& prefix, uint64_t seriesId, uint64_t& deletedCount) { + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $seriesId AS Uint64; @@ -18,7 +19,7 @@ static TStatus DeleteSeries(TSession& session, const TString& prefix, ui64 serie $data = ( SELECT series_id, ($maxUint64 - views) AS rev_views - FROM [series] + FROM series WHERE series_id = $seriesId ); @@ -29,7 +30,7 @@ static TStatus DeleteSeries(TSession& session, const TString& prefix, ui64 serie ON SELECT rev_views, series_id FROM $data; SELECT COUNT(*) AS cnt FROM $data; - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -58,22 +59,22 @@ static TStatus DeleteSeries(TSession& session, const TString& prefix, ui64 serie return result; } -int RunDeleteSeries(TDriver& driver, const TString& prefix, int argc, char** argv) { +int RunDeleteSeries(TDriver& driver, const std::string& prefix, int argc, char** argv) { TOpts opts = TOpts::Default(); - ui64 seriesId; + uint64_t seriesId; opts.AddLongOption("id", "Series id").Required().RequiredArgument("NUM") .StoreResult(&seriesId); TOptsParseResult res(&opts, argc, argv); - ui64 deletedCount = 0; + uint64_t deletedCount = 0; TTableClient client(driver); ThrowOnError(client.RetryOperationSync([&](TSession session) -> TStatus { return DeleteSeries(session, prefix, seriesId, deletedCount); })); - Cout << "Deleted " << deletedCount << " rows" << Endl; + std::cout << "Deleted " << deletedCount << " rows" << std::endl; return 0; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_drop.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_drop.cpp index 810dd49543a4..d7531786d4f7 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_drop.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_drop.cpp @@ -3,18 +3,19 @@ using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// -static void DropTable(TTableClient& client, const TString& path) { +static void DropTable(TTableClient& client, const std::string& path) { ThrowOnError(client.RetryOperationSync([path](TSession session) { return session.DropTable(path).ExtractValueSync(); })); } -int RunDropTables(TDriver& driver, const TString& prefix, int argc, char**) { +int RunDropTables(TDriver& driver, const std::string& prefix, int argc, char**) { if (argc > 1) { - Cerr << "Unexpected arguments after drop_tables" << Endl; + std::cerr << "Unexpected arguments after drop_tables" << std::endl; return 1; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_generate.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_generate.cpp index a36310185759..70398166929f 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_generate.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_generate.cpp @@ -1,11 +1,14 @@ #include "secondary_index.h" -#include #include +#include +#include + using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; namespace { @@ -26,7 +29,8 @@ class TExecutor { } void Stop(bool rethrow = true) { - with_lock (Lock) { + { + std::lock_guard guard(Lock); Stopped = true; } Wait(rethrow); @@ -35,16 +39,16 @@ class TExecutor { void Wait(bool rethrow = true) { Queue.Stop(); if (rethrow) { - with_lock (Lock) { - if (Exception) { - std::rethrow_exception(Exception); - } + std::lock_guard guard(Lock); + if (Exception) { + std::rethrow_exception(Exception); } } } bool Execute(std::function callback) { - with_lock (Lock) { + { + std::lock_guard guard(Lock); if (Stopped) { if (Exception) { std::rethrow_exception(Exception); @@ -54,9 +58,9 @@ class TExecutor { } } - THolder task = MakeHolder(this, std::move(callback)); - if (Queue.Add(task.Get())) { - Y_UNUSED(task.Release()); + std::unique_ptr task = std::make_unique(this, std::move(callback)); + if (Queue.Add(task.get())) { + Y_UNUSED(task.release()); return true; } @@ -74,8 +78,9 @@ class TExecutor { { } void Process(void*) override { - THolder self(this); - with_lock (Owner->Lock) { + std::unique_ptr self(this); + { + std::lock_guard guard(Owner->Lock); if (Owner->Stopped) { return; } @@ -84,11 +89,10 @@ class TExecutor { auto callback = std::move(Callback); callback(); } catch (...) { - with_lock (Owner->Lock) { - if (!Owner->Stopped && !Owner->Exception) { - Owner->Stopped = true; - Owner->Exception = std::current_exception(); - } + std::lock_guard guard(Owner->Lock); + if (!Owner->Stopped && !Owner->Exception) { + Owner->Stopped = true; + Owner->Exception = std::current_exception(); } } } @@ -105,10 +109,10 @@ class TExecutor { //////////////////////////////////////////////////////////////////////////////// -static TStatus InsertSeries(TSession& session, const TString& prefix, const TSeries& series) { - auto queryText = Sprintf(R"( +static TStatus InsertSeries(TSession& session, const std::string& prefix, const TSeries& series) { + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $seriesId AS Uint64; DECLARE $title AS Utf8; @@ -126,7 +130,7 @@ static TStatus InsertSeries(TSession& session, const TString& prefix, const TSer -- Insert above already verified series_id is unique, so it is safe to use upsert UPSERT INTO series_rev_views (rev_views, series_id) VALUES ($revViews, $seriesId); - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -160,11 +164,11 @@ static TStatus InsertSeries(TSession& session, const TString& prefix, const TSer return result; } -int RunGenerateSeries(TDriver& driver, const TString& prefix, int argc, char** argv) { +int RunGenerateSeries(TDriver& driver, const std::string& prefix, int argc, char** argv) { TOpts opts = TOpts::Default(); - ui64 seriesId = 1; - ui64 count = 10; + uint64_t seriesId = 1; + uint64_t count = 10; size_t threads = 10; opts.AddLongOption("start", "First id to generate").Optional().RequiredArgument("NUM") @@ -180,14 +184,16 @@ int RunGenerateSeries(TDriver& driver, const TString& prefix, int argc, char** a executor.Start(threads); size_t generated = 0; + std::mt19937_64 engine; + std::uniform_int_distribution dist(0, 1000000); while (count > 0) { - bool ok = executor.Execute([&client, &prefix, seriesId] { + bool ok = executor.Execute([views = dist(engine), &client, &prefix, seriesId] { TSeries series; series.SeriesId = seriesId; series.Title = TStringBuilder() << "Name " << seriesId; series.SeriesInfo = TStringBuilder() << "Info " << seriesId; series.ReleaseDate = TInstant::Days(TInstant::Now().Days()); - series.Views = RandomNumber(1000000); + series.Views = views; ThrowOnError(client.RetryOperationSync([&prefix, &series](TSession session) -> TStatus { return InsertSeries(session, prefix, series); })); @@ -201,6 +207,6 @@ int RunGenerateSeries(TDriver& driver, const TString& prefix, int argc, char** a } executor.Wait(); - Cout << "Generated " << generated << " new series" << Endl; + std::cout << "Generated " << generated << " new series" << std::endl; return 0; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_list.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_list.cpp index d06194a38b68..59e127d8ba7c 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_list.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_list.cpp @@ -5,10 +5,11 @@ using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// -static void ParseSeries(TVector& results, TResultSetParser&& parser) { +static void ParseSeries(std::vector& results, TResultSetParser&& parser) { results.clear(); while (parser.TryNextRow()) { auto& series = results.emplace_back(); @@ -21,16 +22,16 @@ static void ParseSeries(TVector& results, TResultSetParser&& parser) { } static TStatus ListByViews( - TVector& results, + std::vector& results, TSession& session, - const TString& prefix, - ui64 limit, - ui64 lastSeriesId, - ui64 lastViews) + const std::string& prefix, + uint64_t limit, + uint64_t lastSeriesId, + uint64_t lastViews) { - auto queryText = Sprintf(R"( + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $limit AS Uint64; DECLARE $lastSeriesId AS Uint64; @@ -66,7 +67,7 @@ static TStatus ListByViews( FROM $filter AS t1 INNER JOIN series AS t2 USING (series_id) ORDER BY views DESC, series_id ASC; - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -99,14 +100,14 @@ static TStatus ListByViews( } static TStatus ListByViews( - TVector& results, + std::vector& results, TSession& session, - const TString& prefix, - ui64 limit) + const std::string& prefix, + uint64_t limit) { - auto queryText = Sprintf(R"( + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $limit AS Uint64; @@ -121,7 +122,7 @@ static TStatus ListByViews( FROM $filter AS t1 INNER JOIN series AS t2 USING (series_id) ORDER BY views DESC, series_id ASC; - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -148,15 +149,15 @@ static TStatus ListByViews( } static TStatus ListById( - TVector& results, + std::vector& results, TSession& session, - const TString& prefix, - ui64 limit, - ui64 lastSeriesId) + const std::string& prefix, + uint64_t limit, + uint64_t lastSeriesId) { - auto queryText = Sprintf(R"( + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $limit AS Uint64; DECLARE $lastSeriesId AS Uint64; @@ -166,7 +167,7 @@ static TStatus ListById( WHERE series_id > $lastSeriesId ORDER BY series_id LIMIT $limit; - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -196,14 +197,14 @@ static TStatus ListById( } static TStatus ListById( - TVector& results, + std::vector& results, TSession& session, - const TString& prefix, - ui64 limit) + const std::string& prefix, + uint64_t limit) { - auto queryText = Sprintf(R"( + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $limit AS Uint64; @@ -211,7 +212,7 @@ static TStatus ListById( FROM series ORDER BY series_id LIMIT $limit; - )", prefix.data()); + )", prefix); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -237,13 +238,13 @@ static TStatus ListById( return result; } -int RunListSeries(TDriver& driver, const TString& prefix, int argc, char** argv) { +int RunListSeries(TDriver& driver, const std::string& prefix, int argc, char** argv) { TOpts opts = TOpts::Default(); bool byViews = false; - ui64 limit = 10; - ui64 lastSeriesId = -1; - ui64 lastViews = -1; + uint64_t limit = 10; + uint64_t lastSeriesId = -1; + uint64_t lastViews = -1; opts.AddLongOption("by-views", "Sort by views").NoArgument().SetFlag(&byViews); opts.AddLongOption("limit", "Maximum number of rows").Optional().RequiredArgument("NUM") @@ -255,7 +256,7 @@ int RunListSeries(TDriver& driver, const TString& prefix, int argc, char** argv) TOptsParseResult res(&opts, argc, argv); - TVector results; + std::vector results; TTableClient client(driver); ThrowOnError(client.RetryOperationSync([&](TSession session) -> TStatus { if (byViews) { @@ -274,7 +275,7 @@ int RunListSeries(TDriver& driver, const TString& prefix, int argc, char** argv) })); size_t rows = results.size() + 1; - TVector columns[5]; + std::vector columns[5]; for (size_t i = 0; i < 5; ++i) { columns[i].reserve(rows); } @@ -297,27 +298,27 @@ int RunListSeries(TDriver& driver, const TString& prefix, int argc, char** argv) } } auto printLine = [&]() { - Cout << '+'; + std::cout << '+'; for (size_t i = 0; i < 5; ++i) { for (size_t k = 0; k < widths[i]; ++k) { - Cout << '-'; + std::cout << '-'; } - Cout << '+'; + std::cout << '+'; } - Cout << Endl; + std::cout << std::endl; }; auto printRow = [&](size_t row) { - Cout << '|'; + std::cout << '|'; for (size_t i = 0; i < 5; ++i) { - Cout << ' ' << columns[i][row]; + std::cout << ' ' << columns[i][row]; size_t printed = 1 + GetNumberOfUTF8Chars(columns[i][row]); while (printed < widths[i]) { - Cout << ' '; + std::cout << ' '; ++printed; } - Cout << '|'; + std::cout << '|'; } - Cout << Endl; + std::cout << std::endl; }; printLine(); diff --git a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_update.cpp b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_update.cpp index 14641e79cc18..f040b192a0fd 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_update.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index/secondary_index_update.cpp @@ -3,14 +3,15 @@ using namespace NLastGetopt; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; //////////////////////////////////////////////////////////////////////////////// -int RunUpdateViews(TDriver& driver, const TString& prefix, int argc, char** argv) { +int RunUpdateViews(TDriver& driver, const std::string& prefix, int argc, char** argv) { TOpts opts = TOpts::Default(); - ui64 seriesId; - ui64 newViews; + uint64_t seriesId; + uint64_t newViews; opts.AddLongOption("id", "Series id").Required().RequiredArgument("NUM") .StoreResult(&seriesId); @@ -19,9 +20,9 @@ int RunUpdateViews(TDriver& driver, const TString& prefix, int argc, char** argv TOptsParseResult res(&opts, argc, argv); - TString queryText = Sprintf(R"( + std::string queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%1$s"); + PRAGMA TablePathPrefix("{}"); DECLARE $seriesId AS Uint64; DECLARE $newViews AS Uint64; @@ -46,9 +47,9 @@ int RunUpdateViews(TDriver& driver, const TString& prefix, int argc, char** argv SELECT $newRevViews AS rev_views, series_id FROM $data; SELECT COUNT(*) AS cnt FROM $data; - )", prefix.data()); + )", prefix); - ui64 updatedCount = 0; + uint64_t updatedCount = 0; TTableClient client(driver); ThrowOnError(client.RetryOperationSync([&](TSession session) -> TStatus { @@ -82,6 +83,6 @@ int RunUpdateViews(TDriver& driver, const TString& prefix, int argc, char** argv return result; })); - Cout << "Updated " << updatedCount << " rows" << Endl; + std::cout << "Updated " << updatedCount << " rows" << std::endl; return 0; } diff --git a/ydb/public/sdk/cpp/examples/secondary_index/ya.make b/ydb/public/sdk/cpp/examples/secondary_index/ya.make index 7fdce06496a5..5cb7fc338838 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index/ya.make +++ b/ydb/public/sdk/cpp/examples/secondary_index/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp secondary_index.cpp @@ -13,7 +15,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/CMakeLists.txt b/ydb/public/sdk/cpp/examples/secondary_index_builtin/CMakeLists.txt new file mode 100644 index 000000000000..b46cc79159c0 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/CMakeLists.txt @@ -0,0 +1,40 @@ +add_executable(secondary_index_builtin) + +target_link_libraries(secondary_index_builtin PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(secondary_index_builtin PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index_create.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index_fill.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index_select.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index_drop.cpp + ${YDB_SDK_SOURCE_DIR}/examples/secondary_index_builtin/secondary_index_select_join.cpp +) + +vcs_info(secondary_index_builtin) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(secondary_index_builtin PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(secondary_index_builtin PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(secondary_index_builtin PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/main.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/main.cpp index 32836a78f1a9..331ad7ed531f 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/main.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/main.cpp @@ -1,15 +1,14 @@ #include "secondary_index.h" -#include - using namespace NLastGetopt; using namespace NYdb; +using namespace NYdb::NStatusHelpers; int main(int argc, char** argv) { - TString endpoint; - TString database; - TString command; + std::string endpoint; + std::string database; + std::string command; TOpts opts = TOpts::Default(); @@ -30,14 +29,14 @@ int main(int argc, char** argv) { TCommand cmd = Parse(command.c_str()); if (cmd == TCommand::NONE) { - Cerr << "Unsupported command: " << command << Endl; + std::cerr << "Unsupported command: " << command << std::endl; return 1; } auto config = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); TDriver driver(config); @@ -58,7 +57,7 @@ int main(int argc, char** argv) { } } catch (const TYdbErrorException& e) { - Cerr << "Execution failed: " << e << Endl; + std::cerr << "Execution failed: " << e.what() << std::endl; return 1; } } diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.cpp index fd8214771281..54edb10ddf14 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.cpp @@ -1,6 +1,6 @@ #include "secondary_index.h" -#include +#include TCommand Parse(const char * stringCmd) { @@ -21,19 +21,19 @@ TCommand Parse(const char * stringCmd) { return TCommand::NONE; } -TString JoinPath(const TString& prefix, const TString& path) { +std::string JoinPath(const std::string& prefix, const std::string& path) { if (prefix.empty()) { return path; } - TPathSplitUnix prefixPathSplit(prefix); - prefixPathSplit.AppendComponent(path); + std::filesystem::path prefixPathSplit(prefix); + prefixPathSplit /= path; - return prefixPathSplit.Reconstruct(); + return prefixPathSplit; } -void ParseSelectSeries(TVector& parseResult, TResultSetParser&& parser) { +void ParseSelectSeries(std::vector& parseResult, TResultSetParser&& parser) { parseResult.clear(); while (parser.TryNextRow()) { auto& series = parseResult.emplace_back(); diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.h b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.h index de782b823efa..ed22b2a28d20 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.h +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index.h @@ -1,14 +1,14 @@ #pragma once -#include -#include +#include +#include -#include -#include #include #include -#include -#include + +#include + +#include #define TABLE_USERS "users" #define TABLE_SERIES "series" @@ -25,25 +25,25 @@ enum class TCommand { }; struct TUser { - ui64 UserId; - TString Name; - ui32 Age; - TUser(ui64 userId = 0, TString name = "", ui32 age = 0) + uint64_t UserId; + std::string Name; + uint32_t Age; + TUser(uint64_t userId = 0, std::string name = "", uint32_t age = 0) : UserId(userId) , Name(name) , Age(age) {} }; struct TSeries { - ui64 SeriesId; - TString Title; + uint64_t SeriesId; + std::string Title; TInstant ReleaseDate; - TString Info; - ui64 Views; - ui64 UploadedUserId; + std::string Info; + uint64_t Views; + uint64_t UploadedUserId; - TSeries(ui64 seriesId = 0, TString title = "", TInstant releaseDate = TInstant::Days(0), - TString info = "", ui64 views = 0, ui64 uploadedUserId = 0) + TSeries(uint64_t seriesId = 0, std::string title = "", TInstant releaseDate = TInstant::Days(0), + std::string info = "", uint64_t views = 0, uint64_t uploadedUserId = 0) : SeriesId(seriesId) , Title(title) , ReleaseDate(releaseDate) @@ -52,38 +52,14 @@ struct TSeries { , UploadedUserId(uploadedUserId) {} }; -class TYdbErrorException: public yexception { -public: - TYdbErrorException(NYdb::TStatus status) - : Status(std::move(status)) - { } - - friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { - out << "Status:" << e.Status.GetStatus(); - if (e.Status.GetIssues()) { - out << Endl; - e.Status.GetIssues().PrintTo(out); - } - return out; - } -private: - NYdb::TStatus Status; -}; - -inline void ThrowOnError(NYdb::TStatus status) { - if (!status.IsSuccess()){ - throw TYdbErrorException(status) << status; - } -} - -TString GetCommandsList(); +std::string GetCommandsList(); TCommand Parse(const char *stringCmnd); -TString JoinPath(const TString& prefix, const TString& path); +std::string JoinPath(const std::string& prefix, const std::string& path); -void ParseSelectSeries(TVector& parseResult, TResultSetParser&& parser); +void ParseSelectSeries(std::vector& parseResult, TResultSetParser&& parser); -int Create(NYdb::TDriver& driver, const TString& path); -int Insert(NYdb::TDriver& driver, const TString& path); -int Drop(NYdb::TDriver& driver, const TString& path); -int SelectJoin(NYdb::TDriver& driver, const TString& path, int argc, char **argv); -int Select(NYdb::TDriver& driver, const TString& path, int argc, char **argv); +int Create(NYdb::TDriver& driver, const std::string& path); +int Insert(NYdb::TDriver& driver, const std::string& path); +int Drop(NYdb::TDriver& driver, const std::string& path); +int SelectJoin(NYdb::TDriver& driver, const std::string& path, int argc, char **argv); +int Select(NYdb::TDriver& driver, const std::string& path, int argc, char **argv); diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_create.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_create.cpp index 64a1755042d5..5b794b6dbdb1 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_create.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_create.cpp @@ -2,9 +2,10 @@ using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; using namespace NLastGetopt; -static void CreateSeriesTable(TTableClient& client, const TString& path) { +static void CreateSeriesTable(TTableClient& client, const std::string& path) { ThrowOnError(client.RetryOperationSync([path](TSession session) { @@ -37,7 +38,7 @@ static void CreateSeriesTable(TTableClient& client, const TString& path) { })); } -int Create(TDriver& driver, const TString& path) { +int Create(TDriver& driver, const std::string& path) { TTableClient client(driver); CreateSeriesTable(client, path); return 0; diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_drop.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_drop.cpp index cc0b87fe6283..06d5be52b157 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_drop.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_drop.cpp @@ -1,15 +1,16 @@ #include "secondary_index.h" using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; using namespace NYdb; -static void DropTable(TTableClient& client, const TString& path) { +static void DropTable(TTableClient& client, const std::string& path) { ThrowOnError(client.RetryOperationSync([path] (TSession session) { return session.DropTable(path).ExtractValueSync(); })); } -int Drop(NYdb::TDriver& driver, const TString& path) { +int Drop(NYdb::TDriver& driver, const std::string& path) { TTableClient client(driver); DropTable(client, JoinPath(path, TABLE_SERIES)); diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_fill.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_fill.cpp index d0f6ce2d12f4..6747e2f06f06 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_fill.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_fill.cpp @@ -1,13 +1,13 @@ #include "secondary_index.h" #include -#include using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; -TVector GetSeries() { - TVector series = { +std::vector GetSeries() { + std::vector series = { TSeries(1, "First episode", TInstant::ParseIso8601("2006-01-01"), "Pilot episode.", 1000, 0), TSeries(2, "Second episode", TInstant::ParseIso8601("2006-02-01"), "Jon Snow knows nothing.", 2000, 1), TSeries(3, "Third episode", TInstant::ParseIso8601("2006-03-01"), "Daenerys is the mother of dragons.", 3000, 2), @@ -22,8 +22,8 @@ TVector GetSeries() { return series; } -TVector GetUsers() { - TVector users = { +std::vector GetUsers() { + std::vector users = { TUser(0, "Kit Harrington", 32), TUser(1, "Emilia Clarke", 32), TUser(2, "Jason Momoa", 39), @@ -32,7 +32,7 @@ TVector GetUsers() { return users; } -TParams Build(const TVector& seriesList, const TVector& usersList) { +TParams Build(const std::vector& seriesList, const std::vector& usersList) { TParamsBuilder paramsBuilder; @@ -81,11 +81,11 @@ TParams Build(const TVector& seriesList, const TVector& usersLis return paramsBuilder.Build(); } -static TStatus FillTable(TSession session, const TString& path) { +static TStatus FillTable(TSession session, const std::string& path) { - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $seriesData AS List - using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; using namespace NLastGetopt; -TStatus SelectSeriesWithViews(TSession session, const TString& path, TVector& selectResult, ui64 minViews) { - auto queryText = Sprintf(R"( +TStatus SelectSeriesWithViews(TSession session, const std::string& path, std::vector& selectResult, uint64_t minViews) { + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $minViews AS Uint64; SELECT series_id, title, info, release_date, views, uploaded_user_id FROM `series` VIEW views_index WHERE views >= $minViews - )", path.c_str()); + )", path); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -40,25 +39,25 @@ TStatus SelectSeriesWithViews(TSession session, const TString& path, TVector selectResult; + std::vector selectResult; ThrowOnError(client.RetryOperationSync([path, minViews, &selectResult](TSession session) { return SelectSeriesWithViews(session, path, selectResult, minViews); })); for (auto& item: selectResult) { - Cout << item.SeriesId << ' ' << item.Title << ' ' << item.Info << ' ' - << item.ReleaseDate << ' ' << item.Views << ' ' << item.UploadedUserId << Endl; + std::cout << item.SeriesId << ' ' << item.Title << ' ' << item.Info << ' ' + << item.ReleaseDate.ToString() << ' ' << item.Views << ' ' << item.UploadedUserId << std::endl; } return 0; diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_select_join.cpp b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_select_join.cpp index d5f6c8bd2db3..d2874e121dee 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_select_join.cpp +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/secondary_index_select_join.cpp @@ -1,17 +1,16 @@ #include "secondary_index.h" -#include - using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; using namespace NLastGetopt; -TStatus SelectSeriesWithUserName(TSession session, const TString& path, - TVector& selectResult, const TString& name) { +TStatus SelectSeriesWithUserName(TSession session, const std::string& path, + std::vector& selectResult, const std::string& name) { - auto queryText = Sprintf(R"( + auto queryText = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $userName AS Utf8; @@ -20,7 +19,7 @@ TStatus SelectSeriesWithUserName(TSession session, const TString& path, INNER JOIN `users` VIEW name_index AS t2 ON t1.uploaded_user_id == t2.user_id WHERE t2.name == $userName; - )", path.c_str()); + )", path); auto prepareResult = session.PrepareDataQuery(queryText).ExtractValueSync(); if (!prepareResult.IsSuccess()) { @@ -44,25 +43,25 @@ TStatus SelectSeriesWithUserName(TSession session, const TString& path, return result; } -int SelectJoin(TDriver& driver, const TString& path, int argc, char **argv) { +int SelectJoin(TDriver& driver, const std::string& path, int argc, char **argv) { TOpts opts = TOpts::Default(); - TString name; + std::string name; opts.AddLongOption("name", "User name").Required().RequiredArgument("TYPE") .StoreResult(&name); TOptsParseResult res(&opts, argc, argv); TTableClient client(driver); - TVector selectResult; + std::vector selectResult; ThrowOnError(client.RetryOperationSync([path, &selectResult, name](TSession session) { return SelectSeriesWithUserName(session, path, selectResult, name); })); for (auto& item : selectResult) { - Cout << item.SeriesId << ' ' << item.Title << Endl; + std::cout << item.SeriesId << ' ' << item.Title << std::endl; } return 0; diff --git a/ydb/public/sdk/cpp/examples/secondary_index_builtin/ya.make b/ydb/public/sdk/cpp/examples/secondary_index_builtin/ya.make index ed9099e80611..f16f03c1b2bc 100644 --- a/ydb/public/sdk/cpp/examples/secondary_index_builtin/ya.make +++ b/ydb/public/sdk/cpp/examples/secondary_index_builtin/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp secondary_index.cpp @@ -12,7 +14,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/examples/topic_reader/CMakeLists.txt b/ydb/public/sdk/cpp/examples/topic_reader/CMakeLists.txt new file mode 100644 index 000000000000..ce07f0ebbfe0 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/topic_reader/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(eventloop) +add_subdirectory(simple) +add_subdirectory(transaction) diff --git a/ydb/public/sdk/cpp/examples/topic_reader/eventloop/CMakeLists.txt b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/CMakeLists.txt new file mode 100644 index 000000000000..2cdc984955fa --- /dev/null +++ b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/CMakeLists.txt @@ -0,0 +1,34 @@ +add_executable(persqueue_reader_eventloop) + +target_link_libraries(persqueue_reader_eventloop PUBLIC + yutil + YDB-CPP-SDK::Topic + getopt +) + +target_sources(persqueue_reader_eventloop PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/topic_reader/eventloop/main.cpp +) + +vcs_info(persqueue_reader_eventloop) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(persqueue_reader_eventloop PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(persqueue_reader_eventloop PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(persqueue_reader_eventloop PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/topic_reader/eventloop/main.cpp b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/main.cpp index d2829d2d888a..277faea66610 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/eventloop/main.cpp +++ b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/main.cpp @@ -1,15 +1,12 @@ -#include +#include #include -#include -#include - struct TOptions { - TString Endpoint; - TString Database; - TString TopicPath; - TString ConsumerName; + std::string Endpoint; + std::string Database; + std::string TopicPath; + std::string ConsumerName; bool CommitAfterProcessing = false; bool DisableClusterDiscovery = false; bool UseSecureConnection = false; @@ -19,7 +16,7 @@ struct TOptions { opts.AddHelpOption('h'); opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&Endpoint); - opts.AddLongOption('d', "database", "YDB database name").DefaultValue("/Root").RequiredArgument("PATH") + opts.AddLongOption('d', "database", "YDB database name").DefaultValue("/local").RequiredArgument("PATH") .StoreResult(&Database); opts.AddLongOption('t', "topic-path", "Topic path for reading").Required().RequiredArgument("PATH") .StoreResult(&TopicPath); @@ -38,7 +35,7 @@ struct TOptions { std::shared_ptr ReadSession; void StopHandler(int) { - Cerr << "Stopping session" << Endl; + std::cerr << "Stopping session" << std::endl; if (ReadSession) { ReadSession->Close(TDuration::Seconds(3)); } else { @@ -57,8 +54,8 @@ int main(int argc, const char* argv[]) { .SetNetworkThreadsNum(2) .SetEndpoint(opts.Endpoint) .SetDatabase(opts.Database) - .SetAuthToken(GetEnv("YDB_TOKEN")) - .SetLog(CreateLogBackend("cerr")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : "") + .SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); if (opts.UseSecureConnection) { driverConfig.UseSecureConnection(); @@ -77,7 +74,7 @@ int main(int argc, const char* argv[]) { ReadSession = topicClient.CreateReadSession(settings); - Cerr << "Session was created" << Endl; + std::cerr << "Session was created" << std::endl; // [BEGIN read session process events] // Event loop @@ -86,15 +83,15 @@ int main(int argc, const char* argv[]) { // Wait for next event or ten seconds future.Wait(TDuration::Seconds(10)); // future.Subscribe([](){ - // Cerr << ...; + // std::cerr << ...; // }); // Get event - TMaybe event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); - Cerr << "Got new read session event: " << DebugString(*event) << Endl; + std::optional event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); + std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; if (auto* dataEvent = std::get_if(&*event)) { for (const auto& message : dataEvent->GetMessages()) { - Cerr << "Data message: \"" << message.GetData() << "\"" << Endl; + std::cerr << "Data message: \"" << message.GetData() << "\"" << std::endl; } if (opts.CommitAfterProcessing) { diff --git a/ydb/public/sdk/cpp/examples/topic_reader/eventloop/ya.make b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/ya.make index 79eea9e98953..8f7726227586 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/eventloop/ya.make +++ b/ydb/public/sdk/cpp/examples/topic_reader/eventloop/ya.make @@ -1,11 +1,13 @@ PROGRAM(persqueue_reader_eventloop) +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/getopt ) diff --git a/ydb/public/sdk/cpp/examples/topic_reader/simple/CMakeLists.txt b/ydb/public/sdk/cpp/examples/topic_reader/simple/CMakeLists.txt new file mode 100644 index 000000000000..68846ab215c5 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/topic_reader/simple/CMakeLists.txt @@ -0,0 +1,34 @@ +add_executable(simple_persqueue_reader) + +target_link_libraries(simple_persqueue_reader PUBLIC + yutil + YDB-CPP-SDK::Topic + getopt +) + +target_sources(simple_persqueue_reader PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/topic_reader/simple/main.cpp +) + +vcs_info(simple_persqueue_reader) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(simple_persqueue_reader PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(simple_persqueue_reader PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(simple_persqueue_reader PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/topic_reader/simple/main.cpp b/ydb/public/sdk/cpp/examples/topic_reader/simple/main.cpp index d0fc7c8527b5..5dfe45474462 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/simple/main.cpp +++ b/ydb/public/sdk/cpp/examples/topic_reader/simple/main.cpp @@ -1,15 +1,12 @@ -#include +#include #include -#include -#include - struct TOptions { - TString Endpoint; - TString Database; - TString TopicPath; - TString ConsumerName; + std::string Endpoint; + std::string Database; + std::string TopicPath; + std::string ConsumerName; bool CommitAfterProcessing = false; bool DisableClusterDiscovery = false; bool UseSecureConnection = false; @@ -19,7 +16,7 @@ struct TOptions { opts.AddHelpOption('h'); opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&Endpoint); - opts.AddLongOption('d', "database", "YDB database name").DefaultValue("/Root").RequiredArgument("PATH") + opts.AddLongOption('d', "database", "YDB database name").DefaultValue("/local").RequiredArgument("PATH") .StoreResult(&Database); opts.AddLongOption('t', "topic-path", "Topic path for reading").Required().RequiredArgument("PATH") .StoreResult(&TopicPath); @@ -38,7 +35,7 @@ struct TOptions { std::shared_ptr ReadSession; void StopHandler(int) { - Cerr << "Stopping session" << Endl; + std::cerr << "Stopping session" << std::endl; if (ReadSession) { ReadSession->Close(TDuration::Seconds(3)); } else { @@ -57,8 +54,8 @@ int main(int argc, const char* argv[]) { .SetNetworkThreadsNum(2) .SetEndpoint(opts.Endpoint) .SetDatabase(opts.Database) - .SetAuthToken(GetEnv("YDB_TOKEN")) - .SetLog(CreateLogBackend("cerr")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : "") + .SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); if (opts.UseSecureConnection) { driverConfig.UseSecureConnection(); @@ -83,13 +80,13 @@ int main(int argc, const char* argv[]) { // settings.SetSimpleDataHandlers( settings.EventHandlers_.SimpleDataHandlers( [](NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent& event) { - Cerr << "Data event " << DebugString(event); + std::cerr << "Data event " << DebugString(event); }, opts.CommitAfterProcessing); ReadSession = topicClient.CreateReadSession(settings); // [END Create read session] - Cerr << "Session was created" << Endl; + std::cerr << "Session was created" << std::endl; // Wait SessionClosed event. ReadSession->GetEvent(/*block = */true); diff --git a/ydb/public/sdk/cpp/examples/topic_reader/simple/ya.make b/ydb/public/sdk/cpp/examples/topic_reader/simple/ya.make index 71006ad87161..092d05639141 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/simple/ya.make +++ b/ydb/public/sdk/cpp/examples/topic_reader/simple/ya.make @@ -1,11 +1,13 @@ PROGRAM(simple_persqueue_reader) +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/getopt ) diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/CMakeLists.txt b/ydb/public/sdk/cpp/examples/topic_reader/transaction/CMakeLists.txt new file mode 100644 index 000000000000..64d30b4d8c6a --- /dev/null +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/CMakeLists.txt @@ -0,0 +1,36 @@ +add_executable(read_from_topic_in_transaction) + +target_link_libraries(read_from_topic_in_transaction PUBLIC + yutil + YDB-CPP-SDK::Topic + getopt +) + +target_sources(read_from_topic_in_transaction PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/topic_reader/transaction/application.cpp + ${YDB_SDK_SOURCE_DIR}/examples/topic_reader/transaction/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/topic_reader/transaction/options.cpp +) + +vcs_info(read_from_topic_in_transaction) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(read_from_topic_in_transaction PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(read_from_topic_in_transaction PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(read_from_topic_in_transaction PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.cpp b/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.cpp index 9d11db9f44d5..d30891d3922c 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.cpp +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.cpp @@ -1,7 +1,6 @@ #include "application.h" -#include -TApplication::TRow::TRow(ui64 key, const TString& value) : +TApplication::TRow::TRow(uint64_t key, const std::string& value) : Key(key), Value(value) { @@ -13,8 +12,8 @@ TApplication::TApplication(const TOptions& options) .SetNetworkThreadsNum(2) .SetEndpoint(options.Endpoint) .SetDatabase(options.Database) - .SetAuthToken(GetEnv("YDB_TOKEN")) - .SetLog(CreateLogBackend("cerr", Min(options.LogPriority, TLOG_RESOURCES))); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : "") + .SetLog(std::unique_ptr(CreateLogBackend("cerr", std::min(options.LogPriority, TLOG_RESOURCES)).Release())); if (options.UseSecureConnection) { config.UseSecureConnection(); } @@ -38,7 +37,7 @@ void TApplication::CreateTopicReadSession(const TOptions& options) ReadSession = TopicClient->CreateReadSession(settings); - Cout << "Topic session was created" << Endl; + std::cout << "Topic session was created" << std::endl; } void TApplication::CreateTableSession() @@ -49,7 +48,7 @@ void TApplication::CreateTableSession() TableSession = result.GetSession(); - Cout << "Table session was created" << Endl; + std::cout << "Table session was created" << std::endl; } void TApplication::Run() @@ -121,7 +120,7 @@ void TApplication::CommitTransaction() auto result = Transaction->Commit(settings).GetValueSync(); - Cout << "Commit: " << static_cast(result) << Endl; + std::cout << "Commit: " << ToString(static_cast(result)) << std::endl; } void TApplication::TryCommitTransaction() @@ -149,7 +148,7 @@ void TApplication::InsertRowsIntoTable() { Y_ABORT_UNLESS(Transaction); - TString query = " \ + std::string query = " \ DECLARE $rows AS List(), message.GetData()); + Rows.emplace_back(Dist(MersenneEngine), message.GetData()); } diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.h b/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.h index 2055719495a4..9b81327c723d 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.h +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/application.h @@ -2,12 +2,13 @@ #include "options.h" -#include -#include -#include +#include +#include +#include #include #include +#include class TApplication { public: @@ -20,10 +21,10 @@ class TApplication { private: struct TRow { TRow() = default; - TRow(ui64 key, const TString& value); + TRow(uint64_t key, const std::string& value); - ui64 Key = 0; - TString Value; + uint64_t Key = 0; + std::string Value; }; void CreateTopicReadSession(const TOptions& options); @@ -45,5 +46,8 @@ class TApplication { std::optional Transaction; std::vector PendingStopEvents; std::vector Rows; - TString TablePath; + std::string TablePath; + + std::mt19937_64 MersenneEngine; + std::uniform_int_distribution Dist; }; diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/main.cpp b/ydb/public/sdk/cpp/examples/topic_reader/transaction/main.cpp index 3e6748f19180..f996a8dd314f 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/transaction/main.cpp +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/main.cpp @@ -1,9 +1,7 @@ #include "application.h" #include "options.h" -#include - -#include +#include #include @@ -11,7 +9,7 @@ std::optional App; void StopHandler(int) { - Cout << "Stopping session" << Endl; + std::cout << "Stopping session" << std::endl; if (App) { App->Stop(); } else { @@ -27,10 +25,10 @@ int main(int argc, const char* argv[]) TOptions options(argc, argv); App.emplace(options); - Cout << "Application initialized" << Endl; + std::cout << "Application initialized" << std::endl; App->Run(); - Cout << "Event loop completed" << Endl; + std::cout << "Event loop completed" << std::endl; App->Finalize(); } diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/options.h b/ydb/public/sdk/cpp/examples/topic_reader/transaction/options.h index b40330e2dcd3..ec1901c6d6cc 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/transaction/options.h +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/options.h @@ -1,15 +1,16 @@ #pragma once -#include #include +#include + struct TOptions { - TString Endpoint; - TString Database; - TString TopicPath; - TString ConsumerName; + std::string Endpoint; + std::string Database; + std::string TopicPath; + std::string ConsumerName; bool UseSecureConnection = false; - TString TablePath; + std::string TablePath; ELogPriority LogPriority = TLOG_WARNING; TOptions(int argc, const char* argv[]); diff --git a/ydb/public/sdk/cpp/examples/topic_reader/transaction/ya.make b/ydb/public/sdk/cpp/examples/topic_reader/transaction/ya.make index 9d154e8a9b3f..86cbb04ce3a6 100644 --- a/ydb/public/sdk/cpp/examples/topic_reader/transaction/ya.make +++ b/ydb/public/sdk/cpp/examples/topic_reader/transaction/ya.make @@ -1,5 +1,7 @@ PROGRAM(read_from_topic_in_transaction) +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( application.cpp main.cpp @@ -7,7 +9,7 @@ SRCS( ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic library/cpp/getopt ) diff --git a/ydb/public/sdk/cpp/examples/topic_writer/transaction/main.cpp b/ydb/public/sdk/cpp/examples/topic_writer/transaction/main.cpp index 77030e2e2fd7..c98c8df8e1c9 100644 --- a/ydb/public/sdk/cpp/examples/topic_writer/transaction/main.cpp +++ b/ydb/public/sdk/cpp/examples/topic_writer/transaction/main.cpp @@ -1,20 +1,11 @@ -#include -#include - -void ThrowOnError(const NYdb::TStatus& status) -{ - if (status.IsSuccess()) { - return; - } - - ythrow yexception() << status; -} +#include +#include int main() { - const TString ENDPOINT = "HOST:PORT"; - const TString DATABASE = "DATABASE"; - const TString TOPIC = "PATH/TO/TOPIC"; + const std::string ENDPOINT = "HOST:PORT"; + const std::string DATABASE = "DATABASE"; + const std::string TOPIC = "PATH/TO/TOPIC"; NYdb::TDriverConfig config; config.SetEndpoint(ENDPOINT); diff --git a/ydb/public/sdk/cpp/examples/ttl/CMakeLists.txt b/ydb/public/sdk/cpp/examples/ttl/CMakeLists.txt new file mode 100644 index 000000000000..48a004b4cc64 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/ttl/CMakeLists.txt @@ -0,0 +1,35 @@ +add_executable(ttl) + +target_link_libraries(ttl PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(ttl PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/ttl/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/ttl/ttl.cpp +) + +vcs_info(ttl) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(ttl PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(ttl PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(ttl PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/ttl/main.cpp b/ydb/public/sdk/cpp/examples/ttl/main.cpp index 718381b53a41..031a06d737a2 100644 --- a/ydb/public/sdk/cpp/examples/ttl/main.cpp +++ b/ydb/public/sdk/cpp/examples/ttl/main.cpp @@ -1,10 +1,10 @@ #include "ttl.h" #include -#include using namespace NLastGetopt; using namespace NYdb; +using namespace NYdb::NStatusHelpers; void StopHandler(int) { exit(1); @@ -13,9 +13,9 @@ void StopHandler(int) { int main(int argc, char** argv) { TOpts opts = TOpts::Default(); - TString endpoint; - TString database; - TString path; + std::string endpoint; + std::string database; + std::string path; opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT") .StoreResult(&endpoint); opts.AddLongOption('d', "database", "YDB database name").Required().RequiredArgument("PATH") @@ -35,7 +35,7 @@ int main(int argc, char** argv) { auto driverConfig = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); TDriver driver(driverConfig); if (!Run(driver, path)) { diff --git a/ydb/public/sdk/cpp/examples/ttl/ttl.cpp b/ydb/public/sdk/cpp/examples/ttl/ttl.cpp index 15a68912c214..9ec2eafd5bd6 100644 --- a/ydb/public/sdk/cpp/examples/ttl/ttl.cpp +++ b/ydb/public/sdk/cpp/examples/ttl/ttl.cpp @@ -1,18 +1,36 @@ #include "ttl.h" #include "util.h" -#include -#include +#include using namespace NExample; using namespace NYdb; using namespace NYdb::NTable; +using namespace NYdb::NStatusHelpers; -constexpr ui32 DOC_TABLE_PARTITION_COUNT = 4; -constexpr ui32 EXPIRATION_QUEUE_COUNT = 4; +constexpr uint32_t DOC_TABLE_PARTITION_COUNT = 4; +constexpr uint32_t EXPIRATION_QUEUE_COUNT = 4; + +namespace { + +template +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return std::to_string(opt.value()); + } + return "(NULL)"; +} + +template <> +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return opt.value(); + } + return "(NULL)"; +} //! Creates Documents table and multiple ExpirationQueue tables -static void CreateTables(TTableClient client, const TString& path) { +void CreateTables(TTableClient client, const std::string& path) { // Documents table stores the contents of web pages. // The table is partitioned by hash(Url) in order to evenly distribute the load. ThrowOnError(client.RetryOperationSync([path](TSession session) { @@ -34,7 +52,7 @@ static void CreateTables(TTableClient client, const TString& path) { // Multiple ExpirationQueue tables allow to scale the load. // Each ExpirationQueue table can be handled by a dedicated worker. - for (ui32 i = 0; i < EXPIRATION_QUEUE_COUNT; ++i) { + for (uint32_t i = 0; i < EXPIRATION_QUEUE_COUNT; ++i) { ThrowOnError(client.RetryOperationSync([path, i](TSession session) { auto expirationDesc = TTableBuilder() .AddNullableColumn("timestamp", EPrimitiveType::Uint64) @@ -42,7 +60,7 @@ static void CreateTables(TTableClient client, const TString& path) { .SetPrimaryKeyColumns({"timestamp", "doc_id"}) .Build(); - return session.CreateTable(JoinPath(path, Sprintf("expiration_queue_%" PRIu32, i)), + return session.CreateTable(JoinPath(path, std::format("expiration_queue_{}", i)), std::move(expirationDesc)).GetValueSync(); })); } @@ -51,15 +69,15 @@ static void CreateTables(TTableClient client, const TString& path) { /////////////////////////////////////////////////////////////////////////////// //! Insert or replaces a document. -static TStatus AddDocumentTransaction(TSession session, const TString& path, - const TString& url, const TString& html, ui64 timestamp) +TStatus AddDocumentTransaction(TSession session, const std::string& path, + const std::string& url, const std::string& html, uint64_t timestamp) { // Add an entry to a random expiration queue in order to evenly distribute the load - ui32 queue = rand() % EXPIRATION_QUEUE_COUNT; + uint32_t queue = rand() % EXPIRATION_QUEUE_COUNT; - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $url AS Utf8; DECLARE $html AS Utf8; @@ -72,11 +90,11 @@ static TStatus AddDocumentTransaction(TSession session, const TString& path, VALUES ($doc_id, $url, $html, $timestamp); - REPLACE INTO expiration_queue_%u + REPLACE INTO expiration_queue_{} (`timestamp`, doc_id) VALUES ($timestamp, $doc_id); - )", path.c_str(), queue); + )", path, queue); auto params = session.GetParamsBuilder() .AddParam("$url").Utf8(url).Build() @@ -91,12 +109,12 @@ static TStatus AddDocumentTransaction(TSession session, const TString& path, } //! Reads document contents. -static TStatus ReadDocumentTransaction(TSession session, const TString& path, - const TString& url, TMaybe& resultSet) +TStatus ReadDocumentTransaction(TSession session, const std::string& path, + const std::string& url, std::optional& resultSet) { - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $url AS Utf8; @@ -105,7 +123,7 @@ static TStatus ReadDocumentTransaction(TSession session, const TString& path, SELECT doc_id, url, html, `timestamp` FROM documents WHERE doc_id = $doc_id; - )", path.c_str()); + )", path); auto params = session.GetParamsBuilder() .AddParam("$url").Utf8(url).Build() @@ -124,42 +142,42 @@ static TStatus ReadDocumentTransaction(TSession session, const TString& path, } //! Reads a batch of entries from expiration queue -static TStatus ReadExpiredBatchTransaction(TSession session, const TString& path, const ui32 queue, - const ui64 timestamp, const ui64 prevTimestamp, const ui64 prevDocId, TMaybe& resultSet) +TStatus ReadExpiredBatchTransaction(TSession session, const std::string& path, const uint32_t queue, + const uint64_t timestamp, const uint64_t prevTimestamp, const uint64_t prevDocId, std::optional& resultSet) { - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{0}"); DECLARE $timestamp AS Uint64; DECLARE $prev_timestamp AS Uint64; DECLARE $prev_doc_id AS Uint64; $data = ( - SELECT * - FROM expiration_queue_%u + (SELECT * + FROM expiration_queue_{1} WHERE `timestamp` <= $timestamp AND `timestamp` > $prev_timestamp ORDER BY `timestamp`, doc_id - LIMIT 100 + LIMIT 100) UNION ALL - SELECT * - FROM expiration_queue_%u + (SELECT * + FROM expiration_queue_{1} WHERE `timestamp` = $prev_timestamp AND doc_id > $prev_doc_id ORDER BY `timestamp`, doc_id - LIMIT 100 + LIMIT 100) ); SELECT `timestamp`, doc_id FROM $data ORDER BY `timestamp`, doc_id LIMIT 100; - )", path.c_str(), queue, queue); + )", path, queue); auto params = session.GetParamsBuilder() .AddParam("$timestamp").Uint64(timestamp).Build() @@ -180,12 +198,12 @@ static TStatus ReadExpiredBatchTransaction(TSession session, const TString& path } //! Deletes an expired document -static TStatus DeleteDocumentWithTimestamp(TSession session, const TString& path, const ui32 queue, - const ui64 docId, const ui64 timestamp) +TStatus DeleteDocumentWithTimestamp(TSession session, const std::string& path, const uint32_t queue, + const uint64_t docId, const uint64_t timestamp) { - auto query = Sprintf(R"( + auto query = std::format(R"( --!syntax_v1 - PRAGMA TablePathPrefix("%s"); + PRAGMA TablePathPrefix("{}"); DECLARE $doc_id AS Uint64; DECLARE $timestamp AS Uint64; @@ -193,9 +211,9 @@ static TStatus DeleteDocumentWithTimestamp(TSession session, const TString& path DELETE FROM documents WHERE doc_id = $doc_id AND `timestamp` = $timestamp; - DELETE FROM expiration_queue_%u + DELETE FROM expiration_queue_{} WHERE `timestamp` = $timestamp AND doc_id = $doc_id; - )", path.c_str(), queue); + )", path, queue); auto params = session.GetParamsBuilder() .AddParam("$doc_id").Uint64(docId).Build() @@ -209,47 +227,50 @@ static TStatus DeleteDocumentWithTimestamp(TSession session, const TString& path return result; } + +} + /////////////////////////////////////////////////////////////////////////////// -void AddDocument(TTableClient client, const TString& path, const TString& url, - const TString& html, const ui64 timestamp) +void AddDocument(TTableClient client, const std::string& path, const std::string& url, + const std::string& html, const uint64_t timestamp) { - Cout << "> AddDocument:" << Endl - << " Url: " << url << Endl - << " Timestamp: " << timestamp << Endl; + std::cout << "> AddDocument:" << std::endl + << " Url: " << url << std::endl + << " Timestamp: " << timestamp << std::endl; ThrowOnError(client.RetryOperationSync([path, url, html, timestamp](TSession session) { return AddDocumentTransaction(session, path, url, html, timestamp); })); - Cout << Endl; + std::cout << std::endl; } -void ReadDocument(TTableClient client, const TString& path, const TString& url) { - Cout << "> ReadDocument \"" << url << "\":" << Endl; - TMaybe resultSet; +void ReadDocument(TTableClient client, const std::string& path, const std::string& url) { + std::cout << "> ReadDocument \"" << url << "\":" << std::endl; + std::optional resultSet; ThrowOnError(client.RetryOperationSync([path, url, &resultSet] (TSession session) { return ReadDocumentTransaction(session, path, url, resultSet); })); TResultSetParser parser(*resultSet); if (parser.TryNextRow()) { - Cout << " DocId: " << parser.ColumnParser("doc_id").GetOptionalUint64() << Endl - << " Url: " << parser.ColumnParser("url").GetOptionalUtf8() << Endl - << " Timestamp: " << parser.ColumnParser("timestamp").GetOptionalUint64() << Endl - << " Html: " << parser.ColumnParser("html").GetOptionalUtf8() << Endl; + std::cout << " DocId: " << OptionalToString(parser.ColumnParser("doc_id").GetOptionalUint64()) << std::endl + << " Url: " << OptionalToString(parser.ColumnParser("url").GetOptionalUtf8()) << std::endl + << " Timestamp: " << OptionalToString(parser.ColumnParser("timestamp").GetOptionalUint64()) << std::endl + << " Html: " << OptionalToString(parser.ColumnParser("html").GetOptionalUtf8()) << std::endl; } else { - Cout << " Not found" << Endl; + std::cout << " Not found" << std::endl; } - Cout << Endl; + std::cout << std::endl; } -void DeleteExpired(TTableClient client, const TString& path, const ui32 queue, const ui64 timestamp) { - Cout << "> DeleteExpired from queue #" << queue << ":" << Endl; +void DeleteExpired(TTableClient client, const std::string& path, const uint32_t queue, const uint64_t timestamp) { + std::cout << "> DeleteExpired from queue #" << queue << ":" << std::endl; bool empty = false; - ui64 lastTimestamp = 0; - ui64 lastDocId = 0; + uint64_t lastTimestamp = 0; + uint64_t lastDocId = 0; while (!empty) { - TMaybe resultSet; + std::optional resultSet; ThrowOnError(client.RetryOperationSync([path, queue, timestamp, lastDocId, lastTimestamp, &resultSet] (TSession session) { return ReadExpiredBatchTransaction(session, path, queue, timestamp, lastTimestamp, lastDocId, resultSet); })); @@ -260,18 +281,18 @@ void DeleteExpired(TTableClient client, const TString& path, const ui32 queue, c empty = false; lastDocId = *parser.ColumnParser("doc_id").GetOptionalUint64(); lastTimestamp = *parser.ColumnParser("timestamp").GetOptionalUint64(); - Cout << " DocId: " << lastDocId << " Timestamp: " << lastTimestamp << Endl; + std::cout << " DocId: " << lastDocId << " Timestamp: " << lastTimestamp << std::endl; ThrowOnError(client.RetryOperationSync([path, queue, lastDocId, lastTimestamp] (TSession session) { return DeleteDocumentWithTimestamp(session, path, queue, lastDocId, lastTimestamp); })); } } - Cout << Endl; + std::cout << std::endl; } /////////////////////////////////////////////////////////////////////////////// -bool Run(const TDriver& driver, const TString& path) { +bool Run(const TDriver& driver, const std::string& path) { TTableClient client(driver); try { @@ -290,7 +311,7 @@ bool Run(const TDriver& driver, const TString& path) { ReadDocument(client, path, "https://yandex.ru/"); ReadDocument(client, path, "https://ya.ru/"); - for (ui32 q = 0; q < EXPIRATION_QUEUE_COUNT; ++q) { + for (uint32_t q = 0; q < EXPIRATION_QUEUE_COUNT; ++q) { DeleteExpired(client, path, q, 1); } @@ -306,7 +327,7 @@ bool Run(const TDriver& driver, const TString& path) { "

Yandex

", 3); - for (ui32 q = 0; q < EXPIRATION_QUEUE_COUNT; ++q) { + for (uint32_t q = 0; q < EXPIRATION_QUEUE_COUNT; ++q) { DeleteExpired(client, path, q, 2); } @@ -314,8 +335,7 @@ bool Run(const TDriver& driver, const TString& path) { ReadDocument(client, path, "https://ya.ru/"); } catch (const TYdbErrorException& e) { - Cerr << "Execution failed due to fatal error:" << Endl; - PrintStatus(e.Status); + std::cerr << "Execution failed due to fatal error: " << e.what() << std::endl; return false; } diff --git a/ydb/public/sdk/cpp/examples/ttl/ttl.h b/ydb/public/sdk/cpp/examples/ttl/ttl.h index d419c216680a..99e0804f4bbe 100644 --- a/ydb/public/sdk/cpp/examples/ttl/ttl.h +++ b/ydb/public/sdk/cpp/examples/ttl/ttl.h @@ -1,8 +1,8 @@ #pragma once -#include -#include +#include +#include NYdb::TParams GetTablesDataParams(); -bool Run(const NYdb::TDriver& driver, const TString& path); +bool Run(const NYdb::TDriver& driver, const std::string& path); diff --git a/ydb/public/sdk/cpp/examples/ttl/util.h b/ydb/public/sdk/cpp/examples/ttl/util.h index b596df76fd32..a2a8bd3256a7 100644 --- a/ydb/public/sdk/cpp/examples/ttl/util.h +++ b/ydb/public/sdk/cpp/examples/ttl/util.h @@ -1,41 +1,26 @@ #pragma once -#include -#include +#include + +#include +#include + +#include namespace NExample { using namespace NYdb; using namespace NYdb::NTable; -class TYdbErrorException : public yexception { -public: - TYdbErrorException(const TStatus& status) - : Status(status) {} - - TStatus Status; -}; - -inline void ThrowOnError(const TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } -} - -inline void PrintStatus(const TStatus& status) { - Cerr << "Status: " << status.GetStatus() << Endl; - status.GetIssues().PrintTo(Cerr); -} - -inline TString JoinPath(const TString& basePath, const TString& path) { +inline std::string JoinPath(const std::string& basePath, const std::string& path) { if (basePath.empty()) { return path; } - TPathSplitUnix prefixPathSplit(basePath); - prefixPathSplit.AppendComponent(path); + std::filesystem::path prefixPathSplit(basePath); + prefixPathSplit /= path; - return prefixPathSplit.Reconstruct(); + return prefixPathSplit; } } diff --git a/ydb/public/sdk/cpp/examples/ttl/ya.make b/ydb/public/sdk/cpp/examples/ttl/ya.make index c030e09c278e..749d46cc7a55 100644 --- a/ydb/public/sdk/cpp/examples/ttl/ya.make +++ b/ydb/public/sdk/cpp/examples/ttl/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp ttl.h @@ -9,7 +11,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/examples/vector_index/CMakeLists.txt b/ydb/public/sdk/cpp/examples/vector_index/CMakeLists.txt new file mode 100644 index 000000000000..cd836a00b482 --- /dev/null +++ b/ydb/public/sdk/cpp/examples/vector_index/CMakeLists.txt @@ -0,0 +1,35 @@ +add_executable(vector_index) + +target_link_libraries(vector_index PUBLIC + yutil + getopt + YDB-CPP-SDK::Table +) + +target_sources(vector_index PRIVATE + ${YDB_SDK_SOURCE_DIR}/examples/vector_index/main.cpp + ${YDB_SDK_SOURCE_DIR}/examples/vector_index/vector_index.cpp +) + +vcs_info(vector_index) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(vector_index PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(vector_index PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(vector_index PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/ydb/public/sdk/cpp/examples/vector_index/main.cpp b/ydb/public/sdk/cpp/examples/vector_index/main.cpp index fe310f3cf541..0ba91a42342c 100644 --- a/ydb/public/sdk/cpp/examples/vector_index/main.cpp +++ b/ydb/public/sdk/cpp/examples/vector_index/main.cpp @@ -1,14 +1,11 @@ #include "vector_index.h" -#include -#include - using namespace NLastGetopt; using namespace NYdb; int main(int argc, char** argv) { - TString endpoint; - TString command; + std::string endpoint; + std::string command; TOptions options; TOpts opts = TOpts::Default(); @@ -33,14 +30,14 @@ int main(int argc, char** argv) { ECommand cmd = Parse(command); if (cmd == ECommand::None) { - Cerr << "Unsupported command: " << command << Endl; + std::cerr << "Unsupported command: " << command << std::endl; return 1; } auto config = TDriverConfig() .SetEndpoint(endpoint) .SetDatabase(options.Database) - .SetAuthToken(GetEnv("YDB_TOKEN")); + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); TDriver driver(config); @@ -66,7 +63,7 @@ int main(int argc, char** argv) { break; } } catch (const std::exception& e) { - Cerr << "Execution failed: " << e.what() << Endl; + std::cerr << "Execution failed: " << e.what() << std::endl; } return 1; } diff --git a/ydb/public/sdk/cpp/examples/vector_index/vector_index.cpp b/ydb/public/sdk/cpp/examples/vector_index/vector_index.cpp index d8b452bfc416..535d3c52e00c 100644 --- a/ydb/public/sdk/cpp/examples/vector_index/vector_index.cpp +++ b/ydb/public/sdk/cpp/examples/vector_index/vector_index.cpp @@ -5,9 +5,9 @@ #include template <> -struct std::formatter: std::formatter { +struct std::formatter: std::formatter { template - auto format(const TString& param, FormatContext& fc) const { + auto format(const std::string& param, FormatContext& fc) const { return std::formatter::format(std::string_view{param}, fc); } }; @@ -16,7 +16,7 @@ using namespace NYdb; using namespace NTable; namespace { -constexpr ui64 kBulkSize = 1000; +constexpr uint64_t kBulkSize = 1000; constexpr std::string_view FlatIndex = "flat"; namespace NQuantizer { @@ -37,28 +37,28 @@ bool EqualsICase(std::string_view l, std::string_view r) { void PrintTop(TResultSetParser&& parser) { while (parser.TryNextRow()) { Y_ASSERT(parser.ColumnsCount() >= 1); - Cout << *parser.ColumnParser(0).GetOptionalFloat() << "\t"; + std::cout << *parser.ColumnParser(0).GetOptionalFloat() << "\t"; for (size_t i = 1; i < parser.ColumnsCount(); ++i) { - Cout << *parser.ColumnParser(1).GetOptionalUtf8() << "\t"; + std::cout << *parser.ColumnParser(1).GetOptionalUtf8() << "\t"; } - Cout << "\n"; + std::cout << "\n"; } - Cout << Endl; + std::cout << std::endl; } -TString FullName(const TOptions& options, const TString& name) { - return TString::Join(options.Database, "/", name); +std::string FullName(const TOptions& options, const std::string& name) { + return options.Database + std::string("/") + name; } -TString IndexName(const TOptions& options) { - return TString::Join(options.Table, "_", options.IndexType, "_", options.IndexQuantizer); +std::string IndexName(const TOptions& options) { + return options.Table + std::string("_") + options.IndexType + std::string("_") + options.IndexQuantizer; } -TString FullIndexName(const TOptions& options) { +std::string FullIndexName(const TOptions& options) { return FullName(options, IndexName(options)); } -void DropTable(TTableClient& client, const TString& table) { +void DropTable(TTableClient& client, const std::string& table) { auto r = client.RetryOperationSync([&](TSession session) { TDropTableSettings settings; return session.DropTable(table).ExtractValueSync(); @@ -88,7 +88,7 @@ void CreateFlat(TTableClient& client, const TOptions& options) { } void UpdateFlat(TTableClient& client, const TOptions& options, std::string_view type) { - TString query = std::format(R"( + std::string query = std::format(R"( DECLARE $begin AS Uint64; DECLARE $rows AS Uint64; @@ -103,13 +103,13 @@ void UpdateFlat(TTableClient& client, const TOptions& options, std::string_view options.Embedding, type, type == NQuantizer::Bit ? "Float" : type); - Cout << query << Endl; + std::cout << query << std::endl; auto last = std::chrono::steady_clock::now(); - ui64 current = 0; - ui64 overall = (options.Rows + kBulkSize - 1) / kBulkSize; + uint64_t current = 0; + uint64_t overall = (options.Rows + kBulkSize - 1) / kBulkSize; auto report = [&](auto curr) { - Cout << "Already done " << current << " / " << overall << " upserts, time spent: " << std::chrono::duration{curr - last}.count() << Endl; + std::cout << "Already done " << current << " / " << overall << " upserts, time spent: " << std::chrono::duration{curr - last}.count() << std::endl; last = curr; }; auto waitRequest = [&](auto& request) { @@ -135,11 +135,9 @@ void UpdateFlat(TTableClient& client, const TOptions& options, std::string_view TParamsBuilder paramsBuilder; TRetryOperationSettings retrySettings; retrySettings - .MaxRetries(60) - .GetSessionClientTimeout(TDuration::Seconds(60)) - .Idempotent(true) - .RetryUndefined(true); - for (ui64 i = 0; i < options.Rows; i += kBulkSize) { + .MaxRetries(10) + .GetSessionClientTimeout(TDuration::Seconds(1)); + for (uint64_t i = 0; i < options.Rows; i += kBulkSize) { waitFirst(); paramsBuilder.AddParam("$begin").Uint64(i).Build(); paramsBuilder.AddParam("$rows").Uint64(kBulkSize).Build(); @@ -163,7 +161,7 @@ void UpdateFlat(TTableClient& client, const TOptions& options, std::string_view } void TopKFlat(TTableClient& client, const TOptions& options, std::string_view type) { - TString query = std::format(R"( + std::string query = std::format(R"( $TargetBinary = Knn::ToBinaryStringFloat($Target); $TargetSQ = Knn::ToBinaryString{7}(CAST($Target AS List<{8}>)); @@ -187,7 +185,7 @@ void TopKFlat(TTableClient& client, const TOptions& options, std::string_view ty options.TopK, type, type == NQuantizer::Bit ? "Float" : type); - Cout << query << Endl; + std::cout << query << std::endl; std::ifstream targetFileStream(options.Target); std::stringstream targetStrStream; targetStrStream << targetFileStream.rdbuf(); @@ -285,4 +283,4 @@ int TopK(NYdb::TDriver& driver, const TOptions& options) { } } return 1; -} \ No newline at end of file +} diff --git a/ydb/public/sdk/cpp/examples/vector_index/vector_index.h b/ydb/public/sdk/cpp/examples/vector_index/vector_index.h index 8ebcf5ab85d8..532de0fd3b42 100644 --- a/ydb/public/sdk/cpp/examples/vector_index/vector_index.h +++ b/ydb/public/sdk/cpp/examples/vector_index/vector_index.h @@ -1,14 +1,9 @@ #pragma once -#include -#include +#include +#include #include -#include -#include -#include -#include -#include enum class ECommand { DropIndex, @@ -22,17 +17,17 @@ enum class ECommand { ECommand Parse(std::string_view command); struct TOptions { - TString Database; - TString Table; - TString IndexType; - TString IndexQuantizer; - TString PrimaryKey; - TString Embedding; - TString Distance; - TString Data; - TString Target; - ui64 Rows = 0; - ui64 TopK = 0; + std::string Database; + std::string Table; + std::string IndexType; + std::string IndexQuantizer; + std::string PrimaryKey; + std::string Embedding; + std::string Distance; + std::string Data; + std::string Target; + uint64_t Rows = 0; + uint64_t TopK = 0; }; int DropIndex(NYdb::TDriver& driver, const TOptions& options); diff --git a/ydb/public/sdk/cpp/examples/vector_index/ya.make b/ydb/public/sdk/cpp/examples/vector_index/ya.make index 2c7a5c2a6f55..5fda8195cf12 100644 --- a/ydb/public/sdk/cpp/examples/vector_index/ya.make +++ b/ydb/public/sdk/cpp/examples/vector_index/ya.make @@ -1,5 +1,7 @@ PROGRAM() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( main.cpp vector_index.cpp @@ -7,7 +9,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) END() diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/CMakeLists.txt b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/CMakeLists.txt new file mode 100644 index 000000000000..631efeb3ed7e --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(iam/common) diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h new file mode 100644 index 000000000000..151a81e943d9 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3::NStorageConfig { + +struct TFetchStorageConfigResult : public TStatus { + TFetchStorageConfigResult( + TStatus&& status, + std::string&& config) + : TStatus(std::move(status)) + , Config_(std::move(config)) + {} + + const std::string& GetConfig() const { + return Config_; + } + +private: + std::string Config_; +}; + +using TAsyncFetchStorageConfigResult = NThreading::TFuture; + +struct TStorageConfigSettings : public NYdb::TOperationRequestSettings {}; + +class TStorageConfigClient { +public: + + explicit TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + + ~TStorageConfigClient(); + + // Replace config + TAsyncStatus ReplaceStorageConfig(const std::string& config); + + // Fetch current cluster storage config + TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}); + + // Bootstrap cluster with automatic configuration + TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID); + +private: + class TImpl; + + std::unique_ptr Impl_; +}; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/settings.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/settings.h new file mode 100644 index 000000000000..302898dea417 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/settings.h @@ -0,0 +1,61 @@ +#pragma once + +#include "ssl_credentials.h" + +#include +#include +#include + +#include + +namespace NYdb::inline V3 { + +using TCertificateAndPrivateKey = std::pair; + +struct TCommonClientSettings { + using TSelf = TCommonClientSettings; + + //! NOTE: Options below are discovery state specific options. + //! If client overrides it and no state in cache (no client with same settings) + //! ctor may blocks during discovery (according to DiscoveryMode settings + //! The key of state cache is tuple + //! For performance reason (to avoid extra discovery requests) it is + //! always better to keep instance of client. + + //! Allows to override current database for client + FLUENT_SETTING_OPTIONAL(std::string, Database); + //! Allows to override current discovery endpoint + FLUENT_SETTING_OPTIONAL(std::string, DiscoveryEndpoint); + //! Allows to override current token for client + TSelf& AuthToken(const std::optional& token); + //! Allows to override current credentials provider + FLUENT_SETTING_OPTIONAL(std::shared_ptr, CredentialsProviderFactory); + //! Allows to override discovery mode + FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode); + //! Allows to override current Ssl credentials + FLUENT_SETTING_OPTIONAL(TSslCredentials, SslCredentials); +}; + +template +struct TCommonClientSettingsBase : public TCommonClientSettings { + +#define COMMON_CLIENT_SETTINGS_TO_DERIVED(type, name) \ + TDerived& name(const type& value) { \ + TCommonClientSettings::name(value); \ + return static_cast(*this); \ + } + + COMMON_CLIENT_SETTINGS_TO_DERIVED(std::string, Database); + COMMON_CLIENT_SETTINGS_TO_DERIVED(std::string, DiscoveryEndpoint); + COMMON_CLIENT_SETTINGS_TO_DERIVED(std::optional, AuthToken); + COMMON_CLIENT_SETTINGS_TO_DERIVED(std::shared_ptr, CredentialsProviderFactory); + COMMON_CLIENT_SETTINGS_TO_DERIVED(EDiscoveryMode, DiscoveryMode); + COMMON_CLIENT_SETTINGS_TO_DERIVED(TSslCredentials, SslCredentials); + +#undef COMMON_CLIENT_SETTINGS_TO_DERIVED + +}; + +TCommonClientSettings GetClientSettingsFromConnectionString(const std::string& connectionString); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/ssl_credentials.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/ssl_credentials.h new file mode 100644 index 000000000000..3b452b9d03dd --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/common_client/ssl_credentials.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +struct TSslCredentials { + bool IsEnabled = false; + std::string CaCert; + std::string Cert; + std::string PrivateKey; + + TSslCredentials() = default; + + TSslCredentials(const bool isEnabled, const std::string& caCert = "", const std::string& cert = "", const std::string& privateKey = "") + : IsEnabled(isEnabled), CaCert(caCert), Cert(cert), PrivateKey(privateKey) {} + + bool operator==(const TSslCredentials& other) const { + return IsEnabled == other.IsEnabled && + CaCert == other.CaCert && + Cert == other.Cert && + PrivateKey == other.PrivateKey; + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/coordination/coordination.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/coordination/coordination.h new file mode 100644 index 000000000000..8d4fa0f1f452 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/coordination/coordination.h @@ -0,0 +1,363 @@ +#pragma once + +#include + +namespace Ydb { +namespace Coordination { + class DescribeNodeResult; + class SemaphoreSession; + class SemaphoreDescription; +} +} + +namespace NYdb::inline V3 { + +namespace NScheme { +struct TPermissions; +} + +namespace NCoordination { + +//! Represents result of a call with status +template +class TResult : public TStatus { +public: + template + explicit TResult(TStatusArg&& status, TArgs&&... args) + : TStatus(std::forward(status)) + , Result_(std::forward(args)...) + { } + + // Alow copy constructors with the same type + TResult(TResult&& rhs) = default; + TResult(const TResult& rhs) = default; + + // Don't allow accidental type conversions (even explicit) + template TResult(TResult&& rhs) = delete; + template TResult(const TResult& rhs) = delete; + + const T& GetResult() const { + CheckStatusOk("TResult::GetResult"); + return Result_; + } + + T&& ExtractResult() { + CheckStatusOk("TResult::ExtractResult"); + return std::move(Result_); + } + +private: + T Result_; +}; + +template<> +class TResult : public TStatus { +public: + template + explicit TResult(TStatusArg&& status) + : TStatus(std::forward(status)) + { } + + // Allow copy constructors with the same type + TResult(TResult&& rhs) = default; + TResult(const TResult& rhs) = default; + + // Don't allow accidental type conversions (even explicit) + template TResult(TResult&& rhs) = delete; + template TResult(const TResult& rhs) = delete; + + void GetResult() const { + CheckStatusOk("TResult::GetResult"); + } + + void ExtractResult() { + CheckStatusOk("TResult::ExtractResult"); + } +}; + +template +using TAsyncResult = NThreading::TFuture>; + +//////////////////////////////////////////////////////////////////////////////// + +enum class EConsistencyMode { + UNSET = 0, + STRICT_MODE = 1, + RELAXED_MODE = 2, +}; + +enum class ERateLimiterCountersMode { + UNSET = 0, + AGGREGATED = 1, + DETAILED = 2, +}; + +//! Represents coordination node description +class TNodeDescription { +public: + TNodeDescription(const Ydb::Coordination::DescribeNodeResult& desc); + + const std::optional& GetSelfCheckPeriod() const; + const std::optional& GetSessionGracePeriod() const; + EConsistencyMode GetReadConsistencyMode() const; + EConsistencyMode GetAttachConsistencyMode() const; + ERateLimiterCountersMode GetRateLimiterCountersMode() const; + + const std::string& GetOwner() const; + const std::vector& GetEffectivePermissions() const; + const Ydb::Coordination::DescribeNodeResult& GetProto() const; + +private: + struct TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents session owning or waiting on a semaphore +class TSemaphoreSession { +public: + TSemaphoreSession(); + + TSemaphoreSession(const Ydb::Coordination::SemaphoreSession& desc); + + uint64_t GetOrderId() const { return OrderId_; } + uint64_t GetSessionId() const { return SessionId_; } + TDuration GetTimeout() const { return Timeout_; } + uint64_t GetCount() const { return Count_; } + const std::string& GetData() const { return Data_; } + +private: + uint64_t OrderId_; + uint64_t SessionId_; + TDuration Timeout_; + uint64_t Count_; + std::string Data_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents semaphore description +class TSemaphoreDescription { +public: + TSemaphoreDescription(); + + TSemaphoreDescription(const Ydb::Coordination::SemaphoreDescription& desc); + + const std::string& GetName() const { return Name_; } + const std::string& GetData() const { return Data_; } + uint64_t GetCount() const { return Count_; } + uint64_t GetLimit() const { return Limit_; } + const std::vector& GetOwners() const { return Owners_; } + const std::vector& GetWaiters() const { return Waiters_; } + bool IsEphemeral() const { return IsEphemeral_; } + +private: + std::string Name_; + std::string Data_; + uint64_t Count_; + uint64_t Limit_; + std::vector Owners_; + std::vector Waiters_; + bool IsEphemeral_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +using TDescribeNodeResult = TResult; + +using TAsyncDescribeNodeResult = NThreading::TFuture; + +using TDescribeSemaphoreResult = TResult; + +using TAsyncDescribeSemaphoreResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +template +struct TNodeSettings : public TOperationRequestSettings { + using TSelf = TDerived; + + FLUENT_SETTING_OPTIONAL(TDuration, SelfCheckPeriod); + + FLUENT_SETTING_OPTIONAL(TDuration, SessionGracePeriod); + + FLUENT_SETTING_DEFAULT(EConsistencyMode, ReadConsistencyMode, EConsistencyMode::UNSET); + + FLUENT_SETTING_DEFAULT(EConsistencyMode, AttachConsistencyMode, EConsistencyMode::UNSET); + + FLUENT_SETTING_DEFAULT(ERateLimiterCountersMode, RateLimiterCountersMode, ERateLimiterCountersMode::UNSET); +}; + +struct TCreateNodeSettings : public TNodeSettings { }; +struct TAlterNodeSettings : public TNodeSettings { }; +struct TDropNodeSettings : public TOperationRequestSettings { }; +struct TDescribeNodeSettings : public TOperationRequestSettings { }; + +//////////////////////////////////////////////////////////////////////////////// + +class TSession; + +using TSessionResult = TResult; + +using TAsyncSessionResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +enum class ESessionState { + ATTACHED, + DETACHED, + EXPIRED, +}; + +enum class EConnectionState { + CONNECTING, + ATTACHING, + CONNECTED, + DISCONNECTED, + STOPPED, +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TSessionSettings : public TRequestSettings { + using TSelf = TSessionSettings; + using TStateCallback = std::function; + using TStoppedCallback = std::function; + + FLUENT_SETTING(std::string, Description); + + FLUENT_SETTING(TStateCallback, OnStateChanged); + + FLUENT_SETTING(TStoppedCallback, OnStopped); + + FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Seconds(5)); + + FLUENT_SETTING_DEFAULT(TDuration, ReconnectBackoffDelay, TDuration::MilliSeconds(250)); + + FLUENT_SETTING_DEFAULT(double, ReconnectBackoffMultiplier, 2.0); + + FLUENT_SETTING_DEFAULT(double, ReconnectSessionTimeoutMultiplier, 2.0); + + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Zero()); +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TAcquireSemaphoreSettings { + using TSelf = TAcquireSemaphoreSettings; + using TAcceptedCallback = std::function; + + FLUENT_SETTING(std::string, Data); + + FLUENT_SETTING(TAcceptedCallback, OnAccepted); + + FLUENT_SETTING_DEFAULT(uint64_t, Count, 0); + + FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Max()); + + FLUENT_SETTING_FLAG(Ephemeral); + + FLUENT_SETTING_FLAG_ALIAS(Shared, Count, uint64_t(1)); + + FLUENT_SETTING_FLAG_ALIAS(Exclusive, Count, uint64_t(-1)); +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TDescribeSemaphoreSettings { + using TSelf = TDescribeSemaphoreSettings; + using TChangedCallback = std::function; + + FLUENT_SETTING(TChangedCallback, OnChanged); + + FLUENT_SETTING_FLAG(WatchData); + + FLUENT_SETTING_FLAG(WatchOwners); + + FLUENT_SETTING_FLAG(IncludeOwners); + + FLUENT_SETTING_FLAG(IncludeWaiters); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TClient { +public: + TClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + ~TClient(); + + TAsyncSessionResult StartSession(const std::string& path, + const TSessionSettings& settings = TSessionSettings()); + + TAsyncStatus CreateNode(const std::string& path, + const TCreateNodeSettings& settings = TCreateNodeSettings()); + + TAsyncStatus AlterNode(const std::string& path, + const TAlterNodeSettings& settings = TAlterNodeSettings()); + + TAsyncStatus DropNode(const std::string& path, + const TDropNodeSettings& settings = TDropNodeSettings()); + + TAsyncDescribeNodeResult DescribeNode(const std::string& path, + const TDescribeNodeSettings& settings = TDescribeNodeSettings()); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TSessionContext; + +class TSession { + friend class TSessionContext; + +public: + TSession() = default; + + explicit operator bool() const { + return bool(Impl_); + } + + uint64_t GetSessionId(); + + ESessionState GetSessionState(); + + EConnectionState GetConnectionState(); + + TAsyncResult Close(); + + TAsyncResult Ping(); + + TAsyncResult Reconnect(); + + TAsyncResult AcquireSemaphore(const std::string& name, + const TAcquireSemaphoreSettings& settings); + + TAsyncResult ReleaseSemaphore(const std::string& name); + + TAsyncDescribeSemaphoreResult DescribeSemaphore(const std::string& name, + const TDescribeSemaphoreSettings& settings = TDescribeSemaphoreSettings()); + + TAsyncResult CreateSemaphore(const std::string& name, + uint64_t limit, const std::string& data = std::string()); + + TAsyncResult UpdateSemaphore(const std::string& name, + const std::string& data); + + TAsyncResult DeleteSemaphore(const std::string& name, + bool force = false); + +private: + explicit TSession(TSessionContext* context); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +} +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/datastreams/datastreams.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/datastreams/datastreams.h new file mode 100644 index 000000000000..9b4dada57e9e --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/datastreams/datastreams.h @@ -0,0 +1,373 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3::NDataStreams::V1 { + + template + class TProtoResultWrapper : public NYdb::TStatus { + friend class TDataStreamsClient; + + private: + TProtoResultWrapper( + NYdb::TStatus&& status, + std::unique_ptr result) + : TStatus(std::move(status)) + , Result(std::move(result)) + { } + + public: + const TProtoResult& GetResult() const { + Y_ABORT_UNLESS(Result, "Uninitialized result"); + return *Result; + } + + private: + std::unique_ptr Result; + }; + + enum EStreamMode { + ESM_PROVISIONED = 1, + ESM_ON_DEMAND = 2, + }; + + using TCreateStreamResult = TProtoResultWrapper; + using TDeleteStreamResult = TProtoResultWrapper; + using TDescribeStreamResult = TProtoResultWrapper; + using TPutRecordResult = TProtoResultWrapper; + using TRegisterStreamConsumerResult = TProtoResultWrapper; + using TDeregisterStreamConsumerResult = TProtoResultWrapper; + using TDescribeStreamConsumerResult = TProtoResultWrapper; + using TListStreamsResult = TProtoResultWrapper; + using TListShardsResult = TProtoResultWrapper; + using TPutRecordsResult = TProtoResultWrapper; + using TGetRecordsResult = TProtoResultWrapper; + using TGetShardIteratorResult = TProtoResultWrapper; + // using TSubscribeToShardResult = TProtoResultWrapper; + using TDescribeLimitsResult = TProtoResultWrapper; + using TDescribeStreamSummaryResult = TProtoResultWrapper; + using TDecreaseStreamRetentionPeriodResult = TProtoResultWrapper; + using TIncreaseStreamRetentionPeriodResult = TProtoResultWrapper; + using TUpdateShardCountResult = TProtoResultWrapper; + using TUpdateStreamModeResult = TProtoResultWrapper; + using TListStreamConsumersResult = TProtoResultWrapper; + using TAddTagsToStreamResult = TProtoResultWrapper; + using TDisableEnhancedMonitoringResult = TProtoResultWrapper; + using TEnableEnhancedMonitoringResult = TProtoResultWrapper; + using TListTagsForStreamResult = TProtoResultWrapper; + using TMergeShardsResult = TProtoResultWrapper; + using TRemoveTagsFromStreamResult = TProtoResultWrapper; + using TSplitShardResult = TProtoResultWrapper; + using TStartStreamEncryptionResult = TProtoResultWrapper; + using TStopStreamEncryptionResult = TProtoResultWrapper; + using TUpdateStreamResult = TProtoResultWrapper; + + using TAsyncCreateStreamResult = NThreading::TFuture; + using TAsyncDeleteStreamResult = NThreading::TFuture; + using TAsyncDescribeStreamResult = NThreading::TFuture; + using TAsyncPutRecordResult = NThreading::TFuture; + using TAsyncRegisterStreamConsumerResult = NThreading::TFuture; + using TAsyncDeregisterStreamConsumerResult = NThreading::TFuture; + using TAsyncDescribeStreamConsumerResult = NThreading::TFuture; + using TAsyncListStreamsResult = NThreading::TFuture; + using TAsyncListShardsResult = NThreading::TFuture; + using TAsyncPutRecordsResult = NThreading::TFuture; + using TAsyncGetRecordsResult = NThreading::TFuture; + using TAsyncGetShardIteratorResult = NThreading::TFuture; + // using TAsyncSubscribeToShardResult = NThreading::TFuture; + using TAsyncDescribeLimitsResult = NThreading::TFuture; + using TAsyncDescribeStreamSummaryResult = NThreading::TFuture; + using TAsyncDecreaseStreamRetentionPeriodResult = NThreading::TFuture; + using TAsyncIncreaseStreamRetentionPeriodResult = NThreading::TFuture; + using TAsyncUpdateShardCountResult = NThreading::TFuture; + using TAsyncUpdateStreamModeResult = NThreading::TFuture; + using TAsyncListStreamConsumersResult = NThreading::TFuture; + using TAsyncAddTagsToStreamResult = NThreading::TFuture; + using TAsyncDisableEnhancedMonitoringResult = NThreading::TFuture; + using TAsyncEnableEnhancedMonitoringResult = NThreading::TFuture; + using TAsyncListTagsForStreamResult = NThreading::TFuture; + using TAsyncMergeShardsResult = NThreading::TFuture; + using TAsyncRemoveTagsFromStreamResult = NThreading::TFuture; + using TAsyncSplitShardResult = NThreading::TFuture; + using TAsyncStartStreamEncryptionResult = NThreading::TFuture; + using TAsyncStopStreamEncryptionResult = NThreading::TFuture; + using TAsyncUpdateStreamResult = NThreading::TFuture; + + struct TDataRecord { + std::string Data; + std::string PartitionKey; + std::string ExplicitHashDecimal; + }; + + enum class EAutoPartitioningStrategy: uint32_t { + Unspecified = 0, + Disabled = 1, + ScaleUp = 2, + ScaleUpAndDown = 3, + Paused = 4, + }; + + struct TCreateStreamSettings; + struct TUpdateStreamSettings; + + + template + struct TPartitioningSettingsBuilder; + template + struct TAutoPartitioningSettingsBuilder; + + struct TAutoPartitioningSettings { + friend struct TAutoPartitioningSettingsBuilder; + friend struct TAutoPartitioningSettingsBuilder; + public: + TAutoPartitioningSettings() + : Strategy_(EAutoPartitioningStrategy::Disabled) + , StabilizationWindow_(TDuration::Seconds(0)) + , DownUtilizationPercent_(0) + , UpUtilizationPercent_(0) { + } + TAutoPartitioningSettings(const Ydb::DataStreams::V1::AutoPartitioningSettings& settings); + TAutoPartitioningSettings(EAutoPartitioningStrategy strategy, TDuration stabilizationWindow, uint64_t downUtilizationPercent, uint64_t upUtilizationPercent) + : Strategy_(strategy) + , StabilizationWindow_(stabilizationWindow) + , DownUtilizationPercent_(downUtilizationPercent) + , UpUtilizationPercent_(upUtilizationPercent) {} + + EAutoPartitioningStrategy GetStrategy() const { return Strategy_; }; + TDuration GetStabilizationWindow() const { return StabilizationWindow_; }; + uint32_t GetDownUtilizationPercent() const { return DownUtilizationPercent_; }; + uint32_t GetUpUtilizationPercent() const { return UpUtilizationPercent_; }; + private: + EAutoPartitioningStrategy Strategy_; + TDuration StabilizationWindow_; + uint32_t DownUtilizationPercent_; + uint32_t UpUtilizationPercent_; + }; + + + class TPartitioningSettings { + using TSelf = TPartitioningSettings; + friend struct TPartitioningSettingsBuilder; + friend struct TPartitioningSettingsBuilder; + public: + TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), AutoPartitioningSettings_(){} + TPartitioningSettings(const Ydb::DataStreams::V1::PartitioningSettings& settings); + TPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions, TAutoPartitioningSettings autoscalingSettings = {}) + : MinActivePartitions_(minActivePartitions) + , MaxActivePartitions_(maxActivePartitions) + , AutoPartitioningSettings_(autoscalingSettings) { + } + + uint64_t GetMinActivePartitions() const { return MinActivePartitions_; }; + uint64_t GetMaxActivePartitions() const { return MaxActivePartitions_; }; + TAutoPartitioningSettings GetAutoPartitioningSettings() const { return AutoPartitioningSettings_; }; + private: + uint64_t MinActivePartitions_; + uint64_t MaxActivePartitions_; + TAutoPartitioningSettings AutoPartitioningSettings_; + }; + + struct TCreateStreamSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, ShardCount); + FLUENT_SETTING_OPTIONAL(uint32_t, RetentionPeriodHours); + FLUENT_SETTING_OPTIONAL(uint32_t, RetentionStorageMegabytes); + FLUENT_SETTING(uint64_t, WriteQuotaKbPerSec); + FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + + + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); + }; + + template + struct TAutoPartitioningSettingsBuilder { + using TSelf = TAutoPartitioningSettingsBuilder; + public: + TAutoPartitioningSettingsBuilder(TPartitioningSettingsBuilder& parent, TAutoPartitioningSettings& settings): Parent_(parent), Settings_(settings) {} + + TSelf Strategy(EAutoPartitioningStrategy value) { + Settings_.Strategy_ = value; + return *this; + } + + TSelf StabilizationWindow(TDuration value) { + Settings_.StabilizationWindow_ = value; + return *this; + } + + TSelf DownUtilizationPercent(uint32_t value) { + Settings_.DownUtilizationPercent_ = value; + return *this; + } + + TSelf UpUtilizationPercent(uint32_t value) { + Settings_.UpUtilizationPercent_ = value; + return *this; + } + + TPartitioningSettingsBuilder& EndConfigureAutoPartitioningSettings() { + return Parent_; + } + + private: + TPartitioningSettingsBuilder& Parent_; + TAutoPartitioningSettings& Settings_; + }; + + template + struct TPartitioningSettingsBuilder { + using TSelf = TPartitioningSettingsBuilder; + public: + TPartitioningSettingsBuilder(TSettings& parent): Parent_(parent) {} + + TSelf MinActivePartitions(uint64_t value) { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + (*Parent_.PartitioningSettings_).MinActivePartitions_ = value; + return *this; + } + + TSelf MaxActivePartitions(uint64_t value) { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + (*Parent_.PartitioningSettings_).MaxActivePartitions_ = value; + return *this; + } + + TAutoPartitioningSettingsBuilder BeginConfigureAutoPartitioningSettings() { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + return {*this, (*Parent_.PartitioningSettings_).AutoPartitioningSettings_}; + } + + TSettings& EndConfigurePartitioningSettings() { + return Parent_; + } + + private: + TSettings& Parent_; + }; + + struct TListStreamsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, Limit); + FLUENT_SETTING(std::string, ExclusiveStartStreamName); + FLUENT_SETTING_DEFAULT(bool, Recurse, true); + }; + struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false); + }; + struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, Limit); + FLUENT_SETTING(std::string, ExclusiveStartShardId); + }; + struct TListShardsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(std::string, ExclusiveStartShardId); + FLUENT_SETTING(uint32_t, MaxResults); + FLUENT_SETTING(std::string, NextToken); + FLUENT_SETTING(uint64_t, StreamCreationTimestamp); + }; + struct TGetRecordsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(uint32_t, Limit, 10000); + }; + struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(std::string, StartingSequenceNumber); + FLUENT_SETTING(uint64_t, Timestamp); + }; + struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings {}; + struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings {}; + struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings {}; + struct TDecreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, RetentionPeriodHours); + }; + struct TIncreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, RetentionPeriodHours); + }; + struct TUpdateShardCountSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, TargetShardCount); + }; + struct TUpdateStreamModeSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(EStreamMode, StreamMode, ESM_PROVISIONED); + }; + struct TUpdateStreamSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, TargetShardCount); + FLUENT_SETTING_OPTIONAL(uint32_t, RetentionPeriodHours); + FLUENT_SETTING_OPTIONAL(uint32_t, RetentionStorageMegabytes); + FLUENT_SETTING(uint64_t, WriteQuotaKbPerSec); + FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); + }; + struct TPutRecordSettings : public NYdb::TOperationRequestSettings {}; + struct TPutRecordsSettings : public NYdb::TOperationRequestSettings {}; + struct TRegisterStreamConsumerSettings : public NYdb::TOperationRequestSettings {}; + struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings {}; + struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings {}; + struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(uint32_t, MaxResults); + FLUENT_SETTING(std::string, NextToken); + }; + struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings {}; + struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings {}; + struct TEnableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings {}; + struct TListTagsForStreamSettings : public NYdb::TOperationRequestSettings {}; + struct TMergeShardsSettings : public NYdb::TOperationRequestSettings {}; + struct TRemoveTagsFromStreamSettings : public NYdb::TOperationRequestSettings {}; + struct TSplitShardSettings : public NYdb::TOperationRequestSettings {}; + struct TStartStreamEncryptionSettings : public NYdb::TOperationRequestSettings {}; + struct TStopStreamEncryptionSettings : public NYdb::TOperationRequestSettings {}; + struct TProtoRequestSettings : public NYdb::TOperationRequestSettings {}; + + class TDataStreamsClient { + class TImpl; + + public: + TDataStreamsClient(const NYdb::TDriver& driver, const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings()); + + TAsyncCreateStreamResult CreateStream(const std::string& path, TCreateStreamSettings settings = TCreateStreamSettings()); + TAsyncDeleteStreamResult DeleteStream(const std::string& path, TDeleteStreamSettings settings = TDeleteStreamSettings()); + TAsyncDescribeStreamResult DescribeStream(const std::string& path, TDescribeStreamSettings settings = TDescribeStreamSettings()); + TAsyncPutRecordResult PutRecord(const std::string& path, const TDataRecord& record, TPutRecordSettings settings = TPutRecordSettings()); + TAsyncListStreamsResult ListStreams(TListStreamsSettings settings = TListStreamsSettings()); + TAsyncListShardsResult ListShards(const std::string& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings()); + TAsyncPutRecordsResult PutRecords(const std::string& path, const std::vector& records, TPutRecordsSettings settings = TPutRecordsSettings()); + TAsyncGetRecordsResult GetRecords(const std::string& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings()); + TAsyncGetShardIteratorResult GetShardIterator(const std::string& path, const std::string& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr, + TGetShardIteratorSettings settings = TGetShardIteratorSettings()); + // TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings = TSubscribeToShardSettings()); + TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings = TDescribeLimitsSettings()); + TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const std::string& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings()); + TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const std::string& path, TDecreaseStreamRetentionPeriodSettings settings = TDecreaseStreamRetentionPeriodSettings()); + TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const std::string& path, TIncreaseStreamRetentionPeriodSettings settings = TIncreaseStreamRetentionPeriodSettings()); + TAsyncUpdateShardCountResult UpdateShardCount(const std::string& path, TUpdateShardCountSettings settings = TUpdateShardCountSettings()); + TAsyncUpdateStreamModeResult UpdateStreamMode(const std::string& path, TUpdateStreamModeSettings settings = TUpdateStreamModeSettings()); + TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const std::string& path, const std::string& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings()); + TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const std::string& path, const std::string& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings()); + TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings = TDescribeStreamConsumerSettings()); + TAsyncListStreamConsumersResult ListStreamConsumers(const std::string& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings()); + TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings = TAddTagsToStreamSettings()); + TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings = TDisableEnhancedMonitoringSettings()); + TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings = TEnableEnhancedMonitoringSettings()); + TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings = TListTagsForStreamSettings()); + TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings = TMergeShardsSettings()); + TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings = TRemoveTagsFromStreamSettings()); + TAsyncSplitShardResult SplitShard(TSplitShardSettings settings = TSplitShardSettings()); + TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings = TStartStreamEncryptionSettings()); + TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings = TStopStreamEncryptionSettings()); + TAsyncUpdateStreamResult UpdateStream(const std::string& streamName, TUpdateStreamSettings settings = TUpdateStreamSettings()); + + template + NThreading::TFuture> DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings = TProtoRequestSettings()); + + NThreading::TFuture DiscoveryCompleted(); + + private: + std::shared_ptr Impl_; + }; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/debug/client.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/debug/client.h new file mode 100644 index 000000000000..7ee45e5a4fc2 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/debug/client.h @@ -0,0 +1,83 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NDebug { + +//////////////////////////////////////////////////////////////////////////////// + +class TPlainGrpcPingResult: public TStatus { +public: + TPlainGrpcPingResult(TStatus&& status) + : TStatus(std::move(status)) + {} +}; + +class TGrpcProxyPingResult: public TStatus { +public: + TGrpcProxyPingResult(TStatus&& status) + : TStatus(std::move(status)) + {} +}; + +class TKqpProxyPingResult: public TStatus { +public: + TKqpProxyPingResult(TStatus&& status) + : TStatus(std::move(status)) + {} +}; + +class TSchemeCachePingResult: public TStatus { +public: + TSchemeCachePingResult(TStatus&& status) + : TStatus(std::move(status)) + {} +}; + +class TTxProxyPingResult: public TStatus { +public: + TTxProxyPingResult(TStatus&& status) + : TStatus(std::move(status)) + {} +}; + +//////////////////////////////////////////////////////////////////////////////// + +using TAsyncPlainGrpcPingResult = NThreading::TFuture; +using TAsyncGrpcProxyPingResult = NThreading::TFuture; +using TAsyncKqpProxyPingResult = NThreading::TFuture; +using TAsyncSchemeCachePingResult = NThreading::TFuture; +using TAsyncTxProxyPingResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +struct TPlainGrpcPingSettings : public TOperationRequestSettings {}; +struct TGrpcProxyPingSettings : public TOperationRequestSettings {}; +struct TKqpProxyPingSettings : public TOperationRequestSettings {}; +struct TSchemeCachePingSettings : public TOperationRequestSettings {}; +struct TTxProxyPingSettings : public TOperationRequestSettings {}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TClientSettings : public TCommonClientSettingsBase { +}; + +class TDebugClient { +public: + using TAsyncPlainGrpcPingResult = TAsyncPlainGrpcPingResult; +public: + TDebugClient(const TDriver& driver, const TClientSettings& settings = TClientSettings()); + + TAsyncPlainGrpcPingResult PingPlainGrpc(const TPlainGrpcPingSettings& settings); + TAsyncGrpcProxyPingResult PingGrpcProxy(const TGrpcProxyPingSettings& settings); + TAsyncKqpProxyPingResult PingKqpProxy(const TKqpProxyPingSettings& settings); + + TAsyncSchemeCachePingResult PingSchemeCache(const TSchemeCachePingSettings& settings); + TAsyncTxProxyPingResult PingTxProxy(const TTxProxyPingSettings& settings); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NDebug diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/discovery/discovery.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/discovery/discovery.h new file mode 100644 index 000000000000..f9bf7a2723c8 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/discovery/discovery.h @@ -0,0 +1,144 @@ +#pragma once + +#include + +namespace Ydb { +namespace Discovery { + class ListEndpointsResult; + class WhoAmIResult; + class NodeRegistrationResult; + class NodeLocation; + class NodeInfo; +} // namespace Discovery +} // namespace Ydb + +namespace NYdb::inline V3 { +namespace NDiscovery { + +//////////////////////////////////////////////////////////////////////////////// + +class TListEndpointsSettings : public TSimpleRequestSettings {}; + +struct TWhoAmISettings : public TSimpleRequestSettings { + FLUENT_SETTING_DEFAULT(bool, WithGroups, false); +}; + +struct TNodeLocation { + TNodeLocation() = default; + TNodeLocation(const Ydb::Discovery::NodeLocation& location); + + std::optional DataCenterNum; + std::optional RoomNum; + std::optional RackNum; + std::optional BodyNum; + std::optional Body; + + std::optional DataCenter; + std::optional Module; + std::optional Rack; + std::optional Unit; +}; + +struct TNodeRegistrationSettings : public TSimpleRequestSettings { + FLUENT_SETTING(std::string, Host); + FLUENT_SETTING(uint32_t, Port); + FLUENT_SETTING(std::string, ResolveHost); + FLUENT_SETTING(std::string, Address); + FLUENT_SETTING(TNodeLocation, Location); + FLUENT_SETTING(std::string, DomainPath); + FLUENT_SETTING_DEFAULT(bool, FixedNodeId, false); + FLUENT_SETTING(std::string, Path); +}; + +struct TEndpointInfo { + std::string Address; + uint32_t Port = 0; + float LoadFactor = 0.0; + bool Ssl = false; + std::vector Services; + std::string Location; + uint32_t NodeId = 0; + std::vector IPv4Addrs; + std::vector IPv6Addrs; + std::string SslTargetNameOverride; +}; + +class TListEndpointsResult : public TStatus { +public: + TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& endpoints); + const std::vector& GetEndpointsInfo() const; +private: + std::vector Info_; +}; + +using TAsyncListEndpointsResult = NThreading::TFuture; + +class TWhoAmIResult : public TStatus { +public: + TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto); + const std::string& GetUserName() const; + const std::vector& GetGroups() const; +private: + std::string UserName_; + std::vector Groups_; +}; + +using TAsyncWhoAmIResult = NThreading::TFuture; + +struct TNodeInfo { + TNodeInfo() = default; + TNodeInfo(const Ydb::Discovery::NodeInfo& info); + + uint32_t NodeId; + std::string Host; + uint32_t Port; + std::string ResolveHost; + std::string Address; + TNodeLocation Location; + uint64_t Expire; +}; + +class TNodeRegistrationResult : public TStatus { +public: + TNodeRegistrationResult() : TStatus(EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues()) {} + TNodeRegistrationResult(TStatus&& status, const Ydb::Discovery::NodeRegistrationResult& proto); + uint32_t GetNodeId() const; + const std::string& GetDomainPath() const; + uint64_t GetExpire() const; + uint64_t GetScopeTabletId() const; + bool HasScopeTabletId() const; + uint64_t GetScopePathId() const; + bool HasScopePathId() const; + const std::string& GetNodeName() const; + bool HasNodeName() const; + const std::vector& GetNodes() const; + +private: + uint32_t NodeId_; + std::string DomainPath_; + uint64_t Expire_; + std::optional ScopeTableId_; + std::optional ScopePathId_; + std::optional NodeName_; + std::vector Nodes_; +}; + +using TAsyncNodeRegistrationResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +class TDiscoveryClient { +public: + explicit TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings = TListEndpointsSettings()); + TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings = TWhoAmISettings()); + TAsyncNodeRegistrationResult NodeRegistration(const TNodeRegistrationSettings& settings = TNodeRegistrationSettings()); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +} // namespace NDiscovery +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_dynamic_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_dynamic_config.h new file mode 100644 index 000000000000..5333a66fa784 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_dynamic_config.h @@ -0,0 +1,247 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NDynamicConfig { + +struct TGetConfigResult : public TStatus { + TGetConfigResult( + TStatus&& status, + std::string&& clusterName, + uint64_t version, + std::string&& config, + std::map&& volatileConfigs) + : TStatus(std::move(status)) + , ClusterName_(std::move(clusterName)) + , Version_(version) + , Config_(std::move(config)) + , VolatileConfigs_(std::move(volatileConfigs)) + {} + + const std::string& GetClusterName() const { + return ClusterName_; + } + + uint64_t GetVersion() const { + return Version_; + } + + const std::string& GetConfig() const { + return Config_; + } + + const std::map& GetVolatileConfigs() const { + return VolatileConfigs_; + } + +private: + std::string ClusterName_; + uint64_t Version_; + std::string Config_; + std::map VolatileConfigs_; +}; + +using TAsyncGetConfigResult = NThreading::TFuture; + +struct TGetMetadataResult : public TStatus { + TGetMetadataResult( + TStatus&& status, + std::string&& metadata, + std::map&& volatileConfigs) + : TStatus(std::move(status)) + , Metadata_(std::move(metadata)) + , VolatileConfigs_(std::move(volatileConfigs)) + {} + + const std::string& GetMetadata() const { + return Metadata_; + } + + const std::map& GetVolatileConfigs() const { + return VolatileConfigs_; + } + +private: + std::string Metadata_; + std::map VolatileConfigs_; +}; + +using TAsyncGetMetadataResult = NThreading::TFuture; + +struct TResolveConfigResult : public TStatus { + TResolveConfigResult( + TStatus&& status, + std::string&& config) + : TStatus(std::move(status)) + , Config_(std::move(config)) + {} + + const std::string& GetConfig() const { + return Config_; + } + +private: + std::string Config_; +}; + +using TAsyncResolveConfigResult = NThreading::TFuture; + +struct TGetNodeLabelsResult : public TStatus { + TGetNodeLabelsResult( + TStatus&& status, + std::map&& labels) + : TStatus(std::move(status)) + , Labels_(std::move(labels)) + {} + + const std::map& GetLabels() const { + return Labels_; + } + +private: + std::map Labels_; +}; + +using TAsyncGetNodeLabelsResult = NThreading::TFuture; + +struct TVerboseResolveConfigResult : public TStatus { + struct TLabel { + enum class EType { + Negative = 0, + Empty, + Common, + }; + + EType Type; + std::string Value; + + bool operator<(const TLabel& other) const { + int lhs = static_cast(Type); + int rhs = static_cast(other.Type); + return std::tie(lhs, Value) < std::tie(rhs, other.Value); + } + + bool operator==(const TLabel& other) const { + int lhs = static_cast(Type); + int rhs = static_cast(other.Type); + return std::tie(lhs, Value) == std::tie(rhs, other.Value); + } + }; + + using ConfigByLabelSet = std::map>, std::string>; + + TVerboseResolveConfigResult( + TStatus&& status, + std::set&& labels, + ConfigByLabelSet&& configs) + : TStatus(std::move(status)) + , Labels_(std::move(labels)) + , Configs_(std::move(configs)) + {} + + const std::set& GetLabels() const { + return Labels_; + } + + const ConfigByLabelSet& GetConfigs() const { + return Configs_; + } + +private: + std::set Labels_; + ConfigByLabelSet Configs_; +}; + +using TAsyncVerboseResolveConfigResult = NThreading::TFuture; + + +struct TDynamicConfigClientSettings : public TCommonClientSettingsBase { + using TSelf = TDynamicConfigClientSettings; +}; + +struct TClusterConfigSettings : public NYdb::TOperationRequestSettings {}; + +class TDynamicConfigClient { +public: + class TImpl; + + explicit TDynamicConfigClient(const TDriver& driver); + + // Set config + TAsyncStatus SetConfig(const std::string& config, bool dryRun = false, bool allowUnknownFields = false, const TClusterConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfig(const std::string& config, bool dryRun = false, bool allowUnknownFields = false, const TClusterConfigSettings& settings = {}); + + // Drop config + TAsyncStatus DropConfig( + const std::string& cluster, + uint64_t version, + const TClusterConfigSettings& settings = {}); + + // Add volatile config + TAsyncStatus AddVolatileConfig( + const std::string& config, + const TClusterConfigSettings& settings = {}); + + // Remove specific volatile configs + TAsyncStatus RemoveVolatileConfig( + const std::string& cluster, + uint64_t version, + const std::vector& ids, + const TClusterConfigSettings& settings = {}); + + // Remove all volatile config + TAsyncStatus RemoveAllVolatileConfigs( + const std::string& cluster, + uint64_t version, + const TClusterConfigSettings& settings = {}); + + // Remove specific volatile configs + TAsyncStatus ForceRemoveVolatileConfig( + const std::vector& ids, + const TClusterConfigSettings& settings = {}); + + // Remove all volatile config + TAsyncStatus ForceRemoveAllVolatileConfigs( + const TClusterConfigSettings& settings = {}); + + // Get current cluster configs metadata + TAsyncGetMetadataResult GetMetadata(const TClusterConfigSettings& settings = {}); + + // Get current cluster configs + TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}); + + // Get current cluster configs + TAsyncGetNodeLabelsResult GetNodeLabels(uint64_t nodeId, const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for specific labels + TAsyncResolveConfigResult ResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const std::map& labels, + const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for all label combinations + TAsyncResolveConfigResult ResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for all label combinations + TAsyncVerboseResolveConfigResult VerboseResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const TClusterConfigSettings& settings = {}); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::NDynamicConfig diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h new file mode 100644 index 000000000000..2448d3b01c33 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h @@ -0,0 +1,191 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace Ydb::Replication { + class ConnectionParams; + class ConsistencyLevelGlobal; + class DescribeReplicationResult; + class DescribeReplicationResult_Stats; +} + +namespace NYdb::inline V3 { + class TProtoAccessor; +} + +namespace NYdb::inline V3::NReplication { + +class TDescribeReplicationResult; +using TAsyncDescribeReplicationResult = NThreading::TFuture; + +struct TDescribeReplicationSettings: public TOperationRequestSettings { + using TSelf = TDescribeReplicationSettings; + FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); +}; + +struct TStaticCredentials { + std::string User; + std::string PasswordSecretName; +}; + +struct TOAuthCredentials { + std::string TokenSecretName; +}; + +class TConnectionParams: private TCommonClientSettings { +public: + enum class ECredentials { + Static, + OAuth, + }; + + explicit TConnectionParams(const Ydb::Replication::ConnectionParams& params); + + const std::string& GetDiscoveryEndpoint() const; + const std::string& GetDatabase() const; + bool GetEnableSsl() const; + + ECredentials GetCredentials() const; + const TStaticCredentials& GetStaticCredentials() const; + const TOAuthCredentials& GetOAuthCredentials() const; + +private: + std::variant< + TStaticCredentials, + TOAuthCredentials + > Credentials_; +}; + +struct TRowConsistency { +}; + +class TGlobalConsistency { +public: + explicit TGlobalConsistency(const Ydb::Replication::ConsistencyLevelGlobal& proto); + + const TDuration& GetCommitInterval() const; + +private: + TDuration CommitInterval_; +}; + +class TStats { +public: + TStats() = default; + TStats(const Ydb::Replication::DescribeReplicationResult_Stats& stats); + + const std::optional& GetLag() const; + const std::optional& GetInitialScanProgress() const; + +private: + std::optional Lag_; + std::optional InitialScanProgress_; +}; + +class TRunningState { +public: + TRunningState() = default; + explicit TRunningState(const TStats& stats); + + const TStats& GetStats() const; + +private: + TStats Stats_; +}; + +struct TDoneState {}; + +class TErrorState { + class TImpl; + +public: + explicit TErrorState(NYdb::NIssue::TIssues&& issues); + + const NYdb::NIssue::TIssues& GetIssues() const; + +private: + std::shared_ptr Impl_; +}; + +class TReplicationDescription { +public: + struct TItem { + uint64_t Id; + std::string SrcPath; + std::string DstPath; + TStats Stats; + std::optional SrcChangefeedName; + }; + + enum class EConsistencyLevel { + Row, + Global, + }; + + enum class EState { + Running, + Error, + Done, + }; + + explicit TReplicationDescription(const Ydb::Replication::DescribeReplicationResult& desc); + + const TConnectionParams& GetConnectionParams() const; + const std::vector GetItems() const; + + EConsistencyLevel GetConsistencyLevel() const; + const TGlobalConsistency& GetGlobalConsistency() const; + + EState GetState() const; + const TRunningState& GetRunningState() const; + const TErrorState& GetErrorState() const; + const TDoneState& GetDoneState() const; + +private: + TConnectionParams ConnectionParams_; + std::vector Items_; + + std::variant< + TRowConsistency, + TGlobalConsistency + > ConsistencyLevel_; + + std::variant< + TRunningState, + TErrorState, + TDoneState + > State_; +}; + +class TDescribeReplicationResult: public NScheme::TDescribePathResult { + friend class NYdb::V3::TProtoAccessor; + const Ydb::Replication::DescribeReplicationResult& GetProto() const; + +public: + TDescribeReplicationResult(TStatus&& status, Ydb::Replication::DescribeReplicationResult&& desc); + const TReplicationDescription& GetReplicationDescription() const; + +private: + TReplicationDescription ReplicationDescription_; + std::unique_ptr Proto_; +}; + +class TReplicationClient { + class TImpl; + +public: + TReplicationClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncDescribeReplicationResult DescribeReplication(const std::string& path, + const TDescribeReplicationSettings& settings = TDescribeReplicationSettings()); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NReplication diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_scripting.h new file mode 100644 index 000000000000..e4ec25b674b2 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_scripting.h @@ -0,0 +1,155 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { +namespace NScripting { + +class TExecuteYqlResult : public TStatus { +public: + TExecuteYqlResult(TStatus&& status, std::vector&& resultSets, + const std::optional& queryStats); + + const std::vector& GetResultSets() const; + TResultSet GetResultSet(size_t resultIndex) const; + + TResultSetParser GetResultSetParser(size_t resultIndex) const; + + const std::optional& GetStats() const; + +private: + std::vector ResultSets_; + std::optional QueryStats_; +}; + +class TYqlPartialResult { +public: + TYqlPartialResult(uint32_t resultSetIndex, TResultSet&& resultSet) + : ResultSetIndex_(resultSetIndex) + , ResultSet_(std::move(resultSet)) + {} + + uint32_t GetResultSetIndex() const { return ResultSetIndex_; } + const TResultSet& GetResultSet() const { return ResultSet_; } + +private: + uint32_t ResultSetIndex_; + TResultSet ResultSet_; +}; + +class TYqlResultPart : public TStreamPartStatus { +public: + bool HasPartialResult() const { return PartialResult_.has_value(); } + const TYqlPartialResult& GetPartialResult() const { return *PartialResult_; } + + bool HasQueryStats() const { return QueryStats_.has_value(); } + const NTable::TQueryStats& GetQueryStats() const { return *QueryStats_; } + NTable::TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } + + TYqlResultPart(TStatus&& status) + : TStreamPartStatus(std::move(status)) + {} + + TYqlResultPart(TStatus&& status, const std::optional &queryStats) + : TStreamPartStatus(std::move(status)) + , QueryStats_(queryStats) + {} + + TYqlResultPart(TStatus&& status, TYqlPartialResult&& partialResult, const std::optional &queryStats) + : TStreamPartStatus(std::move(status)) + , PartialResult_(std::move(partialResult)) + , QueryStats_(queryStats) + {} + +private: + std::optional PartialResult_; + std::optional QueryStats_; +}; + +using TAsyncYqlResultPart = NThreading::TFuture; + +class TYqlResultPartIterator : public TStatus { + friend class TScriptingClient; +public: + TAsyncYqlResultPart ReadNext(); + class TReaderImpl; +private: + TYqlResultPartIterator( + std::shared_ptr impl, + TPlainStatus&& status + ); + std::shared_ptr ReaderImpl_; +}; + +class TExplainYqlResult : public TStatus { +public: + TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan); + + std::map GetParameterTypes() const; + const std::string& GetPlan() const; + +private: + ::google::protobuf::Map ParameterTypes_; + std::string Plan_; +}; + +using TAsyncExecuteYqlResult = NThreading::TFuture; +using TAsyncYqlResultPartIterator = NThreading::TFuture; +using TAsyncExplainYqlResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +struct TExecuteYqlRequestSettings : public TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(Ydb::Query::Syntax, Syntax, Ydb::Query::SYNTAX_YQL_V1); + FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); +}; + +enum class ExplainYqlRequestMode { + // Parse = 1, + Validate = 2, + Plan = 3, +}; + +struct TExplainYqlRequestSettings : public TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TScriptingClient { + class TImpl; + +public: + TScriptingClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + //! Returns new params builder + TParamsBuilder GetParamsBuilder(); + + TAsyncExecuteYqlResult ExecuteYqlScript(const std::string& script, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncExecuteYqlResult ExecuteYqlScript(const std::string& script, const TParams& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncExecuteYqlResult ExecuteYqlScript(const std::string& script, TParams&& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const std::string& script, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const std::string& script, const TParams& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const std::string& script, TParams&& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncExplainYqlResult ExplainYqlScript(const std::string& script, + const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings()); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NScripting +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_view.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_view.h new file mode 100644 index 000000000000..7426b03252cf --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_view.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include + +namespace Ydb::View { + class DescribeViewResult; +} + +namespace NYdb::inline V3 { + class TProtoAccessor; +} + +namespace NYdb::inline V3::NView { + +class TDescribeViewResult; +using TAsyncDescribeViewResult = NThreading::TFuture; + +struct TDescribeViewSettings : public TOperationRequestSettings { + using TSelf = TDescribeViewSettings; +}; + +class TViewDescription { +public: + explicit TViewDescription(const Ydb::View::DescribeViewResult& desc); + + const std::string& GetQueryText() const; + +private: + std::string QueryText_; +}; + +class TDescribeViewResult : public NScheme::TDescribePathResult { + friend class NYdb::V3::TProtoAccessor; + const Ydb::View::DescribeViewResult& GetProto() const; + +public: + TDescribeViewResult(TStatus&& status, Ydb::View::DescribeViewResult&& desc); + TViewDescription GetViewDescription() const; + +private: + std::unique_ptr Proto_; +}; + +class TViewClient { + class TImpl; + +public: + TViewClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncDescribeViewResult DescribeView(const std::string& path, + const TDescribeViewSettings& settings = TDescribeViewSettings()); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NView diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h new file mode 100644 index 000000000000..a2fa24ca6250 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h @@ -0,0 +1,149 @@ +#pragma once + +#include "fwd.h" + +#include +#include +#include +#include +#include +#include + +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace NYdb::inline V3 { + +class TGRpcConnectionsImpl; + +//! Represents configuration of YDB driver +class TDriverConfig { + friend class TDriver; + +public: + //! Connection string format: ":///?database=", + //! where "://" can be "grpc://" or "grpcs://" or be absent, "" is endpoint, + //! "/?database=" is optional + TDriverConfig(const std::string& connectionString = ""); + //! Endpoint to initiate connections with Ydb cluster, + //! client will connect to others nodes according to client loadbalancing + TDriverConfig& SetEndpoint(const std::string& endpoint); + //! Set number of network threads, default: 2 + TDriverConfig& SetNetworkThreadsNum(size_t sz); + //! Set number of client pool threads, if 0 adaptive thread pool will be used. + //! NOTE: in case of no zero value it is possible to get deadlock if all threads + //! of this pool is blocked somewhere in user code. + //! default: 0 + TDriverConfig& SetClientThreadsNum(size_t sz); + //! Warning: not recommended to change + //! Set max number of queued responses. 0 - no limit + //! There is a queue to perform async calls to user code, + //! if this queue is full, attempts to enqueue responses inside sdk will be blocked + //! Size of this queue must be greater than max size of all requests inflight + //! Note: if this limit is reached network threads will be blocked. + //! Note: set of this limit can cause deadlock in some case of using async interface + //! This value doesn't make sense if SetClientThreadsNum is 0 + //! default: 0 + TDriverConfig& SetMaxClientQueueSize(size_t sz); + //! Enable Ssl. + //! caCerts - The buffer containing the PEM encoding of the server root certificates. + //! If this parameter is empty, the default roots will be used. + TDriverConfig& UseSecureConnection(const std::string& caCerts = std::string()); + TDriverConfig& UseClientCertificate(const std::string& clientCert, const std::string& clientPrivateKey); + //! Set token, this option can be overridden for client by ClientSettings + TDriverConfig& SetAuthToken(const std::string& token); + //! Set database, this option can be overridden for client by ClientSettings + TDriverConfig& SetDatabase(const std::string& database); + //! Set credentials data, this option can be overridden for client by ClientSettings + TDriverConfig& SetCredentialsProviderFactory(std::shared_ptr credentialsProviderFactory); + //! Set behaviour of discovery routine + //! See EDiscoveryMode enum comments + //! default: EDiscoveryMode::Sync + TDriverConfig& SetDiscoveryMode(EDiscoveryMode discoveryMode); + //! Max number of requests in queue waiting for discovery if "Async" mode chosen + //! default: 100 + TDriverConfig& SetMaxQueuedRequests(size_t sz); + //! Limit using of memory for grpc buffer pool. 0 means disabled. + //! If enabled the size must be greater than size of recieved message. + //! default: 0 + TDriverConfig& SetGrpcMemoryQuota(uint64_t bytes); + //! Specify tcp keep alive settings + //! This option allows to adjust tcp keep alive settings, useful to work + //! with balancers or to detect unexpected connectivity problem. + //! enable - if true enable tcp keep alive and use following settings + //! - if false disable tcp keep alive + //! idle - (Linux only) the interval between the last data packet sent and the first keepalive probe, sec + //! if zero use OS default + //! count - (Linux only) the number of unacknowledged probes to send before considering the connection dead + //! if zero use OS default + //! interval - (Linux only) the interval between subsequential keepalive probes, sec + //! if zero use OS default + //! NOTE: Please read OS documentation and investigate your network topology before touching this option. + //! default: true, 30, 5, 10 for linux, and true and OS default for others POSIX + TDriverConfig& SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval); + //! Enable or disable drain of client logic (e.g. session pool drain) during dtor call + TDriverConfig& SetDrainOnDtors(bool allowed); + //! Set policy for balancing + //! Params is a optionally field to set policy settings + //! default: EBalancingPolicy::UsePreferableLocation + TDriverConfig& SetBalancingPolicy(EBalancingPolicy policy, const std::string& params = std::string()); + //! !!! EXPERIMENTAL !!! + //! Set grpc level keep alive. If keepalive ping was delayed more than given timeout + //! internal grpc routine fails request with TRANSIENT_FAILURE or TRANSPORT_UNAVAILABLE error + //! Note: this timeout should not be too small to prevent fail due to + //! network buffers delay. I.e. values less than 5 seconds may cause request failure + //! even with fast network + //! default: disabled + TDriverConfig& SetGRpcKeepAliveTimeout(TDuration timeout); + TDriverConfig& SetGRpcKeepAlivePermitWithoutCalls(bool permitWithoutCalls); + //! Set inactive socket timeout. + //! Used to close connections, that were inactive for given time. + //! Closes unused connections every 1/10 of timeout, so deletion time is approximate. + //! Use TDuration::Max() to disable. + //! default: 6 minutes + TDriverConfig& SetSocketIdleTimeout(TDuration timeout); + + //! Set maximum incoming message size. + //! Note: this option overrides MaxMessageSize for incoming messages. + //! default: 0 + TDriverConfig& SetMaxInboundMessageSize(uint64_t maxInboundMessageSize); + //! Set maximum outgoing message size. + //! Note: this option overrides MaxMessageSize for outgoing messages. + //! default: 0 + TDriverConfig& SetMaxOutboundMessageSize(uint64_t maxOutboundMessageSize); + //! Note: if this option is unset, default 64_MB message size will be used. + //! default: 0 + TDriverConfig& SetMaxMessageSize(uint64_t maxMessageSize); + + //! Log backend. + TDriverConfig& SetLog(std::unique_ptr&& log); +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents connection pool to the database +class TDriver { + friend std::shared_ptr CreateInternalInterface(const TDriver); + +public: + TDriver(const TDriverConfig& config); + + //! Cancel all currently running and future requests + //! This method is useful to make sure there are no new asynchronous + //! callbacks and it is safe to destroy the driver + //! When wait is true this method will not return until the underlying + //! client thread pool is stopped completely + void Stop(bool wait = false); + + template + void AddExtension(typename TExtension::TParams params = typename TExtension::TParams()); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/fwd.h new file mode 100644 index 000000000000..6d0a1c888d5d --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/fwd.h @@ -0,0 +1,8 @@ +#pragma once + +namespace NYdb::inline V3 { + +class TDriver; +class TDriverConfig; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/export/export.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/export/export.h new file mode 100644 index 000000000000..2f87c880da17 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/export/export.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NExport { + +/// Common +enum class EExportProgress { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + Done = 3, + Cancellation = 4, + Cancelled = 5, + + Unknown = std::numeric_limits::max(), +}; + +struct TExportItemProgress { + uint32_t PartsTotal; + uint32_t PartsCompleted; + TInstant StartTime; + TInstant EndTime; +}; + +/// YT +struct TExportToYtSettings : public TOperationRequestSettings { + struct TItem { + std::string Src; + std::string Dst; + }; + + FLUENT_SETTING(std::string, Host); + FLUENT_SETTING_OPTIONAL(uint16_t, Port); + FLUENT_SETTING(std::string, Token); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(std::string, Description); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_DEFAULT(bool, UseTypeV3, false); +}; + +class TExportToYtResponse : public TOperation { +public: + struct TMetadata { + TExportToYtSettings Settings; + EExportProgress Progress; + std::vector ItemsProgress; + }; + +public: + using TOperation::TOperation; + TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +/// S3 +struct TExportToS3Settings : public TOperationRequestSettings, + public TS3Settings { + using TSelf = TExportToS3Settings; + + enum class EStorageClass { + NOT_SET = 0, + STANDARD = 1, + REDUCED_REDUNDANCY = 2, + STANDARD_IA = 3, + ONEZONE_IA = 4, + INTELLIGENT_TIERING = 5, + GLACIER = 6, + DEEP_ARCHIVE = 7, + OUTPOSTS = 8, + + UNKNOWN = std::numeric_limits::max(), + }; + + struct TItem { + std::string Src; + std::string Dst; + }; + + FLUENT_SETTING_DEFAULT(EStorageClass, StorageClass, EStorageClass::NOT_SET); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(std::string, Description); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(std::string, Compression); +}; + +class TExportToS3Response : public TOperation { +public: + struct TMetadata { + TExportToS3Settings Settings; + EExportProgress Progress; + std::vector ItemsProgress; + }; + +public: + using TOperation::TOperation; + TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +class TExportClient { + class TImpl; + +public: + TExportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + NThreading::TFuture ExportToYt(const TExportToYtSettings& settings); + NThreading::TFuture ExportToS3(const TExportToS3Settings& settings); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NExport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extension_common/extension.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extension_common/extension.h new file mode 100644 index 000000000000..23861eecdc7b --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extension_common/extension.h @@ -0,0 +1,72 @@ +#pragma once + +#include + +#include + +namespace Ydb { +namespace Discovery { + +class ListEndpointsResult; + +} +} + +namespace NYdb::inline V3 { + +class IExtensionApi { +public: + friend class TDriver; +public: + virtual ~IExtensionApi() = default; +private: + void SelfRegister(TDriver driver); +}; + +class IExtension { + friend class TDriver; +public: + virtual ~IExtension() = default; +private: + void SelfRegister(TDriver driver); +}; + +namespace NSdkStats { + +class IStatApi: public IExtensionApi { +public: + static IStatApi* Create(TDriver driver); +public: + virtual void SetMetricRegistry(::NMonitoring::IMetricRegistry* sensorsRegistry) = 0; + virtual void Accept(::NMonitoring::IMetricConsumer* consumer) const = 0; +}; + +class DestroyedClientException: public yexception {}; + +} // namespace NSdkStats + + +class IDiscoveryMutatorApi: public IExtensionApi { +public: + struct TAuxInfo { + std::string_view Database; + std::string_view DiscoveryEndpoint; + }; + using TMutatorCb = std::function; + + static IDiscoveryMutatorApi* Create(TDriver driver); +public: + virtual void SetMutatorCb(TMutatorCb&& mutator) = 0; +}; + + +template +void TDriver::AddExtension(typename TExtension::TParams params) { + typename TExtension::IApi* api = TExtension::IApi::Create(*this); + auto extension = new TExtension(params, api); + extension->SelfRegister(*this); + if (api) + api->SelfRegister(*this); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/discovery_mutator/discovery_mutator.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/discovery_mutator/discovery_mutator.h new file mode 100644 index 000000000000..d82248da25c0 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/discovery_mutator/discovery_mutator.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace NDiscoveryMutator::inline V3 { + +class TDiscoveryMutator: public NYdb::IExtension { +public: + using IApi = NYdb::IDiscoveryMutatorApi; + + using TCb = NYdb::IDiscoveryMutatorApi::TMutatorCb; + + class TParams { + friend class TDiscoveryMutator; + public: + TParams(TCb mutator) + : Mutator_(std::move(mutator)) + { } + + private: + TCb Mutator_; + }; + + TDiscoveryMutator(TParams params, IApi* api) { + api->SetMutatorCb(std::move(params.Mutator_)); + } +}; + +} // namespace NDiscoveryMutator diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_client.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_client.h new file mode 100644 index 000000000000..c2f1775f6a3a --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_client.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace NSolomonStatExtension::inline V3 { + +class TSolomonStatPullExtension: public NYdb::IExtension { +public: + using IApi = NYdb::NSdkStats::IStatApi; + + class TParams { + friend class TSolomonStatPullExtension; + + public: + TParams(const std::string& host + , ui16 port + , const std::string& project + , const std::string& service + , const std::string& cluster + , const std::vector>& labels = {}); + + NMonitoring::TLabels GetLabels() const; + + private: + const std::string Host_; + ui16 Port_; + NMonitoring::TLabels Labels_; + }; + + TSolomonStatPullExtension(const TParams& params, IApi* api); + ~TSolomonStatPullExtension(); + +private: + class TSolomonStatPage: public NMonitoring::IMonPage { + friend class TSolomonStatPullExtension; + public: + TSolomonStatPage(const std::string& title, const std::string& path, IApi* api); + + void Output(NMonitoring::IMonHttpRequest& request) override ; + + private: + IApi* Api_; + }; + +private: + std::shared_ptr MetricRegistry_; + NMonitoring::TMonService2 MonService_; + TIntrusivePtr Page_; +}; + +} // namespace NSolomonStatExtension diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_connector.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_connector.h new file mode 100644 index 000000000000..a33121063b93 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/extensions/solomon_stats/pull_connector.h @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include + +namespace NSolomonStatExtension::inline V3 { + +template +class TMetricRegistryConnector: public NYdb::IExtension { + static NMonitoring::IMetricRegistry* ToRawPtr(NMonitoring::IMetricRegistry* p) { + return p; + } + + static NMonitoring::IMetricRegistry* ToRawPtr(std::shared_ptr p) { + return p.get(); + } + + static NMonitoring::IMetricRegistry* ToRawPtr(TAtomicSharedPtr p) { + return p.Get(); + } + +public: + using IApi = NYdb::NSdkStats::IStatApi; + + class TParams : public TNonCopyable { + friend class TMetricRegistryConnector; + + public: + TParams(TMetricRegistryPtr registry) + : Registry(registry) + {} + + private: + TMetricRegistryPtr Registry; + }; + + TMetricRegistryConnector(const TParams& params, IApi* api) + : MetricRegistry_(params.Registry) + { + api->SetMetricRegistry(ToRawPtr(MetricRegistry_)); + } +private: + TMetricRegistryPtr MetricRegistry_; +}; + +inline void AddMetricRegistry(NYdb::TDriver& driver, NMonitoring::IMetricRegistry* ptr) { + if (!ptr) + return; + using TConnector = TMetricRegistryConnector; + + driver.AddExtension(TConnector::TParams(ptr)); +} + +inline void AddMetricRegistry(NYdb::TDriver& driver, std::shared_ptr ptr) { + if (!ptr) + return; + using TConnector = TMetricRegistryConnector>; + + driver.AddExtension(TConnector::TParams(ptr)); +} + +inline void AddMetricRegistry(NYdb::TDriver& driver, TAtomicSharedPtr ptr) { + if (!ptr) + return; + using TConnector = TMetricRegistryConnector>; + + driver.AddExtension(TConnector::TParams(ptr)); +} + +} // namespace NSolomonStatExtension diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h new file mode 100644 index 000000000000..429ce2b0f8b8 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h @@ -0,0 +1,553 @@ +#pragma once + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3::NFederatedTopic { + +using NTopic::TPrintable; +using TDbInfo = Ydb::FederationDiscovery::DatabaseInfo; + +using TSessionClosedEvent = NTopic::TSessionClosedEvent; + +//! Federated partition session. +struct TFederatedPartitionSession : public TThrRefBase, public TPrintable { + using TPtr = TIntrusivePtr; + +public: + TFederatedPartitionSession(const NTopic::TPartitionSession::TPtr& partitionSession, + std::shared_ptr db, + std::shared_ptr originDb = nullptr, + const std::string& originPath = "") + : PartitionSession(partitionSession) + , ReadSourceDatabase(std::move(db)) + , TopicOriginDatabase(originDb ? std::move(originDb) : ReadSourceDatabase) + , TopicOriginPath(!originPath.empty() ? std::move(originPath) : PartitionSession->GetTopicPath()) + { + Y_ABORT_UNLESS(ReadSourceDatabase); + } + + //! Request partition session status. + //! Result will come to TPartitionSessionStatusEvent. + void RequestStatus() { + return PartitionSession->RequestStatus(); + } + + //! + //! Properties. + //! + + //! Unique identifier of partition session. + //! It is unique within one read session. + ui64 GetPartitionSessionId() const { + return PartitionSession->GetPartitionSessionId(); + } + + //! Topic path. + const std::string& GetTopicPath() const { + return TopicOriginPath; + } + + //! Partition id. + ui64 GetPartitionId() const { + return PartitionSession->GetPartitionId(); + } + + const std::string& GetDatabaseName() const { + return GetTopicOriginDatabaseName(); + } + + const std::string& GetDatabasePath() const { + return GetTopicOriginDatabasePath(); + } + + const std::string& GetDatabaseId() const { + return GetTopicOriginDatabaseId(); + } + + const std::string& GetReadSourceDatabaseName() const { + return ReadSourceDatabase->name(); + } + + const std::string& GetReadSourceDatabasePath() const { + return ReadSourceDatabase->path(); + } + + const std::string& GetReadSourceDatabaseId() const { + return ReadSourceDatabase->id(); + } + + const std::string& GetTopicOriginDatabaseName() const { + return TopicOriginDatabase->name(); + } + + const std::string& GetTopicOriginDatabasePath() const { + return TopicOriginDatabase->path(); + } + + const std::string& GetTopicOriginDatabaseId() const { + return TopicOriginDatabase->id(); + } + +private: + NTopic::TPartitionSession::TPtr PartitionSession; + std::shared_ptr ReadSourceDatabase; + std::shared_ptr TopicOriginDatabase; + std::string TopicOriginPath; +}; + +//! Events for read session. +struct TReadSessionEvent { + class TFederatedPartitionSessionAccessor { + public: + explicit TFederatedPartitionSessionAccessor(TFederatedPartitionSession::TPtr partitionSession) + : FederatedPartitionSession(std::move(partitionSession)) + {} + + inline const TFederatedPartitionSession::TPtr GetFederatedPartitionSession() const { + return FederatedPartitionSession; + } + + protected: + TFederatedPartitionSession::TPtr FederatedPartitionSession; + }; + + template + struct TFederated : public TFederatedPartitionSessionAccessor, public TEvent, public TPrintable> { + using TPrintable>::DebugString; + + TFederated(TEvent event, TFederatedPartitionSession::TPtr federatedPartitionSession) + : TFederatedPartitionSessionAccessor(std::move(federatedPartitionSession)) + , TEvent(std::move(event)) + {} + + const NTopic::TPartitionSession::TPtr& GetPartitionSession() const override { + ythrow yexception() << "GetPartitionSession() method unavailable for federated objects, use GetFederatedPartitionSession() instead"; + } + }; + + using TCommitOffsetAcknowledgementEvent = TFederated; + using TStartPartitionSessionEvent = TFederated; + using TStopPartitionSessionEvent = TFederated; + using TEndPartitionSessionEvent = TFederated; + using TPartitionSessionStatusEvent = TFederated; + using TPartitionSessionClosedEvent = TFederated; + + struct TDataReceivedEvent : public NTopic::TReadSessionEvent::TPartitionSessionAccessor, public TFederatedPartitionSessionAccessor, public TPrintable { + using TMessage = TFederated; + using TCompressedMessage = TFederated; + + public: + TDataReceivedEvent(NTopic::TReadSessionEvent::TDataReceivedEvent event, TFederatedPartitionSession::TPtr federatedPartitionSession); + + const NTopic::TPartitionSession::TPtr& GetPartitionSession() const override { + ythrow yexception() << "GetPartitionSession method unavailable for federated objects, use GetFederatedPartitionSession instead"; + } + + bool HasCompressedMessages() const { + return !CompressedMessages.empty(); + } + + size_t GetMessagesCount() const { + return Messages.size() + CompressedMessages.size(); + } + + //! Get messages. + std::vector& GetMessages() { + CheckMessagesFilled(false); + return Messages; + } + + const std::vector& GetMessages() const { + CheckMessagesFilled(false); + return Messages; + } + + //! Get compressed messages. + std::vector& GetCompressedMessages() { + CheckMessagesFilled(true); + return CompressedMessages; + } + + const std::vector& GetCompressedMessages() const { + CheckMessagesFilled(true); + return CompressedMessages; + } + + //! Commits all messages in batch. + void Commit(); + + private: + void CheckMessagesFilled(bool compressed) const { + Y_ABORT_UNLESS(!Messages.empty() || !CompressedMessages.empty()); + if (compressed && CompressedMessages.empty()) { + ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session"; + } + if (!compressed && Messages.empty()) { + ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session"; + } + } + + private: + std::vector Messages; + std::vector CompressedMessages; + std::vector> OffsetRanges; + }; + + using TEvent = std::variant; +}; + +//! Set of offsets to commit. +//! Class that could store offsets in order to commit them later. +//! This class is not thread safe. +class TDeferredCommit { +public: + //! Add message to set. + void Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message); + + //! Add all messages from dataReceivedEvent to set. + void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); + + //! Add offsets range to set. + void Add(const TFederatedPartitionSession& partitionSession, ui64 startOffset, ui64 endOffset); + + //! Add offset to set. + void Add(const TFederatedPartitionSession& partitionSession, ui64 offset); + + //! Commit all added offsets. + void Commit(); + + TDeferredCommit(); + TDeferredCommit(const TDeferredCommit&) = delete; + TDeferredCommit(TDeferredCommit&&); + TDeferredCommit& operator=(const TDeferredCommit&) = delete; + TDeferredCommit& operator=(TDeferredCommit&&); + + ~TDeferredCommit(); + +private: + class TImpl; + std::unique_ptr Impl; +}; + +//! Event debug string. +std::string DebugString(const TReadSessionEvent::TEvent& event); + + +//! Settings for federated write session. +struct TFederatedWriteSessionSettings : public NTopic::TWriteSessionSettings { + using TSelf = TFederatedWriteSessionSettings; + + //! Preferred database + //! If specified database is unavailable, session will write to other database. + FLUENT_SETTING_OPTIONAL(std::string, PreferredDatabase); + + //! Write to other databases if there are problems with connection + //! to the preferred one. + FLUENT_SETTING_DEFAULT(bool, AllowFallback, true); + + TFederatedWriteSessionSettings() = default; + TFederatedWriteSessionSettings(const TFederatedWriteSessionSettings&) = default; + TFederatedWriteSessionSettings(TFederatedWriteSessionSettings&&) = default; + TFederatedWriteSessionSettings& operator=(const TFederatedWriteSessionSettings&) = default; + TFederatedWriteSessionSettings& operator=(TFederatedWriteSessionSettings&&) = default; + + TFederatedWriteSessionSettings(const std::string& path, const std::string& producerId, const std::string& messageGroupId) + : NTopic::TWriteSessionSettings(path, producerId, messageGroupId) { + } + + TFederatedWriteSessionSettings(const NTopic::TWriteSessionSettings& settings) + : NTopic::TWriteSessionSettings(settings) { + } + TFederatedWriteSessionSettings(NTopic::TWriteSessionSettings&& settings) + : NTopic::TWriteSessionSettings(std::move(settings)) { + } + // TFederatedWriteSessionSettings& operator=(const NTopic::TWriteSessionSettings&); + // TFederatedWriteSessionSettings& operator=(NTopic::TWriteSessionSettings&&); +}; + +//! Settings for read session. +struct TFederatedReadSessionSettings: public NTopic::TReadSessionSettings { + using TSelf = TFederatedReadSessionSettings; + + NTopic::TReadSessionSettings& EventHandlers(const TEventHandlers&) { + ythrow yexception() << "EventHandlers can not be set for federated session, use FederatedEventHandlers instead"; + } + + // Each handler, if set, is wrapped up and passed down to each subsession + struct TFederatedEventHandlers { + using TSelf = TFederatedEventHandlers; + + struct TSimpleDataHandlers { + std::function DataHandler; + bool CommitDataAfterProcessing; + bool GracefulStopAfterCommit; + }; + + //! Set simple handler with data processing and also + //! set other handlers with default behaviour. + //! They automatically commit data after processing + //! and confirm partition session events. + //! + //! Sets the following handlers: + //! DataReceivedHandler: sets DataReceivedHandler to handler that calls dataHandler and (if + //! commitDataAfterProcessing is set) then calls Commit(). CommitAcknowledgementHandler to handler that does + //! nothing. CreatePartitionSessionHandler to handler that confirms event. StopPartitionSessionHandler to + //! handler that confirms event. PartitionSessionStatusHandler to handler that does nothing. + //! PartitionSessionClosedHandler to handler that does nothing. + //! + //! dataHandler: handler of data event. + //! commitDataAfterProcessing: automatically commit data after calling of dataHandler. + //! gracefulReleaseAfterCommit: wait for commit acknowledgements for all inflight data before confirming + //! partition session destroy. + + TSimpleDataHandlers SimpleDataHandlers_; + + TSelf& SimpleDataHandlers(std::function dataHandler, + bool commitDataAfterProcessing = false, bool gracefulStopAfterCommit = true) { + SimpleDataHandlers_.DataHandler = std::move(dataHandler); + SimpleDataHandlers_.CommitDataAfterProcessing = commitDataAfterProcessing; + SimpleDataHandlers_.GracefulStopAfterCommit = gracefulStopAfterCommit; + return static_cast(*this); + } + + //! Data size limit for the DataReceivedHandler handler. + //! The data size may exceed this limit. + FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, std::numeric_limits::max()); + + //! Function to handle data events. + //! If this handler is set, data events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, DataReceivedHandler); + + //! Function to handle commit ack events. + //! If this handler is set, commit ack events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + CommitOffsetAcknowledgementHandler); + + //! Function to handle start partition session events. + //! If this handler is set, create partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + StartPartitionSessionHandler); + + //! Function to handle stop partition session events. + //! If this handler is set, destroy partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + StopPartitionSessionHandler); + + //! Function to handle end partition session events. + //! If this handler is set, end partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + EndPartitionSessionHandler); + + //! Function to handle partition session status events. + //! If this handler is set, partition session status events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + PartitionSessionStatusHandler); + + //! Function to handle partition session closed events. + //! If this handler is set, partition session closed events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + PartitionSessionClosedHandler); + + //! Function to handle session closed events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(NTopic::TSessionClosedHandler, SessionClosedHandler); + + //! Function to handle all event types. + //! If event with current type has no handler for this type of event, + //! this handler (if specified) will be used. + //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. + FLUENT_SETTING(std::function, CommonHandler); + + //! Executor for handlers. + //! If not set, default single threaded executor will be used. + //! Shared between subsessions + FLUENT_SETTING(NTopic::IExecutor::TPtr, HandlersExecutor); + }; + + //! Federated event handlers. + //! See description in TFederatedEventHandlers class. + FLUENT_SETTING(TFederatedEventHandlers, FederatedEventHandlers); + + + + //! Read policy settings + + //! Databases to read from. + //! Default (empty) value means reading from all available databases. + //! Adding duplicates or unavailable databases is okay, they will be ignored. + struct TReadOriginalSettings { + //! Add reading from specified database if it's available. + TReadOriginalSettings& AddDatabase(const std::string& database); + + //! Add reading from several specified databases, if available. + TReadOriginalSettings& AddDatabases(const std::vector& databases); + + //! Add reading from database(s) with the same location as client. + TReadOriginalSettings& AddLocal(); + + std::unordered_set Databases; + }; + +//! Default variant. + //! Read original topics specified in NTopic::TReadSessionSettings::Topics from databases, specified in settings. + //! Discards previously set ReadOriginal and ReadMirrored settings. + TSelf& ReadOriginal(TReadOriginalSettings settings); + + //! Read original and mirrored topics specified in NTopic::TReadSessionSettings::Topics + //! from one specified database. + //! Discards previously set ReadOriginal and ReadMirrored settings. + TSelf& ReadMirrored(const std::string& database); + + bool IsReadMirroredEnabled() { + return ReadMirroredEnabled; + } + + auto GetDatabasesToReadFrom() { + return DatabasesToReadFrom; + } + +private: + // Read policy settings, set via helpers above + bool ReadMirroredEnabled = false; + std::unordered_set DatabasesToReadFrom; +}; + + + +class IFederatedReadSession { +public: + //! Main reader loop. + //! Wait for next reader event. + virtual NThreading::TFuture WaitEvent() = 0; + + //! Main reader loop. + //! Get read session events. + //! Blocks until event occurs if "block" is set. + //! + //! maxEventsCount: maximum events count in batch. + //! maxByteSize: total size limit of data messages in batch. + //! block: block until event occurs. + //! + //! If maxEventsCount is not specified, + //! read session chooses event batch size automatically. + virtual std::vector + GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt, + size_t maxByteSize = std::numeric_limits::max()) = 0; + + //! Get single event. + virtual std::optional + GetEvent(bool block = false, size_t maxByteSize = std::numeric_limits::max()) = 0; + + //! Close read session. + //! Waits for all commit acknowledgments to arrive. + //! Force close after timeout. + //! This method is blocking. + //! When session is closed, + //! TSessionClosedEvent arrives. + virtual bool Close(TDuration timeout = TDuration::Max()) = 0; + + //! Reader counters with different stats (see TReaderConuters). + virtual NTopic::TReaderCounters::TPtr GetCounters() const = 0; + + //! Get unique identifier of read session. + virtual std::string GetSessionId() const = 0; + + virtual ~IFederatedReadSession() = default; +}; + +struct TFederatedTopicClientSettings : public TCommonClientSettingsBase { + using TSelf = TFederatedTopicClientSettings; + + //! Default executor for compression tasks. + FLUENT_SETTING_DEFAULT(NTopic::IExecutor::TPtr, DefaultCompressionExecutor, NTopic::CreateThreadPoolExecutor(2)); + + //! Default executor for callbacks. + FLUENT_SETTING_DEFAULT(NTopic::IExecutor::TPtr, DefaultHandlersExecutor, NTopic::CreateThreadPoolExecutor(1)); + + //! Connection timeoout for federation discovery. + FLUENT_SETTING_DEFAULT(TDuration, ConnectionTimeout, TDuration::Seconds(30)); + + //! Retry policy enables automatic retries for non-fatal errors. + FLUENT_SETTING_DEFAULT(NTopic::IRetryPolicy::TPtr, RetryPolicy, NTopic::IRetryPolicy::GetDefaultPolicy()); +}; + +class TFederatedTopicClient { +public: + class TImpl; + + // executors from settings are passed to subclients + TFederatedTopicClient(const TDriver& driver, const TFederatedTopicClientSettings& settings = {}); + + void ProvideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl); + + //! Create read session. + std::shared_ptr CreateReadSession(const TFederatedReadSessionSettings& settings); + + //! Create write session. + // std::shared_ptr CreateSimpleBlockingWriteSession(const TFederatedWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TFederatedWriteSessionSettings& settings); + +protected: + void OverrideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::NFederatedTopic + +namespace NYdb::inline V3::NTopic { + +using namespace NFederatedTopic; + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; +template<> +void TPrintable>::DebugString(TStringBuilder& res, bool) const; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/helpers/helpers.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/helpers/helpers.h new file mode 100644 index 000000000000..94357086de38 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/helpers/helpers.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +//! Checks the following environment variables and creates TDriverConfig with the first appeared: +//! YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS= — service account key file, +//! YDB_ANONYMOUS_CREDENTIALS="1" — uses anonymous access (used for test installation), +//! YDB_METADATA_CREDENTIALS="1" — uses metadata service, +//! YDB_ACCESS_TOKEN_CREDENTIALS= — access token (for example, IAM-token). +//! YDB_OAUTH2_KEY_FILE= - OAuth 2.0 RFC8693 token exchange credentials parameters json file +//! If grpcs protocol is given in endpoint (or protocol is empty), enables SSL and uses +//! certificate from resourses and user cert from env variable "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" +TDriverConfig CreateFromEnvironment(const std::string& connectionString = ""); +//! Creates TDriverConfig from cloud service account key file +TDriverConfig CreateFromSaKeyFile(const std::string& saKeyFile, const std::string& connectionString = ""); + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt new file mode 100644 index 000000000000..667ae4bd632c --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt @@ -0,0 +1,10 @@ +_ydb_sdk_add_library(client-iam-types INTERFACE) + +target_link_libraries(client-iam-types + INTERFACE + client-ydb_types-credentials + library-jwt + yutil +) + +_ydb_sdk_install_targets(client-iam-types) diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/types.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/types.h new file mode 100644 index 000000000000..8f5fe3dac67b --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/types.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +namespace NYdb { +namespace NIam { + +constexpr std::string_view DEFAULT_ENDPOINT = "iam.api.cloud.yandex.net"; +constexpr bool DEFAULT_ENABLE_SSL = true; + +constexpr std::string_view DEFAULT_HOST = "169.254.169.254"; +constexpr uint32_t DEFAULT_PORT = 80; + +constexpr TDuration DEFAULT_REFRESH_PERIOD = TDuration::Hours(1); +constexpr TDuration DEFAULT_REQUEST_TIMEOUT = TDuration::Seconds(10); + +} + +struct TIamHost { + std::string Host = std::string(NIam::DEFAULT_HOST); + uint32_t Port = NIam::DEFAULT_PORT; + TDuration RefreshPeriod = NIam::DEFAULT_REFRESH_PERIOD; +}; + +struct TIamEndpoint { + std::string Endpoint = std::string(NIam::DEFAULT_ENDPOINT); + TDuration RefreshPeriod = NIam::DEFAULT_REFRESH_PERIOD; + TDuration RequestTimeout = NIam::DEFAULT_REQUEST_TIMEOUT; + bool EnableSsl = NIam::DEFAULT_ENABLE_SSL; +}; + +struct TIamJwtFilename : TIamEndpoint { std::string JwtFilename; }; + +struct TIamJwtContent : TIamEndpoint { std::string JwtContent; }; + +struct TIamJwtParams : TIamEndpoint { TJwtParams JwtParams; }; + +struct TIamOAuth : TIamEndpoint { std::string OAuthToken; }; + + +inline TJwtParams ReadJwtKeyFile(const std::string& filename) { + std::ifstream input(filename, std::ios::in); + return ParseJwtParams({std::istreambuf_iterator(input), std::istreambuf_iterator()}); +} + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/ya.make b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/ya.make new file mode 100644 index 000000000000..c21c0cf9999c --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + types.h +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/jwt + ydb/public/sdk/cpp/src/client/types/credentials +) + +END() diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/iam.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/iam.h new file mode 100644 index 000000000000..81b41068ca40 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/iam.h @@ -0,0 +1,19 @@ +#pragma once + +#include "common/types.h" + +namespace NYdb::inline V3 { + +/// Acquire an IAM token using a local metadata service on a virtual machine. +TCredentialsProviderFactoryPtr CreateIamCredentialsProviderFactory(const TIamHost& params = {}); + +/// Acquire an IAM token using a JSON Web Token (JWT) file name. +TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactory(const TIamJwtFilename& params); + +/// Acquire an IAM token using JSON Web Token (JWT) contents. +TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactory(const TIamJwtContent& param); + +// Acquire an IAM token using a user OAuth token. +TCredentialsProviderFactoryPtr CreateIamOAuthCredentialsProviderFactory(const TIamOAuth& params); + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam_private/iam.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam_private/iam.h new file mode 100644 index 000000000000..c4373c9478af --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam_private/iam.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +/// Acquire an IAM token using a JSON Web Token (JWT) file name. +TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactoryPrivate(const TIamJwtFilename& params); + +/// Acquire an IAM token using JSON Web Token (JWT) contents. +TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactoryPrivate(const TIamJwtContent& param); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h new file mode 100644 index 000000000000..3e6b191cbf2c --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NImport { + +/// Common +enum class EImportProgress { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + BuildIndexes = 3, + Done = 4, + Cancellation = 5, + Cancelled = 6, + + Unknown = std::numeric_limits::max(), +}; + +struct TImportItemProgress { + uint32_t PartsTotal; + uint32_t PartsCompleted; + TInstant StartTime; + TInstant EndTime; +}; + +/// S3 +struct TImportFromS3Settings : public TOperationRequestSettings, + public TS3Settings { + using TSelf = TImportFromS3Settings; + + struct TItem { + std::string Src; + std::string Dst; + }; + + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(std::string, Description); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(bool, NoACL); + FLUENT_SETTING_OPTIONAL(bool, SkipChecksumValidation); +}; + +class TImportFromS3Response : public TOperation { +public: + struct TMetadata { + TImportFromS3Settings Settings; + EImportProgress Progress; + std::vector ItemsProgress; + }; + +public: + using TOperation::TOperation; + TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +/// Data +struct TImportYdbDumpDataSettings : public TOperationRequestSettings { + using TSelf = TImportYdbDumpDataSettings; + + FLUENT_SETTING_VECTOR(std::string, Columns); + + using TOperationRequestSettings::TOperationRequestSettings; +}; + +class TImportDataResult : public TStatus { +public: + explicit TImportDataResult(TStatus&& status); +}; + +using TAsyncImportDataResult = NThreading::TFuture; + +class TImportClient { + class TImpl; + +public: + TImportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + NThreading::TFuture ImportFromS3(const TImportFromS3Settings& settings); + + // ydb dump format + TAsyncImportDataResult ImportData(const std::string& table, std::string&& data, const TImportYdbDumpDataSettings& settings); + TAsyncImportDataResult ImportData(const std::string& table, const std::string& data, const TImportYdbDumpDataSettings& settings); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NImport +} // namespace NYdb + +template<> +inline void Out(IOutputStream& o, const NYdb::NImport::TImportFromS3Response& x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/monitoring/monitoring.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/monitoring/monitoring.h new file mode 100644 index 000000000000..f8dfc2799416 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/monitoring/monitoring.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +namespace Ydb { +namespace Monitoring { + class SelfCheckResult; +} +} + +namespace NYdb::inline V3 { + +class TProtoAccessor; + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////// + +enum class EStatusFlag { + UNSPECIFIED = 0, + GREY = 1, + GREEN = 2, + BLUE = 3, + YELLOW = 4, + ORANGE = 5, + RED = 6, +}; + +struct TSelfCheckSettings : public TOperationRequestSettings{ + FLUENT_SETTING_OPTIONAL(bool, ReturnVerboseStatus); + FLUENT_SETTING_OPTIONAL(EStatusFlag, MinimumStatus); + FLUENT_SETTING_OPTIONAL(uint32_t, MaximumLevel); +}; + +class TSelfCheckResult : public TStatus { + friend class NYdb::V3::TProtoAccessor; +public: + TSelfCheckResult(TStatus&& status, Ydb::Monitoring::SelfCheckResult&& result); +private: + class TImpl; + std::shared_ptr Impl_; +}; + +using TAsyncSelfCheckResult = NThreading::TFuture; + +class TMonitoringClient { + class TImpl; + +public: + TMonitoringClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncSelfCheckResult SelfCheck(const TSelfCheckSettings& settings = TSelfCheckSettings()); +private: + std::shared_ptr Impl_; +}; + +} +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h new file mode 100644 index 000000000000..6416f6cc81b2 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { +namespace NOperation { + +template +class TOperationsList : public TStatus { +public: + TOperationsList(TStatus&& status) + : TStatus(std::move(status)) + { + } + + TOperationsList(TStatus&& status, std::vector&& operations, const std::string& nextPageToken) + : TStatus(std::move(status)) + , Operations_(std::move(operations)) + , NextPageToken_(nextPageToken) + { + } + + const std::vector& GetList() const { return Operations_; } + const std::string& NextPageToken() const { return NextPageToken_; } + + std::string ToJsonString() const { + std::stringstream json; + json << "{\"operations\":["; + + bool first = true; + for (const auto& operation : GetList()) { + json << (first ? "" : ",") << operation.ToJsonString(); + first = false; + } + + json << "],\"nextPageToken\":\"" << NextPageToken() << "\"}"; + return json.str(); + } + +private: + std::vector Operations_; + std::string NextPageToken_; +}; + +class TOperationClient { + class TImpl; + +public: + TOperationClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + template + NThreading::TFuture Get(const TOperation::TOperationId& id); + + TAsyncStatus Cancel(const TOperation::TOperationId& id); + TAsyncStatus Forget(const TOperation::TOperationId& id); + + template + NThreading::TFuture> List(ui64 pageSize = 0, const std::string& pageToken = std::string()); + +private: + template + NThreading::TFuture> List(const std::string& kind, ui64 pageSize, const std::string& pageToken); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NOperation +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/fwd.h new file mode 100644 index 000000000000..4d6af5b0829d --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/fwd.h @@ -0,0 +1,9 @@ +#pragma once + +namespace NYdb::inline V3 { + +class TParams; +class TParamValueBuilder; +class TParamsBuilder; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/params.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/params.h new file mode 100644 index 000000000000..89a30d0ce687 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/params.h @@ -0,0 +1,99 @@ +#pragma once + +#include "fwd.h" + +#include +#include + +#include + +namespace Ydb { + class TypedValue; +} + +namespace NYdb::inline V3 { + +namespace NScripting { + class TScriptingClient; +} + +class TProtoAccessor; + +namespace NTable { + class TTableClient; + class TSession; + class TDataQuery; +} + +namespace NExperimental { + class TStreamQueryClient; +} + +namespace NQuery { + class TExecQueryImpl; + class TQueryClient; +} + +class TParams { + friend class TParamsBuilder; + friend class NTable::TTableClient; + friend class NTable::TSession; + friend class NTable::TDataQuery; + friend class NScripting::TScriptingClient; + friend class NExperimental::TStreamQueryClient; + friend class NQuery::TExecQueryImpl; + friend class NQuery::TQueryClient; + friend class NYdb::V3::TProtoAccessor; +public: + bool Empty() const; + + std::map GetValues() const; + std::optional GetValue(const std::string& name) const; + +private: + TParams(::google::protobuf::Map&& protoMap); + + ::google::protobuf::Map* GetProtoMapPtr(); + const ::google::protobuf::Map& GetProtoMap() const; + + class TImpl; + std::shared_ptr Impl_; +}; + +class TParamValueBuilder : public TValueBuilderBase { + friend class TParamsBuilder; +public: + TParamsBuilder& Build(); + bool Finished(); + +private: + TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto); + + TParamsBuilder& Owner_; + bool Finished_; +}; + +class TParamsBuilder : public TMoveOnly { + friend class NTable::TDataQuery; +public: + TParamsBuilder(TParamsBuilder&&); + TParamsBuilder(); + TParamsBuilder(const std::map& typeInfo); + + ~TParamsBuilder(); + + TParamValueBuilder& AddParam(const std::string& name); + TParamsBuilder& AddParam(const std::string& name, const TValue& value); + + bool HasTypeInfo() const; + + TParams Build(); + +private: + TParamsBuilder(const ::google::protobuf::Map& typeInfo); + + class TImpl; + std::unique_ptr Impl_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h new file mode 100644 index 000000000000..7d928dedc280 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NYdb::inline V3 { + +class TResultSet; +class TValue; +class TType; +class TParams; + +namespace NTable { +class TTableDescription; +class TIndexDescription; +} + +//! Provides access to raw protobuf values of YDB API entities. It is not recommended to use this +//! class in client applications as it add dependency on API protobuf format which is subject to +//! change. Use functionality provided by YDB SDK classes. +class TProtoAccessor { +public: + static const Ydb::Type& GetProto(const TType& type); + static const Ydb::Value& GetProto(const TValue& value); + static const Ydb::ResultSet& GetProto(const TResultSet& resultSet); + static const ::google::protobuf::Map& GetProtoMap(const TParams& params); + static ::google::protobuf::Map* GetProtoMapPtr(TParams& params); + static const Ydb::TableStats::QueryStats& GetProto(const NTable::TQueryStats& queryStats); + static const Ydb::Table::DescribeTableResult& GetProto(const NTable::TTableDescription& tableDescription); + static const Ydb::Topic::DescribeTopicResult& GetProto(const NYdb::NTopic::TTopicDescription& topicDescription); + static const Ydb::Topic::DescribeConsumerResult& GetProto(const NYdb::NTopic::TConsumerDescription& consumerDescription); + static const Ydb::Monitoring::SelfCheckResult& GetProto(const NYdb::NMonitoring::TSelfCheckResult& selfCheckResult); + static const Ydb::Coordination::DescribeNodeResult& GetProto(const NYdb::NCoordination::TNodeDescription &describeNodeResult); + static const Ydb::Replication::DescribeReplicationResult& GetProto(const NYdb::NReplication::TDescribeReplicationResult& desc); + static const Ydb::View::DescribeViewResult& GetProto(const NYdb::NView::TDescribeViewResult& desc); + + static NTable::TQueryStats FromProto(const Ydb::TableStats::QueryStats& queryStats); + static NTable::TTableDescription FromProto(const Ydb::Table::CreateTableRequest& request); + static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndex& tableIndex); + static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc); + + static NTable::TChangefeedDescription FromProto(const Ydb::Table::Changefeed& changefeed); + static NTable::TChangefeedDescription FromProto(const Ydb::Table::ChangefeedDescription& changefeed); + + static Ydb::Table::ValueSinceUnixEpochModeSettings::Unit GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value); + static NTable::TValueSinceUnixEpochModeSettings::EUnit FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value); + + static Ydb::Topic::MeteringMode GetProto(NTopic::EMeteringMode mode); + static NTopic::EMeteringMode FromProto(Ydb::Topic::MeteringMode mode); + + // exports & imports + template static typename TProtoSettings::Scheme GetProto(ES3Scheme value); + template static ES3Scheme FromProto(typename TProtoSettings::Scheme value); + static Ydb::Export::ExportToS3Settings::StorageClass GetProto(NExport::TExportToS3Settings::EStorageClass value); + static NExport::TExportToS3Settings::EStorageClass FromProto(Ydb::Export::ExportToS3Settings::StorageClass value); + static NExport::EExportProgress FromProto(Ydb::Export::ExportProgress::Progress value); + static NImport::EImportProgress FromProto(Ydb::Import::ImportProgress::Progress value); +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h new file mode 100644 index 000000000000..980bf2b07926 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h @@ -0,0 +1,266 @@ +#pragma once + +#include "fwd.h" + +#include "query.h" +#include "tx.h" + +#include +#include +#include +#include + +namespace NYdb::inline V3 { + class TProtoAccessor; + + namespace NRetry::Async { + template + class TRetryContext; + } // namespace NRetry::Async + namespace NRetry::Sync { + template + class TRetryContext; + } // namespace NRetry::Sync +} + +namespace NYdb::inline V3::NQuery { + +struct TCreateSessionSettings : public TSimpleRequestSettings { + TCreateSessionSettings(); +}; + +using TAsyncCreateSessionResult = NThreading::TFuture; +using TRetryOperationSettings = NYdb::NRetry::TRetryOperationSettings; + +struct TSessionPoolSettings { + using TSelf = TSessionPoolSettings; + + // Max number of sessions client can get from session pool + FLUENT_SETTING_DEFAULT(uint32_t, MaxActiveSessions, 50); + + // Max time session to be in idle state before closing + FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); + + // Min number of session in session pool. + // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. + FLUENT_SETTING_DEFAULT(uint32_t, MinPoolSize, 10); +}; + +struct TClientSettings : public TCommonClientSettingsBase { + using TSessionPoolSettings = TSessionPoolSettings; + using TSelf = TClientSettings; + FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); +}; + +// ! WARNING: Experimental API +// ! This API is currently in experimental state and is a subject for changes. +// ! No backward and/or forward compatibility guarantees are provided. +// ! DO NOT USE for production workloads. +class TQueryClient { + friend class TSession; + friend class NRetry::Async::TRetryContext; + friend class NRetry::Async::TRetryContext; + friend class NRetry::Sync::TRetryContext; + +public: + using TQueryResultFunc = std::function; + using TQueryFunc = std::function; + using TQuerySyncFunc = std::function; + using TQueryWithoutSessionFunc = std::function; + using TQueryWithoutSessionSyncFunc = std::function; + using TSettings = TClientSettings; + using TSession = TSession; + using TCreateSessionSettings = TCreateSessionSettings; + using TAsyncCreateSessionResult = TAsyncCreateSessionResult; + +public: + TQueryClient(const TDriver& driver, const TClientSettings& settings = TClientSettings()); + + TAsyncExecuteQueryResult ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryResult ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryResult RetryQuery(TQueryResultFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TAsyncStatus RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TAsyncStatus RetryQuery(TQueryWithoutSessionFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TStatus RetryQuerySync(const TQuerySyncFunc& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TStatus RetryQuerySync(const TQueryWithoutSessionSyncFunc& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TAsyncExecuteQueryResult RetryQuery(const std::string& query, const TTxControl& txControl, + TDuration timeout, bool isIndempotent); + + NThreading::TFuture ExecuteScript(const std::string& script, + const TExecuteScriptSettings& settings = TExecuteScriptSettings()); + + NThreading::TFuture ExecuteScript(const std::string& script, + const TParams& params, const TExecuteScriptSettings& settings = TExecuteScriptSettings()); + + TAsyncFetchScriptResultsResult FetchScriptResults(const NKikimr::NOperationId::TOperationId& operationId, int64_t resultSetIndex, + const TFetchScriptResultsSettings& settings = TFetchScriptResultsSettings()); + + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + + //! Returns number of active sessions given via session pool + int64_t GetActiveSessionCount() const; + + //! Returns the maximum number of sessions in session pool + int64_t GetActiveSessionsLimit() const; + + //! Returns the size of session pool + int64_t GetCurrentPoolSize() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +class TSession { + friend class TQueryClient; + friend class TTransaction; + friend class TExecuteQueryIterator; +public: + const std::string& GetId() const; + + TAsyncExecuteQueryResult ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryResult ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings = TExecuteQuerySettings()); + + TAsyncBeginTransactionResult BeginTransaction(const TTxSettings& txSettings, + const TBeginTxSettings& settings = TBeginTxSettings()); + + class TImpl; +private: + TSession(); + TSession(std::shared_ptr client, TSession::TImpl* sessionImpl); + + std::shared_ptr Client_; + std::shared_ptr SessionImpl_; +}; + +class TCreateSessionResult: public TStatus { + friend class TSession::TImpl; +public: + TCreateSessionResult(TStatus&& status, TSession&& session); + TSession GetSession() const; + +private: + TSession Session_; +}; + +class TTransaction { + friend class TQueryClient; + friend class TExecuteQueryIterator::TReaderImpl; +public: + const std::string& GetId() const { + return TxId_; + } + + bool IsActive() const { + return !TxId_.empty(); + } + + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); + TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); + + TSession GetSession() const { + return Session_; + } + +private: + TTransaction(const TSession& session, const std::string& txId); + + TSession Session_; + std::string TxId_; +}; + +class TBeginTransactionResult : public TStatus { +public: + TBeginTransactionResult(TStatus&& status, TTransaction transaction); + + const TTransaction& GetTransaction() const; + +private: + TTransaction Transaction_; +}; + +class TExecuteQueryPart : public TStreamPartStatus { +public: + bool HasResultSet() const { return ResultSet_.has_value(); } + uint64_t GetResultSetIndex() const { return ResultSetIndex_; } + const TResultSet& GetResultSet() const { return *ResultSet_; } + TResultSet ExtractResultSet() { return std::move(*ResultSet_); } + + const std::optional& GetStats() const { return Stats_; } + const std::optional& GetTransaction() const { return Transaction_; } + + TExecuteQueryPart(TStatus&& status, std::optional&& queryStats, std::optional&& tx) + : TStreamPartStatus(std::move(status)) + , Stats_(std::move(queryStats)) + , Transaction_(std::move(tx)) + {} + + TExecuteQueryPart(TStatus&& status, TResultSet&& resultSet, int64_t resultSetIndex, + std::optional&& queryStats, std::optional&& tx) + : TStreamPartStatus(std::move(status)) + , ResultSet_(std::move(resultSet)) + , ResultSetIndex_(resultSetIndex) + , Stats_(std::move(queryStats)) + , Transaction_(std::move(tx)) + {} + +private: + std::optional ResultSet_; + int64_t ResultSetIndex_ = 0; + std::optional Stats_; + std::optional Transaction_; +}; + +class TExecuteQueryResult : public TStatus { +public: + const std::vector& GetResultSets() const; + TResultSet GetResultSet(size_t resultIndex) const; + TResultSetParser GetResultSetParser(size_t resultIndex) const; + + const std::optional& GetStats() const { return Stats_; } + + std::optional GetTransaction() const {return Transaction_; } + + TExecuteQueryResult(TStatus&& status) + : TStatus(std::move(status)) + {} + + TExecuteQueryResult(TStatus&& status, std::vector&& resultSets, + std::optional&& stats, std::optional&& tx) + : TStatus(std::move(status)) + , ResultSets_(std::move(resultSets)) + , Stats_(std::move(stats)) + , Transaction_(std::move(tx)) + {} + +private: + std::vector ResultSets_; + std::optional Stats_; + std::optional Transaction_; +}; + +} // namespace NYdb::V3::NQuery diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/fwd.h new file mode 100644 index 000000000000..08b7f4e67521 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/fwd.h @@ -0,0 +1,37 @@ +#pragma once + +namespace NYdb::inline V3::NQuery { + +struct TClientSettings; +struct TSessionPoolSettings; +struct TCreateSessionSettings; +struct TExecuteQuerySettings; +struct TBeginTxSettings; +struct TCommitTxSettings; +struct TRollbackTxSettings; +struct TExecuteScriptSettings; +struct TFetchScriptResultsSettings; +struct TTxOnlineSettings; +struct TTxSettings; + +class TQueryClient; +class TSession; + +class TCreateSessionResult; +class TBeginTransactionResult; +class TExecuteQueryResult; +class TCommitTransactionResult; +class TFetchScriptResultsResult; + +class TExecuteQueryPart; +class TExecuteQueryIterator; + +class TTransaction; +struct TTxControl; + +class TQueryContent; +class TResultSetMeta; +class TScriptExecutionOperation; +class TExecStats; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h new file mode 100644 index 000000000000..4e3616a43695 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h @@ -0,0 +1,187 @@ +#pragma once + +#include "fwd.h" + +#include "stats.h" + +#include +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NQuery { + +enum class ESyntax { + Unspecified = 0, + YqlV1 = 1, // YQL + Pg = 2, // PostgresQL +}; + +enum class EExecMode { + Unspecified = 0, + Parse = 10, + Validate = 20, + Explain = 30, + Execute = 50, +}; + +enum class EStatsMode { + Unspecified = 0, + None = 10, + Basic = 20, + Full = 30, + Profile = 40, +}; + +std::optional ParseStatsMode(std::string_view statsMode); +std::string_view StatsModeToString(const EStatsMode statsMode); + +enum class EExecStatus { + Unspecified = 0, + Starting = 10, + Aborted = 20, + Canceled = 30, + Completed = 40, + Failed = 50, +}; + +using TAsyncExecuteQueryPart = NThreading::TFuture; + +class TExecuteQueryIterator : public TStatus { + friend class TExecQueryImpl; +public: + class TReaderImpl; + + TAsyncExecuteQueryPart ReadNext(); + +private: + TExecuteQueryIterator( + std::shared_ptr impl, + TPlainStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) {} + + std::shared_ptr ReaderImpl_; +}; + +using TAsyncExecuteQueryIterator = NThreading::TFuture; + +struct TExecuteQuerySettings : public TRequestSettings { + FLUENT_SETTING_OPTIONAL(uint32_t, OutputChunkMaxSize); + FLUENT_SETTING_DEFAULT(ESyntax, Syntax, ESyntax::YqlV1); + FLUENT_SETTING_DEFAULT(EExecMode, ExecMode, EExecMode::Execute); + FLUENT_SETTING_DEFAULT(EStatsMode, StatsMode, EStatsMode::None); + FLUENT_SETTING_OPTIONAL(bool, ConcurrentResultSets); + FLUENT_SETTING(std::string, ResourcePool); +}; + +struct TBeginTxSettings : public TRequestSettings {}; +struct TCommitTxSettings : public TRequestSettings {}; +struct TRollbackTxSettings : public TRequestSettings {}; + + + +class TCommitTransactionResult : public TStatus { +public: + TCommitTransactionResult(TStatus&& status); +}; + +using TAsyncBeginTransactionResult = NThreading::TFuture; +using TAsyncCommitTransactionResult = NThreading::TFuture; + +struct TExecuteScriptSettings : public TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(ESyntax, Syntax, ESyntax::YqlV1); + FLUENT_SETTING_DEFAULT(EExecMode, ExecMode, EExecMode::Execute); + FLUENT_SETTING_DEFAULT(EStatsMode, StatsMode, EStatsMode::None); + FLUENT_SETTING(TDuration, ResultsTtl); + FLUENT_SETTING(std::string, ResourcePool); +}; + +class TQueryContent { +public: + TQueryContent() = default; + + TQueryContent(const std::string& text, ESyntax syntax) + : Text(text) + , Syntax(syntax) + {} + + std::string Text; + ESyntax Syntax = ESyntax::Unspecified; +}; + +class TResultSetMeta { +public: + TResultSetMeta() = default; + + explicit TResultSetMeta(const std::vector& columns) + : Columns(columns) + {} + + explicit TResultSetMeta(std::vector&& columns) + : Columns(std::move(columns)) + {} + + std::vector Columns; +}; + +class TScriptExecutionOperation : public TOperation { +public: + struct TMetadata { + std::string ExecutionId; + EExecStatus ExecStatus = EExecStatus::Unspecified; + EExecMode ExecMode = EExecMode::Unspecified; + + TQueryContent ScriptContent; + TExecStats ExecStats; + std::vector ResultSetsMeta; + }; + + using TOperation::TOperation; + TScriptExecutionOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const { + return Metadata_; + } + +private: + TMetadata Metadata_; +}; + +struct TFetchScriptResultsSettings : public TRequestSettings { + FLUENT_SETTING(std::string, FetchToken); + FLUENT_SETTING_DEFAULT(uint64_t, RowsLimit, 1000); +}; + +class TFetchScriptResultsResult : public TStatus { +public: + bool HasResultSet() const { return ResultSet_.has_value(); } + uint64_t GetResultSetIndex() const { return ResultSetIndex_; } + const TResultSet& GetResultSet() const { return *ResultSet_; } + TResultSet ExtractResultSet() { return std::move(*ResultSet_); } + const std::string& GetNextFetchToken() const { return NextFetchToken_; } + + explicit TFetchScriptResultsResult(TStatus&& status) + : TStatus(std::move(status)) + {} + + TFetchScriptResultsResult(TStatus&& status, TResultSet&& resultSet, int64_t resultSetIndex, const std::string& nextFetchToken) + : TStatus(std::move(status)) + , ResultSet_(std::move(resultSet)) + , ResultSetIndex_(resultSetIndex) + , NextFetchToken_(nextFetchToken) + {} + +private: + std::optional ResultSet_; + int64_t ResultSetIndex_ = 0; + std::string NextFetchToken_; +}; + +using TAsyncFetchScriptResultsResult = NThreading::TFuture; +using TAsyncExecuteQueryResult = NThreading::TFuture; + +} // namespace NYdb::V3::NQuery diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/stats.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/stats.h new file mode 100644 index 000000000000..902478b44643 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/stats.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include +#include +#include + +namespace Ydb::TableStats { + class QueryStats; +} + +namespace NYdb::inline V3 { + class TProtoAccessor; +} + +namespace NYdb::inline V3::NQuery { + +class TExecStats { + friend class NYdb::V3::TProtoAccessor; + +public: + TExecStats() = default; + + explicit TExecStats(Ydb::TableStats::QueryStats&& proto); + explicit TExecStats(const Ydb::TableStats::QueryStats& proto); + + std::string ToString(bool withPlan = false) const; + + std::optional GetPlan() const; + std::optional GetAst() const; + + TDuration GetTotalDuration() const; + TDuration GetTotalCpuTime() const; + +private: + const Ydb::TableStats::QueryStats& GetProto() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NQuery diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/tx.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/tx.h new file mode 100644 index 000000000000..f435bc8fe450 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/tx.h @@ -0,0 +1,119 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3::NQuery { + +struct TTxOnlineSettings { + using TSelf = TTxOnlineSettings; + + FLUENT_SETTING_DEFAULT(bool, AllowInconsistentReads, false); + + TTxOnlineSettings() {} +}; + +struct TTxSettings { + using TSelf = TTxSettings; + + TTxSettings() + : Mode_(TS_SERIALIZABLE_RW) {} + + static TTxSettings SerializableRW() { + return TTxSettings(TS_SERIALIZABLE_RW); + } + + static TTxSettings OnlineRO(const TTxOnlineSettings& settings = TTxOnlineSettings()) { + return TTxSettings(TS_ONLINE_RO).OnlineSettings(settings); + } + + static TTxSettings StaleRO() { + return TTxSettings(TS_STALE_RO); + } + + static TTxSettings SnapshotRO() { + return TTxSettings(TS_SNAPSHOT_RO); + } + + static TTxSettings SnapshotRW() { + return TTxSettings(TS_SNAPSHOT_RW); + } + + void Out(IOutputStream& out) const { + switch (Mode_) { + case TS_SERIALIZABLE_RW: + out << "SerializableRW"; + break; + case TS_ONLINE_RO: + out << "OnlineRO"; + break; + case TS_STALE_RO: + out << "StaleRO"; + break; + case TS_SNAPSHOT_RO: + out << "SnapshotRO"; + break; + case TS_SNAPSHOT_RW: + out << "SnapshotRW"; + break; + default: + out << "Unknown"; + break; + } + } + + enum ETransactionMode { + TS_SERIALIZABLE_RW, + TS_ONLINE_RO, + TS_STALE_RO, + TS_SNAPSHOT_RO, + TS_SNAPSHOT_RW, + }; + + FLUENT_SETTING(TTxOnlineSettings, OnlineSettings); + + ETransactionMode GetMode() const { + return Mode_; + } +private: + TTxSettings(ETransactionMode mode) + : Mode_(mode) {} + + ETransactionMode Mode_; +}; + +struct TTxControl { + using TSelf = TTxControl; + + static TTxControl Tx(const std::string& txId) { + return TTxControl(txId); + } + + static TTxControl BeginTx(const TTxSettings& settings = TTxSettings()) { + return TTxControl(settings); + } + + static TTxControl NoTx() { + return TTxControl(); + } + + const std::optional TxId_; + const std::optional TxSettings_; + FLUENT_SETTING_FLAG(CommitTx); + + bool HasTx() const { return TxId_.has_value() || TxSettings_.has_value(); } + +private: + TTxControl() {} + + TTxControl(const std::string& txId) + : TxId_(txId) {} + + TTxControl(const TTxSettings& txSettings) + : TxSettings_(txSettings) {} +}; + +} // namespace NYdb::V3::NQuery diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/rate_limiter/rate_limiter.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/rate_limiter/rate_limiter.h new file mode 100644 index 000000000000..040e65f8f77f --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/rate_limiter/rate_limiter.h @@ -0,0 +1,174 @@ +#pragma once + +#include + +namespace Ydb::RateLimiter { +class DescribeResourceResult; +class HierarchicalDrrSettings; +} // namespace Ydb::RateLimiter + +namespace NYdb::inline V3::NRateLimiter { + +// Settings for hierarchical deficit round robin (HDRR) algorithm. +template +struct THierarchicalDrrSettings : public TOperationRequestSettings { + using TSelf = TDerived; + + // Resource consumption speed limit. + // Value is required for root resource. + // Must be nonnegative. + FLUENT_SETTING_OPTIONAL(double, MaxUnitsPerSecond); + + // Maximum burst size of resource consumption across the whole cluster + // divided by max_units_per_second. + // Default value is 1. + // This means that maximum burst size might be equal to max_units_per_second. + // Must be nonnegative. + FLUENT_SETTING_OPTIONAL(double, MaxBurstSizeCoefficient); + + // Prefetch in local bucket up to PrefetchCoefficient*MaxUnitsPerSecond units (full size). + // Default value is inherited from parent or 0.2 for root. + // Disables prefetching if any negative value is set + // (It is useful to avoid bursts in case of large number of local buckets). + FLUENT_SETTING_OPTIONAL(double, PrefetchCoefficient); + + void DisablePrefetching() { + PrefetchCoefficient_ = -1.0; + } + + // Prefetching starts if there is less than PrefetchWatermark fraction of full local bucket left. + // Default value is inherited from parent or 0.75 for root. + // Must be nonnegative and less than or equal to 1. + FLUENT_SETTING_OPTIONAL(double, PrefetchWatermark); +}; + +// Settings for create resource request. +struct TCreateResourceSettings : public THierarchicalDrrSettings { +}; + +// Settings for alter resource request. +struct TAlterResourceSettings : public THierarchicalDrrSettings { +}; + +// Settings for drop resource request. +struct TDropResourceSettings : public TOperationRequestSettings {}; + +// Settings for list resources request. +struct TListResourcesSettings : public TOperationRequestSettings { + using TSelf = TListResourcesSettings; + + // List resources recursively, including children. + FLUENT_SETTING_FLAG(Recursive); +}; + +// Settings for describe resource request. +struct TDescribeResourceSettings : public TOperationRequestSettings {}; + +// Result for list resources request. +struct TListResourcesResult : public TStatus { + TListResourcesResult(TStatus status, std::vector paths); + + // Paths of listed resources inside a specified coordination node. + const std::vector& GetResourcePaths() const { + return ResourcePaths_; + } + +private: + std::vector ResourcePaths_; +}; + +// Settings for acquire resource request. +struct TAcquireResourceSettings : public TOperationRequestSettings { + using TSelf = TAcquireResourceSettings; + + FLUENT_SETTING_OPTIONAL(uint64_t, Amount); + FLUENT_SETTING_FLAG(IsUsedAmount); +}; + +using TAsyncListResourcesResult = NThreading::TFuture; + +// Result for describe resource request. +struct TDescribeResourceResult : public TStatus { + struct THierarchicalDrrProps { + THierarchicalDrrProps(const Ydb::RateLimiter::HierarchicalDrrSettings&); + + // Resource consumption speed limit. + std::optional GetMaxUnitsPerSecond() const { + return MaxUnitsPerSecond_; + } + + // Maximum burst size of resource consumption across the whole cluster + // divided by max_units_per_second. + std::optional GetMaxBurstSizeCoefficient() const { + return MaxBurstSizeCoefficient_; + } + + // Prefetch in local bucket up to PrefetchCoefficient*MaxUnitsPerSecond units (full size). + std::optional GetPrefetchCoefficient() const { + return PrefetchCoefficient_; + } + + // Prefetching starts if there is less than PrefetchWatermark fraction of full local bucket left. + std::optional GetPrefetchWatermark() const { + return PrefetchWatermark_; + } + + private: + std::optional MaxUnitsPerSecond_; + std::optional MaxBurstSizeCoefficient_; + std::optional PrefetchCoefficient_; + std::optional PrefetchWatermark_; + }; + + TDescribeResourceResult(TStatus status, const Ydb::RateLimiter::DescribeResourceResult& result); + + // Path of resource inside a coordination node. + const std::string& GetResourcePath() const { + return ResourcePath_; + } + + const THierarchicalDrrProps& GetHierarchicalDrrProps() const { + return HierarchicalDrrProps_; + } + +private: + std::string ResourcePath_; + THierarchicalDrrProps HierarchicalDrrProps_; +}; + +using TAsyncDescribeResourceResult = NThreading::TFuture; + +// Rate limiter client. +class TRateLimiterClient { +public: + TRateLimiterClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + // Create a new resource in existing coordination node. + TAsyncStatus CreateResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TCreateResourceSettings& = {}); + + // Update a resource in coordination node. + TAsyncStatus AlterResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAlterResourceSettings& = {}); + + // Delete a resource from coordination node. + TAsyncStatus DropResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDropResourceSettings& = {}); + + // List resources in given coordination node. + TAsyncListResourcesResult ListResources(const std::string& coordinationNodePath, const std::string& resourcePath, const TListResourcesSettings& = {}); + + // Describe properties of resource in coordination node. + TAsyncDescribeResourceResult DescribeResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDescribeResourceSettings& = {}); + + // Acquire resources's units inside a coordination node. + // If CancelAfter is set greater than zero and less than OperationTimeout + // and resource is not ready after CancelAfter time, + // the result code of this operation will be CANCELLED and resource will not be spent. + // It is recommended to specify both OperationTimeout and CancelAfter. + // CancelAfter should be less than OperationTimeout. + TAsyncStatus AcquireResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAcquireResourceSettings& = {}); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NRateLimiter diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_ca.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_ca.h new file mode 100644 index 000000000000..e3b3bf055b98 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_ca.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +std::string GetRootCertificate(); + +} // namespace NYdb \ No newline at end of file diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_resources.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_resources.h new file mode 100644 index 000000000000..02a8e26681b7 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/resources/ydb_resources.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +extern const char* YDB_AUTH_TICKET_HEADER; +extern const char* YDB_DATABASE_HEADER; +extern const char* YDB_TRACE_ID_HEADER; +extern const char* OTEL_TRACE_HEADER; +extern const char* YDB_SDK_BUILD_INFO_HEADER; +extern const char* YDB_REQUEST_TYPE_HEADER; +extern const char* YDB_CONSUMED_UNITS_HEADER; +extern const char* YDB_SERVER_HINTS; +extern const char* YDB_CLIENT_CAPABILITIES; +extern const char* YDB_SESSION_CLOSE; +extern const char* YDB_CLIENT_CAPABILITY_SESSION_BALANCER; +extern const char* YDB_APPLICATION_NAME; +extern const char* YDB_CLIENT_PID; + +std::string GetSdkSemver(); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/fwd.h new file mode 100644 index 000000000000..98f7ba8cba49 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/fwd.h @@ -0,0 +1,8 @@ +#pragma once + +namespace NYdb::inline V3 { + +class TResultSet; +class TResultSetParser; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h new file mode 100644 index 000000000000..4fe057b663b8 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h @@ -0,0 +1,110 @@ +#pragma once + +#include "fwd.h" + +#include + +#include + +namespace Ydb { + class ResultSet; +} + +namespace NYdb::inline V3 { + +class TProtoAccessor; + +struct TColumn { + std::string Name; + TType Type; + + TColumn(const std::string& name, const TType& type) + : Name(name) + , Type(type) {} + + std::string ToString() const; + void Out(IOutputStream& o) const; +}; + +bool operator==(const TColumn& col1, const TColumn& col2); +bool operator!=(const TColumn& col1, const TColumn& col2); + +//! Collection of rows, represents result of query or part of the result in case of stream operations +class TResultSet { + friend class TResultSetParser; + friend class NYdb::V3::TProtoAccessor; +public: + TResultSet(const Ydb::ResultSet& proto); + TResultSet(Ydb::ResultSet&& proto); + + //! Returns number of columns + size_t ColumnsCount() const; + + //! Returns number of rows in result set (which is partial in case of stream operations) + size_t RowsCount() const; + + //! Returns true if result set was truncated + bool Truncated() const; + + //! Returns meta information (name, type) for columns + const std::vector& GetColumnsMeta() const; + +private: + const Ydb::ResultSet& GetProto() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//! Note: TResultSetParser - mutable object, iteration thougth it changes internal state +class TResultSetParser : public TMoveOnly { +public: + TResultSetParser(TResultSetParser&&); + TResultSetParser(const TResultSet& resultSet); + + ~TResultSetParser(); + + //! Returns number of columns + size_t ColumnsCount() const; + + //! Returns number of rows + size_t RowsCount() const; + + //! Set iterator to the next result row. + //! On success TryNextRow will reset all column parsers to the values in next row. + //! Column parsers are invalid before the first TryNextRow call. + bool TryNextRow(); + + //! Returns index for column with specified name. + //! If there is no column with such name, then -1 is returned. + ssize_t ColumnIndex(const std::string& columnName); + + //! Returns column value parser for column with specified index. + //! State of the parser is preserved until next TryNextRow call. + TValueParser& ColumnParser(size_t columnIndex); + + //! Returns column value parser for column with specified name. + //! State of the parser is preserved until next TryNextRow call. + TValueParser& ColumnParser(const std::string& columnName); + + //! Returns TValue for column with specified index. + //! TValue will have copy of coresponding data so this method + //! is less effective compare with + //! direct TValueParser constructed by ColumnParser call + TValue GetValue(size_t columnIndex) const; + + //! Returns TValue for column with specified name. + //! TValue will have copy of coresponding data so this method + //! is less effective compare with + //! direct TValueParser constructed by ColumnParser call + TValue GetValue(const std::string& columnName) const; + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +using TResultSets = std::vector; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h new file mode 100644 index 000000000000..748bd3a4bf79 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3::NRetry { + +struct TBackoffSettings { + using TSelf = TBackoffSettings; + + FLUENT_SETTING_DEFAULT(TDuration, SlotDuration, TDuration::Seconds(1)); + FLUENT_SETTING_DEFAULT(uint32_t, Ceiling, 6); + FLUENT_SETTING_DEFAULT(double, UncertainRatio, 0.5); +}; + +struct TRetryOperationSettings { + using TSelf = TRetryOperationSettings; + + FLUENT_SETTING_DEFAULT(uint32_t, MaxRetries, 10); + FLUENT_SETTING_DEFAULT(bool, RetryNotFound, true); + FLUENT_SETTING_DEFAULT(TDuration, GetSessionClientTimeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT(TDuration, MaxTimeout, TDuration::Max()); + FLUENT_SETTING_DEFAULT(TBackoffSettings, FastBackoffSettings, DefaultFastBackoffSettings()); + FLUENT_SETTING_DEFAULT(TBackoffSettings, SlowBackoffSettings, DefaultSlowBackoffSettings()); + FLUENT_SETTING_FLAG(Idempotent); + FLUENT_SETTING_FLAG(Verbose); + FLUENT_SETTING_FLAG(RetryUndefined); + + static TBackoffSettings DefaultFastBackoffSettings() { + return TBackoffSettings() + .Ceiling(10) + .SlotDuration(TDuration::MilliSeconds(5)) + .UncertainRatio(0.5); + } + + static TBackoffSettings DefaultSlowBackoffSettings() { + return TBackoffSettings() + .Ceiling(6) + .SlotDuration(TDuration::Seconds(1)) + .UncertainRatio(0.5); + } +}; + +} // namespace NYdb::V3::NRetry diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h new file mode 100644 index 000000000000..1351d61875d5 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h @@ -0,0 +1,211 @@ +#pragma once + +#include + +namespace Ydb { + class VirtualTimestamp; + namespace Scheme { + class Entry; + class ModifyPermissionsRequest; + class Permissions; + } +} + +namespace NYdb::inline V3 { +namespace NScheme { + +//////////////////////////////////////////////////////////////////////////////// + +struct TPermissions { + TPermissions(const std::string& subject) + : Subject(subject) + {} + TPermissions(const std::string& subject, const std::vector& names) + : Subject(subject) + , PermissionNames(names) + {} + TPermissions(const ::Ydb::Scheme::Permissions& proto); + + std::string Subject; + std::vector PermissionNames; + + void SerializeTo(::Ydb::Scheme::Permissions& proto) const; +}; + +enum class ESchemeEntryType : i32 { + Unknown = -1, + Directory = 1, + Table = 2, + PqGroup = 3, + SubDomain = 4, + RtmrVolume = 5, + BlockStoreVolume = 6, + CoordinationNode = 7, + ColumnStore = 12, + ColumnTable = 13, + Sequence = 15, + Replication = 16, + Topic = 17, + ExternalTable = 18, + ExternalDataSource = 19, + View = 20, + ResourcePool = 21, +}; + +struct TVirtualTimestamp { + uint64_t PlanStep = 0; + uint64_t TxId = 0; + + TVirtualTimestamp() = default; + TVirtualTimestamp(uint64_t planStep, uint64_t txId); + TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); + + std::string ToString() const; + void Out(IOutputStream& out) const; + + bool operator<(const TVirtualTimestamp& rhs) const; + bool operator<=(const TVirtualTimestamp& rhs) const; + bool operator>(const TVirtualTimestamp& rhs) const; + bool operator>=(const TVirtualTimestamp& rhs) const; + bool operator==(const TVirtualTimestamp& rhs) const; + bool operator!=(const TVirtualTimestamp& rhs) const; +}; + +struct TSchemeEntry { + std::string Name; + std::string Owner; + ESchemeEntryType Type; + std::vector EffectivePermissions; + std::vector Permissions; + uint64_t SizeBytes = 0; + TVirtualTimestamp CreatedAt; + + TSchemeEntry() = default; + TSchemeEntry(const ::Ydb::Scheme::Entry& proto); + + void Out(IOutputStream& out) const; + + // Fills ModifyPermissionsRequest proto from this entry + void SerializeTo(::Ydb::Scheme::ModifyPermissionsRequest& request) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDescribePathResult; +class TListDirectoryResult; + +using TAsyncDescribePathResult = NThreading::TFuture; +using TAsyncListDirectoryResult = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +struct TMakeDirectorySettings : public TOperationRequestSettings {}; + +struct TRemoveDirectorySettings : public TOperationRequestSettings {}; + +struct TDescribePathSettings : public TOperationRequestSettings {}; + +struct TListDirectorySettings : public TOperationRequestSettings {}; + +enum class EModifyPermissionsAction { + Grant, + Revoke, + Set, + Chown +}; + +struct TModifyPermissionsSettings : public TOperationRequestSettings { + TModifyPermissionsSettings& AddGrantPermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Grant, permissions); + return *this; + } + + TModifyPermissionsSettings& AddRevokePermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Revoke, permissions); + return *this; + } + + TModifyPermissionsSettings& AddSetPermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Set, permissions); + return *this; + } + + TModifyPermissionsSettings& AddChangeOwner(const std::string& owner) { + AddAction(EModifyPermissionsAction::Chown, TPermissions(owner)); + return *this; + } + + TModifyPermissionsSettings& AddClearAcl() { + ClearAcl_ = true; + return *this; + } + + TModifyPermissionsSettings& AddInterruptInheritance(bool value) { + SetInterruptInheritance_ = true; + InterruptInheritanceValue_ = value; + return *this; + } + + std::vector> Actions_; + bool ClearAcl_ = false; + bool SetInterruptInheritance_ = false; + bool InterruptInheritanceValue_ = false; + void AddAction(EModifyPermissionsAction action, const TPermissions& permissions) { + Actions_.emplace_back(std::pair{action, permissions}); + } + + TModifyPermissionsSettings() = default; + explicit TModifyPermissionsSettings(const ::Ydb::Scheme::ModifyPermissionsRequest& request); +}; + +class TSchemeClient { + class TImpl; + +public: + TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncStatus MakeDirectory(const std::string& path, + const TMakeDirectorySettings& settings = TMakeDirectorySettings()); + + TAsyncStatus RemoveDirectory(const std::string& path, + const TRemoveDirectorySettings& settings = TRemoveDirectorySettings()); + + TAsyncDescribePathResult DescribePath(const std::string& path, + const TDescribePathSettings& settings = TDescribePathSettings()); + + TAsyncListDirectoryResult ListDirectory(const std::string& path, + const TListDirectorySettings& settings = TListDirectorySettings()); + + TAsyncStatus ModifyPermissions(const std::string& path, + const TModifyPermissionsSettings& data); + +private: + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDescribePathResult : public TStatus { +public: + TDescribePathResult(TStatus&& status, const TSchemeEntry& entry); + const TSchemeEntry& GetEntry() const; + + void Out(IOutputStream& out) const; + +private: + TSchemeEntry Entry_; +}; + +class TListDirectoryResult : public TDescribePathResult { +public: + TListDirectoryResult(TStatus&& status, const TSchemeEntry& self, std::vector&& children); + const std::vector& GetChildren() const; + + void Out(IOutputStream& out) const; + +private: + std::vector Children_; +}; + +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/fwd.h new file mode 100644 index 000000000000..04c947ef6be6 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/fwd.h @@ -0,0 +1,116 @@ +#pragma once + +namespace NYdb::inline V3::NTable { + +class TKeyBound; +class TKeyRange; + +struct TTableColumn; +struct TAlterTableColumn; + +struct TPartitionStats; + +struct TSequenceDescription; +class TChangefeedDescription; +class TIndexDescription; +class TColumnFamilyDescription; +class TTableDescription; + +struct TExplicitPartitions; + +struct TRenameIndex; + +struct TGlobalIndexSettings; +struct TVectorIndexSettings; +struct TKMeansTreeSettings; +struct TCreateSessionSettings; +struct TSessionPoolSettings; +struct TClientSettings; +struct TBulkUpsertSettings; +struct TReadRowsSettings; +struct TStreamExecScanQuerySettings; +struct TTxOnlineSettings; +struct TCreateTableSettings; +struct TDropTableSettings; +struct TAlterTableSettings; +struct TCopyTableSettings; +struct TCopyTablesSettings; +struct TRenameTablesSettings; +struct TDescribeTableSettings; +struct TExplainDataQuerySettings; +struct TPrepareDataQuerySettings; +struct TExecDataQuerySettings; +struct TExecSchemeQuerySettings; +struct TBeginTxSettings; +struct TCommitTxSettings; +struct TRollbackTxSettings; +struct TCloseSessionSettings; +struct TKeepAliveSettings; +struct TReadTableSettings; + +class TPartitioningSettings; +class TDateTypeColumnModeSettings; +class TValueSinceUnixEpochModeSettings; +class TTtlTierSettings; +class TTtlSettings; +class TAlterTtlSettings; +class TStorageSettings; +class TReadReplicasSettings; +class TTxSettings; + +struct TColumnFamilyPolicy; +struct TStoragePolicy; +struct TPartitioningPolicy; +struct TReplicationPolicy; + +class TBuildIndexOperation; + +class TTtlDeleteAction; +class TTtlEvictToExternalStorageAction; + +class TStorageSettingsBuilder; +class TPartitioningSettingsBuilder; +class TColumnFamilyBuilder; +class TTableStorageSettingsBuilder; +class TTableColumnFamilyBuilder; +class TTablePartitioningSettingsBuilder; +class TTableBuilder; +class TAlterStorageSettingsBuilder; +class TAlterColumnFamilyBuilder; +class TAlterTtlSettingsBuilder; +class TAlterAttributesBuilder; +class TAlterPartitioningSettingsBuilder; + +class TPrepareQueryResult; +class TExplainQueryResult; +class TDescribeTableResult; +class TDataQueryResult; +class TBeginTransactionResult; +class TCommitTransactionResult; +class TCreateSessionResult; +class TKeepAliveResult; +class TBulkUpsertResult; +class TReadRowsResult; + +template +class TSimpleStreamPart; + +class TScanQueryPart; + +class TTablePartIterator; +class TScanQueryPartIterator; + +class TReadTableSnapshot; + +class TCopyItem; +class TRenameItem; + +class TDataQuery; + +class TTransaction; +class TTxControl; + +class TSession; +class TTableClient; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/query_stats/stats.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/query_stats/stats.h new file mode 100644 index 000000000000..38740d2dd13c --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/query_stats/stats.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +class TDuration; + +namespace Ydb { + namespace TableStats { + class QueryStats; + } + + namespace Table { + class QueryStatsCollection; + } +} + +namespace NYdb::inline V3 { + +class TProtoAccessor; + +namespace NScripting { + +class TScriptingClient; +class TYqlResultPartIterator; + +} // namespace NScripting + +namespace NTable { + +enum class ECollectQueryStatsMode { + None = 0, // Stats collection is disabled + Basic = 1, // Aggregated stats of reads, updates and deletes per table + Full = 2, // Add per-stage execution profile and query plan on top of Basic mode + Profile = 3 // Detailed execution stats including stats for individual tasks and channels +}; + +using TQueryStats = NQuery::TExecStats; + +std::optional ParseQueryStatsMode(std::string_view statsMode); + +std::string_view QueryStatsModeToString(ECollectQueryStatsMode statsMode); + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h new file mode 100644 index 000000000000..98a15f9284ab --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h @@ -0,0 +1,2198 @@ +#pragma once + +#include "fwd.h" + +#include "table_enum.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace Ydb { +namespace Table { + +class StorageSettings; +class ColumnFamily; +class CreateTableRequest; +class Changefeed; +class ChangefeedDescription; +class DescribeTableResult; +class ExplicitPartitions; +class GlobalIndexSettings; +class VectorIndexSettings; +class KMeansTreeSettings; +class PartitioningSettings; +class DateTypeColumnModeSettings; +class TtlSettings; +class TtlTier; +class TableIndex; +class TableIndexDescription; +class ValueSinceUnixEpochModeSettings; +class EvictionToExternalStorageSettings; + +} // namespace Table +} // namespace Ydb + +namespace NYdb::inline V3 { + +namespace NRetry::Async { +template +class TRetryContext; +} // namespace NRetry::Async + +namespace NRetry::Sync { +template +class TRetryContext; +} // namespace NRetry::Sync + +namespace NScheme { +struct TPermissions; +} // namespace NScheme + +namespace NTable { + +//////////////////////////////////////////////////////////////////////////////// + +class TKeyBound { +public: + static TKeyBound Inclusive(const TValue& value) { + return TKeyBound(value, true); + } + + static TKeyBound Exclusive(const TValue& value) { + return TKeyBound(value, false); + } + + bool IsInclusive() const { + return Equal_; + } + + const TValue& GetValue() const { + return Value_; + } +private: + TKeyBound(const TValue& value, bool equal = false) + : Value_(value) + , Equal_(equal) + {} + TValue Value_; + bool Equal_; +}; + +class TKeyRange { +public: + TKeyRange(const std::optional& from, const std::optional& to) + : From_(from) + , To_(to) {} + + const std::optional& From() const { + return From_; + } + + const std::optional& To() const { + return To_; + } +private: + std::optional From_; + std::optional To_; +}; + +struct TSequenceDescription { + struct TSetVal { + int64_t NextValue; + bool NextUsed; + }; + std::optional SetVal; +}; + +struct TTableColumn { + std::string Name; + TType Type; + std::string Family; + std::optional NotNull; + std::optional SequenceDescription; + + TTableColumn() = default; + + TTableColumn(std::string name, + TType type, + std::string family = std::string(), + std::optional notNull = std::nullopt, + std::optional sequenceDescription = std::nullopt) + : Name(std::move(name)) + , Type(std::move(type)) + , Family(std::move(family)) + , NotNull(std::move(notNull)) + , SequenceDescription(std::move(sequenceDescription)) + { } + + // Conversion from TColumn for API compatibility + TTableColumn(const TColumn& column) + : Name(column.Name) + , Type(column.Type) + { } + + // Conversion from TColumn for API compatibility + TTableColumn(TColumn&& column) + : Name(std::move(column.Name)) + , Type(std::move(column.Type)) + { } + + // Conversion to TColumn for API compatibility + operator TColumn() const { + return TColumn(Name, Type); + } +}; + +struct TAlterTableColumn { + std::string Name; + std::string Family; + + TAlterTableColumn() = default; + + explicit TAlterTableColumn(std::string name) + : Name(std::move(name)) + { } + + TAlterTableColumn(std::string name, std::string family) + : Name(std::move(name)) + , Family(std::move(family)) + { } +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents table partitioning settings +class TPartitioningSettings { +public: + TPartitioningSettings(); + explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto); + + const Ydb::Table::PartitioningSettings& GetProto() const; + + std::optional GetPartitioningBySize() const; + std::optional GetPartitioningByLoad() const; + uint64_t GetPartitionSizeMb() const; + uint64_t GetMinPartitionsCount() const; + uint64_t GetMaxPartitionsCount() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +struct TExplicitPartitions { + using TSelf = TExplicitPartitions; + + FLUENT_SETTING_VECTOR(TValue, SplitPoints); + + static TExplicitPartitions FromProto(const Ydb::Table::ExplicitPartitions& proto); + + void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; +}; + +struct TGlobalIndexSettings { + using TUniformOrExplicitPartitions = std::variant; + + TPartitioningSettings PartitioningSettings; + TUniformOrExplicitPartitions Partitions; + + static TGlobalIndexSettings FromProto(const Ydb::Table::GlobalIndexSettings& proto); + + void SerializeTo(Ydb::Table::GlobalIndexSettings& proto) const; +}; + +struct TVectorIndexSettings { +public: + enum class EMetric { + Unspecified = 0, + InnerProduct, + CosineSimilarity, + CosineDistance, + Manhattan, + Euclidean, + }; + + enum class EVectorType { + Unspecified = 0, + Float, + Uint8, + Int8, + Bit, + }; + + EMetric Metric = EMetric::Unspecified; + EVectorType VectorType = EVectorType::Unspecified; + uint32_t VectorDimension = 0; + + static TVectorIndexSettings FromProto(const Ydb::Table::VectorIndexSettings& proto); + + void SerializeTo(Ydb::Table::VectorIndexSettings& settings) const; + + void Out(IOutputStream &o) const; +}; + +struct TKMeansTreeSettings { +public: + enum class EMetric { + Unspecified = 0, + InnerProduct, + CosineSimilarity, + CosineDistance, + Manhattan, + Euclidean, + }; + + enum class EVectorType { + Unspecified = 0, + Float, + Uint8, + Int8, + Bit, + }; + + TVectorIndexSettings Settings; + uint32_t Clusters = 0; + uint32_t Levels = 0; + + static TKMeansTreeSettings FromProto(const Ydb::Table::KMeansTreeSettings& proto); + + void SerializeTo(Ydb::Table::KMeansTreeSettings& settings) const; + + void Out(IOutputStream &o) const; +}; + +//! Represents index description +class TIndexDescription { + friend class NYdb::V3::TProtoAccessor; + +public: + TIndexDescription( + const std::string& name, + EIndexType type, + const std::vector& indexColumns, + const std::vector& dataColumns = {}, + const std::vector& globalIndexSettings = {}, + const std::variant& specializedIndexSettings = {} + ); + + TIndexDescription( + const std::string& name, + const std::vector& indexColumns, + const std::vector& dataColumns = {}, + const std::vector& globalIndexSettings = {} + ); + + const std::string& GetIndexName() const; + EIndexType GetIndexType() const; + const std::vector& GetIndexColumns() const; + const std::vector& GetDataColumns() const; + const std::variant& GetVectorIndexSettings() const; + uint64_t GetSizeBytes() const; + + void SerializeTo(Ydb::Table::TableIndex& proto) const; + std::string ToString() const; + void Out(IOutputStream& o) const; + +private: + explicit TIndexDescription(const Ydb::Table::TableIndex& tableIndex); + explicit TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc); + + template + static TIndexDescription FromProto(const TProto& proto); + +private: + std::string IndexName_; + EIndexType IndexType_; + std::vector IndexColumns_; + std::vector DataColumns_; + std::vector GlobalIndexSettings_; + std::variant SpecializedIndexSettings_; + uint64_t SizeBytes_ = 0; +}; + +struct TRenameIndex { + std::string SourceName_; + std::string DestinationName_; + bool ReplaceDestination_ = false; +}; + +bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs); +bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs); + +class TBuildIndexOperation : public TOperation { +public: + using TOperation::TOperation; + TBuildIndexOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + + struct TMetadata { + EBuildIndexState State; + float Progress; + std::string Path; + std::optional Desctiption; + }; + + const TMetadata& Metadata() const; +private: + TMetadata Metadata_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents changefeed description +class TChangefeedDescription { + friend class NYdb::V3::TProtoAccessor; + +public: + class TInitialScanProgress { + public: + TInitialScanProgress(); + explicit TInitialScanProgress(uint32_t total, uint32_t completed); + + TInitialScanProgress& operator+=(const TInitialScanProgress& other); + + uint32_t GetPartsTotal() const; + uint32_t GetPartsCompleted() const; + float GetProgress() const; // percentage + + private: + uint32_t PartsTotal; + uint32_t PartsCompleted; + }; + +public: + TChangefeedDescription(const std::string& name, EChangefeedMode mode, EChangefeedFormat format); + + // Enable virtual timestamps + TChangefeedDescription& WithVirtualTimestamps(); + // Enable resolved timestamps + TChangefeedDescription& WithResolvedTimestamps(const TDuration& interval); + // Customise retention period of underlying topic (24h by default). + TChangefeedDescription& WithRetentionPeriod(const TDuration& value); + // Initial scan will output the current state of the table first + TChangefeedDescription& WithInitialScan(); + // Attributes + TChangefeedDescription& AddAttribute(const std::string& key, const std::string& value); + TChangefeedDescription& SetAttributes(const std::unordered_map& attrs); + TChangefeedDescription& SetAttributes(std::unordered_map&& attrs); + // Value that will be emitted in the `awsRegion` field of the record in DynamoDBStreamsJson format + TChangefeedDescription& WithAwsRegion(const std::string& value); + + const std::string& GetName() const; + EChangefeedMode GetMode() const; + EChangefeedFormat GetFormat() const; + EChangefeedState GetState() const; + bool GetVirtualTimestamps() const; + const std::optional& GetResolvedTimestamps() const; + bool GetInitialScan() const; + const std::unordered_map& GetAttributes() const; + const std::string& GetAwsRegion() const; + const std::optional& GetInitialScanProgress() const; + + void SerializeTo(Ydb::Table::Changefeed& proto) const; + void SerializeTo(Ydb::Table::ChangefeedDescription& proto) const; + std::string ToString() const; + void Out(IOutputStream& o) const; + + template + void SerializeCommonFields(TProto& proto) const; + +private: + explicit TChangefeedDescription(const Ydb::Table::Changefeed& proto); + explicit TChangefeedDescription(const Ydb::Table::ChangefeedDescription& proto); + + template + static TChangefeedDescription FromProto(const TProto& proto); + +private: + std::string Name_; + EChangefeedMode Mode_; + EChangefeedFormat Format_; + EChangefeedState State_ = EChangefeedState::Unknown; + bool VirtualTimestamps_ = false; + std::optional ResolvedTimestamps_; + std::optional RetentionPeriod_; + bool InitialScan_ = false; + std::unordered_map Attributes_; + std::string AwsRegion_; + std::optional InitialScanProgress_; +}; + +bool operator==(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs); +bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs); + +//////////////////////////////////////////////////////////////////////////////// + +struct TPartitionStats { + uint64_t Rows = 0; + uint64_t Size = 0; + uint32_t LeaderNodeId = 0; +}; + +class TDateTypeColumnModeSettings { +public: + explicit TDateTypeColumnModeSettings(const std::string& columnName, const TDuration& applyAfter); + void SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const; + + const std::string& GetColumnName() const; + const TDuration& GetExpireAfter() const; + +private: + std::string ColumnName_; + TDuration ApplyAfter_; +}; + +class TValueSinceUnixEpochModeSettings { +public: + enum class EUnit { + Seconds, + MilliSeconds, + MicroSeconds, + NanoSeconds, + + Unknown = std::numeric_limits::max() + }; + +public: + explicit TValueSinceUnixEpochModeSettings(const std::string& columnName, EUnit columnUnit, const TDuration& applyAfter); + void SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const; + + const std::string& GetColumnName() const; + EUnit GetColumnUnit() const; + const TDuration& GetExpireAfter() const; + + static void Out(IOutputStream& o, EUnit unit); + static std::string ToString(EUnit unit); + static EUnit UnitFromString(const std::string& value); + +private: + std::string ColumnName_; + EUnit ColumnUnit_; + TDuration ApplyAfter_; +}; + +class TTtlDeleteAction {}; +class TTtlEvictToExternalStorageAction { +public: + TTtlEvictToExternalStorageAction(const std::string& storageName); + void SerializeTo(Ydb::Table::EvictionToExternalStorageSettings& proto) const; + + std::string GetStorage() const; + +private: + std::string Storage_; +}; + +class TTtlTierSettings { +public: + using TExpression = std::variant< + TDateTypeColumnModeSettings, + TValueSinceUnixEpochModeSettings + >; + + using TAction = std::variant< + TTtlDeleteAction, + TTtlEvictToExternalStorageAction + >; + +public: + explicit TTtlTierSettings(const TExpression& expression, const TAction& action); + + static std::optional FromProto(const Ydb::Table::TtlTier& tier); + void SerializeTo(Ydb::Table::TtlTier& proto) const; + + const TExpression& GetExpression() const; + const TAction& GetAction() const; + +private: + TExpression Expression_; + TAction Action_; +}; + +//! Represents ttl settings +class TTtlSettings { +private: + using TMode = std::variant< + TDateTypeColumnModeSettings, + TValueSinceUnixEpochModeSettings + >; + +public: + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + enum class EMode { + DateTypeColumn = 0, + ValueSinceUnixEpoch = 1, + }; + + explicit TTtlSettings(const std::vector& tiers); + + explicit TTtlSettings(const std::string& columnName, const TDuration& expireAfter); + const TDateTypeColumnModeSettings& GetDateTypeColumn() const; + explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, uint32_t runIntervalSeconds); + + explicit TTtlSettings(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter); + const TValueSinceUnixEpochModeSettings& GetValueSinceUnixEpoch() const; + explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, uint32_t runIntervalSeconds); + + static std::optional FromProto(const Ydb::Table::TtlSettings& proto); + void SerializeTo(Ydb::Table::TtlSettings& proto) const; + EMode GetMode() const; + + TTtlSettings& SetRunInterval(const TDuration& value); + const TDuration& GetRunInterval() const; + + const std::vector& GetTiers() const; + +private: + std::vector Tiers_; + TDuration RunInterval_ = TDuration::Zero(); +}; + +class TAlterTtlSettings { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + TAlterTtlSettings() + : Action_(true) + {} + + template + explicit TAlterTtlSettings(Args&&... args) + : Action_(TTtlSettings(std::forward(args)...)) + {} + +public: + enum class EAction { + Drop = 0, + Set = 1, + }; + + static TAlterTtlSettings Drop() { + return TAlterTtlSettings(); + } + + template + static TAlterTtlSettings Set(Args&&... args) { + return TAlterTtlSettings(std::forward(args)...); + } + + EAction GetAction() const; + const TTtlSettings& GetTtlSettings() const; + +private: + std::variant< + bool, // EAction::Drop + TTtlSettings // EAction::Set + > Action_; +}; + +//! Represents table storage settings +class TStorageSettings { +public: + TStorageSettings(); + explicit TStorageSettings(const Ydb::Table::StorageSettings& proto); + + const Ydb::Table::StorageSettings& GetProto() const; + + std::optional GetTabletCommitLog0() const; + std::optional GetTabletCommitLog1() const; + std::optional GetExternal() const; + std::optional GetStoreExternalBlobs() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//! Represents column family description +class TColumnFamilyDescription { +public: + explicit TColumnFamilyDescription(const Ydb::Table::ColumnFamily& desc); + + const Ydb::Table::ColumnFamily& GetProto() const; + + const std::string& GetName() const; + std::optional GetData() const; + std::optional GetCompression() const; + std::optional GetKeepInMemory() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//! Represents table read replicas settings +class TReadReplicasSettings { +public: + enum class EMode { + PerAz = 0, + AnyAz = 1 + }; + + TReadReplicasSettings(EMode mode, uint64_t readReplicasCount); + + EMode GetMode() const; + uint64_t GetReadReplicasCount() const; + +private: + EMode Mode_; + uint64_t ReadReplicasCount_; +}; + +enum class EStoreType { + Row = 0, + Column = 1 +}; + +//! Represents table description +class TTableDescription { + friend class TTableBuilder; + friend class NYdb::V3::TProtoAccessor; + + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TTableDescription(Ydb::Table::DescribeTableResult&& desc, const TDescribeTableSettings& describeSettings); + + const std::vector& GetPrimaryKeyColumns() const; + // DEPRECATED: use GetTableColumns() + std::vector GetColumns() const; + std::vector GetTableColumns() const; + std::vector GetIndexDescriptions() const; + std::vector GetChangefeedDescriptions() const; + std::optional GetTtlSettings() const; + // Deprecated. Use GetTtlSettings() instead + std::optional GetTiering() const; + EStoreType GetStoreType() const; + + // Deprecated. Use GetEntry() of TDescribeTableResult instead + const std::string& GetOwner() const; + const std::vector& GetPermissions() const; + const std::vector& GetEffectivePermissions() const; + + const std::vector& GetKeyRanges() const; + + // Folow options related to table statistics + // flag WithTableStatistics must be set + + // Number of partition + uint64_t GetPartitionsCount() const; + // Approximate number of rows + uint64_t GetTableRows() const; + // Approximate size of table (bytes) + uint64_t GetTableSize() const; + // Timestamp of last modification + TInstant GetModificationTime() const; + // Timestamp of table creation + TInstant GetCreationTime() const; + + // Returns partition statistics for table + // flag WithTableStatistics and WithPartitionStatistics must be set + const std::vector& GetPartitionStats() const; + + // Returns storage settings of the table + const TStorageSettings& GetStorageSettings() const; + + // Returns column families of the table + const std::vector& GetColumnFamilies() const; + + // Attributes + const std::unordered_map& GetAttributes() const; + + // Returns partitioning settings of the table + const TPartitioningSettings& GetPartitioningSettings() const; + + // Bloom filter by key + std::optional GetKeyBloomFilter() const; + + // Returns read replicas settings of the table + std::optional GetReadReplicasSettings() const; + + // Fills CreateTableRequest proto from this description + void SerializeTo(Ydb::Table::CreateTableRequest& request) const; + +private: + TTableDescription(); + explicit TTableDescription(const Ydb::Table::CreateTableRequest& request); + + void AddColumn(const std::string& name, const Ydb::Type& type, const std::string& family, std::optional notNull, std::optional sequenceDescription); + void SetPrimaryKeyColumns(const std::vector& primaryKeyColumns); + + // common + void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns); + void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns); + void AddSecondaryIndex(const TIndexDescription& indexDescription); + // sync + void AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + void AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // async + void AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + void AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // unique + void AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + void AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // vector KMeansTree + void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); + void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); + + // default + void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + + void SetTtlSettings(TTtlSettings&& settings); + void SetTtlSettings(const TTtlSettings& settings); + + void SetStorageSettings(const TStorageSettings& settings); + void AddColumnFamily(const TColumnFamilyDescription& desc); + void AddAttribute(const std::string& key, const std::string& value); + void SetAttributes(const std::unordered_map& attrs); + void SetAttributes(std::unordered_map&& attrs); + void SetCompactionPolicy(const std::string& name); + void SetUniformPartitions(uint64_t partitionsCount); + void SetPartitionAtKeys(const TExplicitPartitions& keys); + void SetPartitioningSettings(const TPartitioningSettings& settings); + void SetKeyBloomFilter(bool enabled); + void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount); + void SetStoreType(EStoreType type); + const Ydb::Table::DescribeTableResult& GetProto() const; + + class TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TStorageSettingsBuilder { +public: + TStorageSettingsBuilder(); + ~TStorageSettingsBuilder(); + + TStorageSettingsBuilder& SetTabletCommitLog0(const std::string& media); + TStorageSettingsBuilder& SetTabletCommitLog1(const std::string& media); + TStorageSettingsBuilder& SetExternal(const std::string& media); + TStorageSettingsBuilder& SetStoreExternalBlobs(bool enabled); + + TStorageSettings Build() const; + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TPartitioningSettingsBuilder { +public: + TPartitioningSettingsBuilder(); + ~TPartitioningSettingsBuilder(); + + TPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled); + TPartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled); + TPartitioningSettingsBuilder& SetPartitionSizeMb(uint64_t sizeMb); + TPartitioningSettingsBuilder& SetMinPartitionsCount(uint64_t count); + TPartitioningSettingsBuilder& SetMaxPartitionsCount(uint64_t count); + + TPartitioningSettings Build() const; + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TColumnFamilyBuilder { +public: + explicit TColumnFamilyBuilder(const std::string& name); + ~TColumnFamilyBuilder(); + + TColumnFamilyBuilder& SetData(const std::string& media); + TColumnFamilyBuilder& SetCompression(EColumnFamilyCompression compression); + TColumnFamilyBuilder& SetKeepInMemory(bool enabled); + + TColumnFamilyDescription Build() const; + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TTableStorageSettingsBuilder { +public: + explicit TTableStorageSettingsBuilder(TTableBuilder& parent) + : Parent_(parent) + { } + + TTableStorageSettingsBuilder& SetTabletCommitLog0(const std::string& media) { + Builder_.SetTabletCommitLog0(media); + return *this; + } + + TTableStorageSettingsBuilder& SetTabletCommitLog1(const std::string& media) { + Builder_.SetTabletCommitLog1(media); + return *this; + } + + TTableStorageSettingsBuilder& SetExternal(const std::string& media) { + Builder_.SetExternal(media); + return *this; + } + + TTableStorageSettingsBuilder& SetStoreExternalBlobs(bool enabled) { + Builder_.SetStoreExternalBlobs(enabled); + return *this; + } + + TTableBuilder& EndStorageSettings(); + +private: + TTableBuilder& Parent_; + TStorageSettingsBuilder Builder_; +}; + +class TTableColumnFamilyBuilder { +public: + TTableColumnFamilyBuilder(TTableBuilder& parent, const std::string& name) + : Parent_(parent) + , Builder_(name) + { } + + TTableColumnFamilyBuilder& SetData(const std::string& media) { + Builder_.SetData(media); + return *this; + } + + TTableColumnFamilyBuilder& SetCompression(EColumnFamilyCompression compression) { + Builder_.SetCompression(compression); + return *this; + } + + TTableColumnFamilyBuilder& SetKeepInMemory(bool enabled) { + Builder_.SetKeepInMemory(enabled); + return *this; + } + + TTableBuilder& EndColumnFamily(); + +private: + TTableBuilder& Parent_; + TColumnFamilyBuilder Builder_; +}; + +class TTablePartitioningSettingsBuilder { +public: + explicit TTablePartitioningSettingsBuilder(TTableBuilder& parent) + : Parent_(parent) + { } + + TTablePartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { + Builder_.SetPartitioningBySize(enabled); + return *this; + } + + TTablePartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled) { + Builder_.SetPartitioningByLoad(enabled); + return *this; + } + + TTablePartitioningSettingsBuilder& SetPartitionSizeMb(uint64_t sizeMb) { + Builder_.SetPartitionSizeMb(sizeMb); + return *this; + } + + TTablePartitioningSettingsBuilder& SetMinPartitionsCount(uint64_t count) { + Builder_.SetMinPartitionsCount(count); + return *this; + } + + TTablePartitioningSettingsBuilder& SetMaxPartitionsCount(uint64_t count) { + Builder_.SetMaxPartitionsCount(count); + return *this; + } + + TTableBuilder& EndPartitioningSettings(); + +private: + TTableBuilder& Parent_; + TPartitioningSettingsBuilder Builder_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TTableBuilder { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TTableBuilder() = default; + + TTableBuilder& SetStoreType(EStoreType type); + + TTableBuilder& AddNullableColumn(const std::string& name, const EPrimitiveType& type, const std::string& family = std::string()); + TTableBuilder& AddNullableColumn(const std::string& name, const TDecimalType& type, const std::string& family = std::string()); + TTableBuilder& AddNullableColumn(const std::string& name, const TPgType& type, const std::string& family = std::string()); + TTableBuilder& AddNonNullableColumn(const std::string& name, const EPrimitiveType& type, const std::string& family = std::string()); + TTableBuilder& AddNonNullableColumn(const std::string& name, const TDecimalType& type, const std::string& family = std::string()); + TTableBuilder& AddNonNullableColumn(const std::string& name, const TPgType& type, const std::string& family = std::string()); + TTableBuilder& SetPrimaryKeyColumns(const std::vector& primaryKeyColumns); + TTableBuilder& SetPrimaryKeyColumn(const std::string& primaryKeyColumn); + TTableBuilder& AddSerialColumn(const std::string& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const std::string& family = std::string()); + + // common + TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription); + TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns); + TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns); + TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::string& indexColumn); + + // sync + TTableBuilder& AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + TTableBuilder& AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + TTableBuilder& AddSyncSecondaryIndex(const std::string& indexName, const std::string& indexColumn); + + // async + TTableBuilder& AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + TTableBuilder& AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + TTableBuilder& AddAsyncSecondaryIndex(const std::string& indexName, const std::string& indexColumn); + + // unique + TTableBuilder& AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + TTableBuilder& AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + + // vector KMeansTree + TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); + TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); + + // default + TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); + TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::string& indexColumn); + + TTableBuilder& SetTtlSettings(TTtlSettings&& settings); + TTableBuilder& SetTtlSettings(const TTtlSettings& settings); + TTableBuilder& SetTtlSettings(const std::string& columnName, const TDuration& expireAfter = TDuration::Zero()); + TTableBuilder& SetTtlSettings(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); + + TTableBuilder& SetStorageSettings(const TStorageSettings& settings); + + TTableBuilder& AddColumnFamily(const TColumnFamilyDescription& desc); + + TTableBuilder& AddAttribute(const std::string& key, const std::string& value); + TTableBuilder& SetAttributes(const std::unordered_map& attrs); + TTableBuilder& SetAttributes(std::unordered_map&& attrs); + + TTableBuilder& SetCompactionPolicy(const std::string& name); + + // UniformPartitions and PartitionAtKeys are mutually exclusive + TTableBuilder& SetUniformPartitions(uint64_t partitionsCount); + TTableBuilder& SetPartitionAtKeys(const TExplicitPartitions& keys); + + TTableBuilder& SetPartitioningSettings(const TPartitioningSettings& settings); + + TTableBuilder& SetKeyBloomFilter(bool enabled); + + TTableBuilder& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount); + + TTableStorageSettingsBuilder BeginStorageSettings() { + return TTableStorageSettingsBuilder(*this); + } + + TTableColumnFamilyBuilder BeginColumnFamily(const std::string& name) { + return TTableColumnFamilyBuilder(*this, name); + } + + TTablePartitioningSettingsBuilder BeginPartitioningSettings() { + return TTablePartitioningSettingsBuilder(*this); + } + + TTableDescription Build(); + +private: + TTableDescription TableDescription_; +}; + +inline TTableBuilder& TTableStorageSettingsBuilder::EndStorageSettings() { + return Parent_.SetStorageSettings(Builder_.Build()); +} + +inline TTableBuilder& TTableColumnFamilyBuilder::EndColumnFamily() { + return Parent_.AddColumnFamily(Builder_.Build()); +} + +inline TTableBuilder& TTablePartitioningSettingsBuilder::EndPartitioningSettings() { + return Parent_.SetPartitioningSettings(Builder_.Build()); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TCopyItem { +public: + TCopyItem(const std::string& source, const std::string& destination); + + const std::string& SourcePath() const; + const std::string& DestinationPath() const; + + TCopyItem& SetOmitIndexes(); + bool OmitIndexes() const; + + void Out(IOutputStream& out) const; + +private: + std::string Source_; + std::string Destination_; + bool OmitIndexes_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TRenameItem { +public: + TRenameItem(const std::string& source, const std::string& destination); + + const std::string& SourcePath() const; + const std::string& DestinationPath() const; + + TRenameItem& SetReplaceDestination(); + bool ReplaceDestination() const; +private: + std::string Source_; + std::string Destination_; + bool ReplaceDestination_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +using TAsyncCreateSessionResult = NThreading::TFuture; +using TAsyncDataQueryResult = NThreading::TFuture; +using TAsyncPrepareQueryResult = NThreading::TFuture; +using TAsyncExplainDataQueryResult = NThreading::TFuture; +using TAsyncDescribeTableResult = NThreading::TFuture; +using TAsyncBeginTransactionResult = NThreading::TFuture; +using TAsyncCommitTransactionResult = NThreading::TFuture; +using TAsyncTablePartIterator = NThreading::TFuture; +using TAsyncKeepAliveResult = NThreading::TFuture; +using TAsyncBulkUpsertResult = NThreading::TFuture; +using TAsyncReadRowsResult = NThreading::TFuture; +using TAsyncScanQueryPartIterator = NThreading::TFuture; + +//////////////////////////////////////////////////////////////////////////////// + +struct TCreateSessionSettings : public TOperationRequestSettings {}; + +using TBackoffSettings = NYdb::V3::NRetry::TBackoffSettings; +using TRetryOperationSettings = NYdb::V3::NRetry::TRetryOperationSettings; + +struct TSessionPoolSettings { + using TSelf = TSessionPoolSettings; + + // Max number of sessions client can get from session pool + FLUENT_SETTING_DEFAULT(uint32_t, MaxActiveSessions, 50); + + // Max number of attempt to create session inside session pool + // to handle OVERLOADED error + FLUENT_SETTING_DEFAULT(uint32_t, RetryLimit, 5); + + // Max time session to be in idle state in session pool before + // keep alive start to touch it + FLUENT_SETTING_DEFAULT(TDuration, KeepAliveIdleThreshold, TDuration::Minutes(5)); + + // Max time session to be in idle state before closing + FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); + + // Min number of session in session pool. + // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. + FLUENT_SETTING_DEFAULT(uint32_t, MinPoolSize, 10); +}; + +struct TClientSettings : public TCommonClientSettingsBase { + using TSelf = TClientSettings; + using TSessionPoolSettings = TSessionPoolSettings; + + // Enable client query cache. Client query cache is used to map query text to + // prepared query id for ExecuteDataQuery calls on client side. + // Starting from YDB 20-4, server query cache is enabled by default, which + // make use of client cache unnecessary. Use of server cache is preferred + // as it doesn't require client-server synchronization and can recompile + // query on demand without client interaction. + // The recommended value is False. + FLUENT_SETTING_DEFAULT(bool, UseQueryCache, false); + FLUENT_SETTING_DEFAULT(uint32_t, QueryCacheSize, 1000); + FLUENT_SETTING_DEFAULT(bool, KeepDataQueryText, true); + + // Min allowed session variation coefficient (%) to start session balancing. + // Variation coefficient is a ratio of the standard deviation sigma to the mean + // Example: + // - 3 hosts with [90, 100, 110] sessions per host. Cv will be 10% + // - add new host ([90, 100, 110, 0] sessions per host). Cv will be 77% + // Balancing is will be performed if calculated cv greater than MinSessionCV + // Zero - disable this feature + FLUENT_SETTING_DEFAULT(uint32_t, MinSessionCV, 20); + + // Allow migrate requests between session during session balancing + FLUENT_SETTING_DEFAULT(bool, AllowRequestMigration, true); + + // Settings of session pool + FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); +}; + +struct TBulkUpsertSettings : public TOperationRequestSettings { + // Format setting proto serialized into string. If not set format defaults are used. + // I.e. it's Ydb.Table.CsvSettings for CSV. + FLUENT_SETTING_DEFAULT(std::string, FormatSettings, ""); +}; + +struct TReadRowsSettings : public TOperationRequestSettings { +}; + +struct TStreamExecScanQuerySettings : public TRequestSettings { + // Return query plan without actual query execution + FLUENT_SETTING_DEFAULT(bool, Explain, false); + + // Collect runtime statistics with a given detalization mode + FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); + + // Collect full query compilation diagnostics + FLUENT_SETTING_DEFAULT(bool, CollectFullDiagnostics, false); +}; + +enum class EDataFormat { + ApacheArrow = 1, + CSV = 2, +}; + +class TTableClient { + friend class TSession; + friend class TTransaction; + friend class TSessionPool; + friend class NRetry::Sync::TRetryContext; + friend class NRetry::Async::TRetryContext; + +public: + using TOperationFunc = std::function; + using TOperationSyncFunc = std::function; + using TOperationWithoutSessionFunc = std::function; + using TOperationWithoutSessionSyncFunc = std::function; + using TSettings = TClientSettings; + using TSession = TSession; + using TCreateSessionSettings = TCreateSessionSettings; + using TAsyncCreateSessionResult = TAsyncCreateSessionResult; + +public: + TTableClient(const TDriver& driver, const TClientSettings& settings = TClientSettings()); + + //! Creates new session + TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + + //! Returns session from session pool, + //! if all sessions are occupied will be generated session with CLIENT_RESOURCE_EXHAUSTED status. + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + + //! Returns number of active sessions given via session pool + int64_t GetActiveSessionCount() const; + + //! Returns the maximum number of sessions in session pool + int64_t GetActiveSessionsLimit() const; + + //! Returns the size of session pool + int64_t GetCurrentPoolSize() const; + + //! Returns new table builder + TTableBuilder GetTableBuilder(); + //! Returns new params builder + TParamsBuilder GetParamsBuilder() const; + //! Returns new type builder + TTypeBuilder GetTypeBuilder(); + + TAsyncStatus RetryOperation(TOperationFunc&& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + template + TAsyncStatus RetryOperation(std::function(TSession session)>&& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + template + TAsyncStatus RetryOperation(const std::function(TSession session)>& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + TStatus RetryOperationSync(const TOperationSyncFunc& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + TAsyncStatus RetryOperation(TOperationWithoutSessionFunc&& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + template + TAsyncStatus RetryOperation(std::function(TTableClient& tableClient)>&& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + template + TAsyncStatus RetryOperation(const std::function(TTableClient& tableClient)>& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + TStatus RetryOperationSync(const TOperationWithoutSessionSyncFunc& operation, + const TRetryOperationSettings& settings = TRetryOperationSettings()); + + //! Stop all client internal routines, drain session pools + //! Sessions returned to the session pool after this call will be closed + //! Using the client after call this method causes UB + NThreading::TFuture Stop(); + + //! Non-transactional fast bulk write. + //! Interanlly it uses an implicit session and thus doesn't need a session to be passed. + //! "rows" parameter must be a list of structs where each stuct represents one row. + //! It must contain all key columns but not necessarily all non-key columns. + //! Similar to UPSERT statement only values of specified columns will be updated. + TAsyncBulkUpsertResult BulkUpsert(const std::string& table, TValue&& rows, + const TBulkUpsertSettings& settings = TBulkUpsertSettings()); + TAsyncBulkUpsertResult BulkUpsert(const std::string& table, EDataFormat format, + const std::string& data, const std::string& schema = {}, const TBulkUpsertSettings& settings = TBulkUpsertSettings()); + + TAsyncReadRowsResult ReadRows(const std::string& table, TValue&& keys, const std::vector& columns = {}, + const TReadRowsSettings& settings = TReadRowsSettings()); + + TAsyncScanQueryPartIterator StreamExecuteScanQuery(const std::string& query, + const TStreamExecScanQuerySettings& settings = TStreamExecScanQuerySettings()); + + TAsyncScanQueryPartIterator StreamExecuteScanQuery(const std::string& query, const TParams& params, + const TStreamExecScanQuerySettings& settings = TStreamExecScanQuerySettings()); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TTxOnlineSettings { + using TSelf = TTxOnlineSettings; + + TTxOnlineSettings() {} + + FLUENT_SETTING_DEFAULT(bool, AllowInconsistentReads, false); +}; + +class TTxSettings { + friend class TTableClient; + +public: + using TSelf = TTxSettings; + + TTxSettings() + : Mode_(TS_SERIALIZABLE_RW) {} + + static TTxSettings SerializableRW() { + return TTxSettings(TS_SERIALIZABLE_RW); + } + + static TTxSettings OnlineRO(const TTxOnlineSettings& settings = TTxOnlineSettings()) { + return TTxSettings(TS_ONLINE_RO).OnlineSettings(settings); + } + + static TTxSettings StaleRO() { + return TTxSettings(TS_STALE_RO); + } + + static TTxSettings SnapshotRO() { + return TTxSettings(TS_SNAPSHOT_RO); + } + + static TTxSettings SnapshotRW() { + return TTxSettings(TS_SNAPSHOT_RW); + } + + void Out(IOutputStream& out) const { + switch (Mode_) { + case TS_SERIALIZABLE_RW: + out << "SerializableRW"; + break; + case TS_ONLINE_RO: + out << "OnlineRO"; + break; + case TS_STALE_RO: + out << "StaleRO"; + break; + case TS_SNAPSHOT_RO: + out << "SnapshotRO"; + break; + case TS_SNAPSHOT_RW: + out << "SnapshotRW"; + break; + default: + out << "Unknown"; + break; + } + } + +private: + enum ETransactionMode { + TS_SERIALIZABLE_RW, + TS_ONLINE_RO, + TS_STALE_RO, + TS_SNAPSHOT_RO, + TS_SNAPSHOT_RW, + }; + + FLUENT_SETTING(TTxOnlineSettings, OnlineSettings); + +private: + TTxSettings(ETransactionMode mode) + : Mode_(mode) {} + + ETransactionMode Mode_; +}; + +enum class EAutoPartitioningPolicy { + Disabled = 1, + AutoSplit = 2, + AutoSplitMerge = 3 +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TColumnFamilyPolicy { + using TSelf = TColumnFamilyPolicy; + + FLUENT_SETTING_OPTIONAL(std::string, Name); + + FLUENT_SETTING_OPTIONAL(std::string, Data); + + FLUENT_SETTING_OPTIONAL(std::string, External); + + FLUENT_SETTING_OPTIONAL(bool, KeepInMemory); + + FLUENT_SETTING_OPTIONAL(bool, Compressed); +}; + +struct TStoragePolicy { + using TSelf = TStoragePolicy; + + FLUENT_SETTING_OPTIONAL(std::string, PresetName); + + FLUENT_SETTING_OPTIONAL(std::string, SysLog); + + FLUENT_SETTING_OPTIONAL(std::string, Log); + + FLUENT_SETTING_OPTIONAL(std::string, Data); + + FLUENT_SETTING_OPTIONAL(std::string, External); + + FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies); +}; + +struct TPartitioningPolicy { + using TSelf = TPartitioningPolicy; + + FLUENT_SETTING_OPTIONAL(std::string, PresetName); + + FLUENT_SETTING_OPTIONAL(EAutoPartitioningPolicy, AutoPartitioning); + + FLUENT_SETTING_OPTIONAL(uint64_t, UniformPartitions); + + FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions); +}; + +struct TReplicationPolicy { + using TSelf = TReplicationPolicy; + + FLUENT_SETTING_OPTIONAL(std::string, PresetName); + + FLUENT_SETTING_OPTIONAL(uint32_t, ReplicasCount); + + FLUENT_SETTING_OPTIONAL(bool, CreatePerAvailabilityZone); + + FLUENT_SETTING_OPTIONAL(bool, AllowPromotion); +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TCreateTableSettings : public TOperationRequestSettings { + using TSelf = TCreateTableSettings; + + FLUENT_SETTING_OPTIONAL(std::string, PresetName); + + FLUENT_SETTING_OPTIONAL(std::string, ExecutionPolicy); + + FLUENT_SETTING_OPTIONAL(std::string, CompactionPolicy); + + FLUENT_SETTING_OPTIONAL(TPartitioningPolicy, PartitioningPolicy); + + FLUENT_SETTING_OPTIONAL(TStoragePolicy, StoragePolicy); + + FLUENT_SETTING_OPTIONAL(TReplicationPolicy, ReplicationPolicy); +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TDropTableSettings : public TOperationRequestSettings { + using TOperationRequestSettings::TOperationRequestSettings; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TAlterStorageSettingsBuilder { +public: + explicit TAlterStorageSettingsBuilder(TAlterTableSettings& parent) + : Parent_(parent) + { } + + TAlterStorageSettingsBuilder& SetTabletCommitLog0(const std::string& media) { + Builder_.SetTabletCommitLog0(media); + return *this; + } + + TAlterStorageSettingsBuilder& SetTabletCommitLog1(const std::string& media) { + Builder_.SetTabletCommitLog1(media); + return *this; + } + + TAlterStorageSettingsBuilder& SetExternal(const std::string& media) { + Builder_.SetExternal(media); + return *this; + } + + TAlterStorageSettingsBuilder& SetStoreExternalBlobs(bool enabled) { + Builder_.SetStoreExternalBlobs(enabled); + return *this; + } + + TAlterTableSettings& EndAlterStorageSettings(); + +private: + TAlterTableSettings& Parent_; + TStorageSettingsBuilder Builder_; +}; + +class TAlterColumnFamilyBuilder { +public: + TAlterColumnFamilyBuilder(TAlterTableSettings& parent, const std::string& name) + : Parent_(parent) + , Builder_(name) + { } + + TAlterColumnFamilyBuilder& SetData(const std::string& media) { + Builder_.SetData(media); + return *this; + } + + TAlterColumnFamilyBuilder& SetCompression(EColumnFamilyCompression compression) { + Builder_.SetCompression(compression); + return *this; + } + + TAlterColumnFamilyBuilder& SetKeepInMemory(bool enabled) { + Builder_.SetKeepInMemory(enabled); + return *this; + } + + TAlterTableSettings& EndAddColumnFamily(); + TAlterTableSettings& EndAlterColumnFamily(); + +private: + TAlterTableSettings& Parent_; + TColumnFamilyBuilder Builder_; +}; + +class TAlterTtlSettingsBuilder { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TAlterTtlSettingsBuilder(TAlterTableSettings& parent); + + TAlterTtlSettingsBuilder& Drop(); + TAlterTtlSettingsBuilder& Set(TTtlSettings&& settings); + TAlterTtlSettingsBuilder& Set(const TTtlSettings& settings); + TAlterTtlSettingsBuilder& Set(const std::string& columnName, const TDuration& expireAfter = TDuration::Zero()); + TAlterTtlSettingsBuilder& Set(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); + + TAlterTableSettings& EndAlterTtlSettings(); + +private: + TAlterTableSettings& Parent_; + + class TImpl; + std::shared_ptr Impl_; +}; + +class TAlterAttributesBuilder { +public: + TAlterAttributesBuilder(TAlterTableSettings& parent) + : Parent_(parent) + { } + + TAlterAttributesBuilder& Alter(const std::string& key, const std::string& value) { + AlterAttributes_[key] = value; + return *this; + } + + TAlterAttributesBuilder& Add(const std::string& key, const std::string& value) { + return Alter(key, value); + } + + TAlterAttributesBuilder& Drop(const std::string& key) { + return Alter(key, ""); + } + + TAlterTableSettings& EndAlterAttributes(); + +private: + TAlterTableSettings& Parent_; + std::unordered_map AlterAttributes_; +}; + +class TAlterPartitioningSettingsBuilder { +public: + explicit TAlterPartitioningSettingsBuilder(TAlterTableSettings& parent) + : Parent_(parent) + { } + + TAlterPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { + Builder_.SetPartitioningBySize(enabled); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled) { + Builder_.SetPartitioningByLoad(enabled); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetPartitionSizeMb(uint64_t sizeMb) { + Builder_.SetPartitionSizeMb(sizeMb); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetMinPartitionsCount(uint64_t count) { + Builder_.SetMinPartitionsCount(count); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetMaxPartitionsCount(uint64_t count) { + Builder_.SetMaxPartitionsCount(count); + return *this; + } + + TAlterTableSettings& EndAlterPartitioningSettings(); + +private: + TAlterTableSettings& Parent_; + TPartitioningSettingsBuilder Builder_; +}; + +struct TAlterTableSettings : public TOperationRequestSettings { + using TSelf = TAlterTableSettings; + using TAlterAttributes = std::unordered_map; + + TAlterTableSettings(); + + FLUENT_SETTING_VECTOR(TTableColumn, AddColumns); + + FLUENT_SETTING_VECTOR(std::string, DropColumns); + + FLUENT_SETTING_VECTOR(TAlterTableColumn, AlterColumns); + + FLUENT_SETTING_VECTOR(TIndexDescription, AddIndexes); + FLUENT_SETTING_VECTOR(std::string, DropIndexes); + FLUENT_SETTING_VECTOR(TRenameIndex, RenameIndexes); + + FLUENT_SETTING_VECTOR(TChangefeedDescription, AddChangefeeds); + FLUENT_SETTING_VECTOR(std::string, DropChangefeeds); + + TSelf& AlterColumnFamily(std::string name, std::string family) { + AlterColumns_.emplace_back(std::move(name), std::move(family)); + return *this; + } + + FLUENT_SETTING_OPTIONAL(TStorageSettings, AlterStorageSettings); + + FLUENT_SETTING_VECTOR(TColumnFamilyDescription, AddColumnFamilies); + FLUENT_SETTING_VECTOR(TColumnFamilyDescription, AlterColumnFamilies); + + // workaround for MSVC + TSelf& AlterTtlSettings(const std::optional& value); + const std::optional& GetAlterTtlSettings() const; + + FLUENT_SETTING(TAlterAttributes, AlterAttributes); + + FLUENT_SETTING(std::string, SetCompactionPolicy); + + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, AlterPartitioningSettings); + + FLUENT_SETTING_OPTIONAL(bool, SetKeyBloomFilter); + + FLUENT_SETTING_OPTIONAL(TReadReplicasSettings, SetReadReplicasSettings); + TSelf& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { + SetReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); + return *this; + } + + TAlterStorageSettingsBuilder BeginAlterStorageSettings() { + return TAlterStorageSettingsBuilder(*this); + } + + TAlterColumnFamilyBuilder BeginAddColumnFamily(const std::string& name) { + return TAlterColumnFamilyBuilder(*this, name); + } + + TAlterColumnFamilyBuilder BeginAlterColumnFamily(const std::string& name) { + return TAlterColumnFamilyBuilder(*this, name); + } + + TAlterTtlSettingsBuilder BeginAlterTtlSettings() { + return TAlterTtlSettingsBuilder(*this); + } + + TAlterAttributesBuilder BeginAlterAttributes() { + return TAlterAttributesBuilder(*this); + } + + TAlterPartitioningSettingsBuilder BeginAlterPartitioningSettings() { + return TAlterPartitioningSettingsBuilder(*this); + } + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +inline TAlterTableSettings& TAlterStorageSettingsBuilder::EndAlterStorageSettings() { + return Parent_.AlterStorageSettings(Builder_.Build()); +} + +inline TAlterTableSettings& TAlterColumnFamilyBuilder::EndAddColumnFamily() { + return Parent_.AppendAddColumnFamilies(Builder_.Build()); +} + +inline TAlterTableSettings& TAlterColumnFamilyBuilder::EndAlterColumnFamily() { + return Parent_.AppendAlterColumnFamilies(Builder_.Build()); +} + +inline TAlterTableSettings& TAlterAttributesBuilder::EndAlterAttributes() { + return Parent_.AlterAttributes(AlterAttributes_); +} + +inline TAlterTableSettings& TAlterPartitioningSettingsBuilder::EndAlterPartitioningSettings() { + return Parent_.AlterPartitioningSettings(Builder_.Build()); +} + +//////////////////////////////////////////////////////////////////////////////// + +struct TCopyTableSettings : public TOperationRequestSettings {}; + +struct TCopyTablesSettings : public TOperationRequestSettings {}; + +struct TRenameTablesSettings : public TOperationRequestSettings {}; + +struct TDescribeTableSettings : public TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(bool, WithKeyShardBoundary, false); + FLUENT_SETTING_DEFAULT(bool, WithTableStatistics, false); + FLUENT_SETTING_DEFAULT(bool, WithPartitionStatistics, false); + FLUENT_SETTING_DEFAULT(bool, WithSetVal, false); + FLUENT_SETTING_DEFAULT(bool, WithShardNodesInfo, false); +}; + +struct TExplainDataQuerySettings : public TOperationRequestSettings { + FLUENT_SETTING_DEFAULT(bool, WithCollectFullDiagnostics, false); +}; + +struct TPrepareDataQuerySettings : public TOperationRequestSettings {}; + +struct TExecDataQuerySettings : public TOperationRequestSettings { + FLUENT_SETTING_OPTIONAL(bool, KeepInQueryCache); + + FLUENT_SETTING_OPTIONAL(ECollectQueryStatsMode, CollectQueryStats); +}; + +struct TExecSchemeQuerySettings : public TOperationRequestSettings {}; + +struct TBeginTxSettings : public TOperationRequestSettings {}; + +struct TCommitTxSettings : public TOperationRequestSettings { + FLUENT_SETTING_OPTIONAL(ECollectQueryStatsMode, CollectQueryStats); +}; + +struct TRollbackTxSettings : public TOperationRequestSettings {}; + +struct TCloseSessionSettings : public TOperationRequestSettings {}; + +struct TKeepAliveSettings : public TOperationRequestSettings {}; + +struct TReadTableSettings : public TRequestSettings { + + using TSelf = TReadTableSettings; + + FLUENT_SETTING_OPTIONAL(TKeyBound, From); + + FLUENT_SETTING_OPTIONAL(TKeyBound, To); + + FLUENT_SETTING_VECTOR(std::string, Columns); + + FLUENT_SETTING_FLAG(Ordered); + + FLUENT_SETTING_OPTIONAL(uint64_t, RowLimit); + + FLUENT_SETTING_OPTIONAL(bool, UseSnapshot); + + FLUENT_SETTING_OPTIONAL(uint64_t, BatchLimitBytes); + + FLUENT_SETTING_OPTIONAL(uint64_t, BatchLimitRows); + + FLUENT_SETTING_OPTIONAL(bool, ReturnNotNullAsOptional); +}; + +using TPrecommitTransactionCallback = std::function; + +//! Represents all session operations +//! Session is transparent logic representation of connection +class TSession { + friend class TTableClient; + friend class TDataQuery; + friend class TTransaction; + friend class TSessionPool; + +public: + //! The following methods perform corresponding calls. + //! Results are NThreading::TFuture where T is corresponding result. + TAsyncStatus CreateTable(const std::string& path, TTableDescription&& tableDesc, + const TCreateTableSettings& settings = TCreateTableSettings()); + + TAsyncStatus DropTable(const std::string& path, const TDropTableSettings& settings = TDropTableSettings()); + + TAsyncStatus AlterTable(const std::string& path, const TAlterTableSettings& settings = TAlterTableSettings()); + + // Same as AlterTable but may return operation in case of long running + TAsyncOperation AlterTableLong(const std::string& path, const TAlterTableSettings& settings = TAlterTableSettings()); + + TAsyncStatus CopyTable(const std::string& src, const std::string& dst, + const TCopyTableSettings& settings = TCopyTableSettings()); + + TAsyncStatus CopyTables(const std::vector& copyItems, + const TCopyTablesSettings& settings = TCopyTablesSettings()); + + TAsyncStatus RenameTables(const std::vector& renameItems, + const TRenameTablesSettings& settings = TRenameTablesSettings()); + + TAsyncDescribeTableResult DescribeTable(const std::string& path, + const TDescribeTableSettings& settings = TDescribeTableSettings()); + + TAsyncBeginTransactionResult BeginTransaction(const TTxSettings& txSettings = TTxSettings(), + const TBeginTxSettings& settings = TBeginTxSettings()); + + TAsyncExplainDataQueryResult ExplainDataQuery(const std::string& query, + const TExplainDataQuerySettings& settings = TExplainDataQuerySettings()); + + TAsyncPrepareQueryResult PrepareDataQuery(const std::string& query, + const TPrepareDataQuerySettings& settings = TPrepareDataQuerySettings()); + + TAsyncDataQueryResult ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + + TAsyncDataQueryResult ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + + TAsyncDataQueryResult ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + TParams&& params, const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + + TAsyncStatus ExecuteSchemeQuery(const std::string& query, + const TExecSchemeQuerySettings& settings = TExecSchemeQuerySettings()); + + TAsyncTablePartIterator ReadTable(const std::string& path, + const TReadTableSettings& settings = TReadTableSettings()); + + TAsyncStatus Close(const TCloseSessionSettings& settings = TCloseSessionSettings()); + + TAsyncKeepAliveResult KeepAlive(const TKeepAliveSettings& settings = TKeepAliveSettings()); + + void InvalidateQueryCache(); + + //! Returns new table builder + TTableBuilder GetTableBuilder(); + //! Returns new params builder + TParamsBuilder GetParamsBuilder(); + //! Returns new type builder + TTypeBuilder GetTypeBuilder(); + //! Returns session id + const std::string& GetId() const; + + class TImpl; +private: + TSession(std::shared_ptr client, const std::string& sessionId, const std::string& endpointId, bool isOwnedBySessionPool); + TSession(std::shared_ptr client, std::shared_ptr SessionImpl_); + + std::shared_ptr Client_; + std::shared_ptr SessionImpl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template +TAsyncStatus TTableClient::RetryOperation( + std::function(TSession session)>&& operation, + const TRetryOperationSettings& settings) +{ + return RetryOperation([operation = std::move(operation)] (TSession session) { + return operation(session).Apply([] (const NThreading::TFuture& result) { + return NThreading::MakeFuture(result.GetValue()); + }); + }, settings); +} + +template +TAsyncStatus TTableClient::RetryOperation( + const std::function(TSession session)>& operation, + const TRetryOperationSettings& settings) +{ + return RetryOperation([operation] (TSession session) { + return operation(session).Apply([] (const NThreading::TFuture& result) { + return NThreading::MakeFuture(result.GetValue()); + }); + }, settings); +} + +template +TAsyncStatus TTableClient::RetryOperation( + std::function(TTableClient& tableClient)>&& operation, + const TRetryOperationSettings& settings) +{ + return RetryOperation([operation = std::move(operation)] (TTableClient& tableClient) { + return operation(tableClient).Apply([] (const NThreading::TFuture& result) { + return NThreading::MakeFuture(result.GetValue()); + }); + }, settings); +} + +template +TAsyncStatus TTableClient::RetryOperation( + const std::function(TTableClient& tableClient)>& operation, + const TRetryOperationSettings& settings) +{ + return RetryOperation([operation] (TTableClient& tableClient) { + return operation(tableClient).Apply([] (const NThreading::TFuture& result) { + return NThreading::MakeFuture(result.GetValue()); + }); + }, settings); +} + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents data transaction +class TTransaction { + friend class TTableClient; +public: + const std::string& GetId() const; + bool IsActive() const; + + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); + TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); + + TSession GetSession() const; + void AddPrecommitCallback(TPrecommitTransactionCallback cb); + +private: + TTransaction(const TSession& session, const std::string& txId); + + TAsyncStatus Precommit() const; + + class TImpl; + + std::shared_ptr TransactionImpl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TTxControl { + friend class TTableClient; + +public: + using TSelf = TTxControl; + + static TTxControl Tx(const TTransaction& tx) { + return TTxControl(tx); + } + + static TTxControl BeginTx(const TTxSettings& settings = TTxSettings()) { + return TTxControl(settings); + } + + FLUENT_SETTING_FLAG(CommitTx); + +private: + TTxControl(const TTransaction& tx); + TTxControl(const TTxSettings& begin); + +private: + std::optional Tx_; + TTxSettings BeginTx_; +}; + +//! Represents query identificator (e.g. used for prepared query) +class TDataQuery { + friend class TTableClient; + friend class TSession; + +public: + const std::string& GetId() const; + const std::optional& GetText() const; + TParamsBuilder GetParamsBuilder() const; + + TAsyncDataQueryResult Execute(const TTxControl& txControl, + const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + + TAsyncDataQueryResult Execute(const TTxControl& txControl, const TParams& params, + const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + + TAsyncDataQueryResult Execute(const TTxControl& txControl, TParams&& params, + const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + +private: + TDataQuery(const TSession& session, const std::string& text, const std::string& id); + TDataQuery(const TSession& session, const std::string& text, const std::string& id, + const ::google::protobuf::Map& types); + + class TImpl; + std::shared_ptr Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents result of PrepareDataQuery call. +//! If result is successful TDataQuery should be used for next +//! ExecuteDataQuery calls +class TPrepareQueryResult : public TStatus { +public: + TPrepareQueryResult(TStatus&& status, const TDataQuery& query, bool fromCache); + + TDataQuery GetQuery() const; + bool IsQueryFromCache() const; + +private: + TDataQuery PreparedQuery_; + bool FromCache_; +}; + +//! Represents result of ExplainDataQuery call +class TExplainQueryResult : public TStatus { +public: + TExplainQueryResult(TStatus&& status, std::string&& plan, std::string&& ast, std::string&& diagnostics); + + const std::string& GetPlan() const; + const std::string& GetAst() const; + const std::string& GetDiagnostics() const; + +private: + std::string Plan_; + std::string Ast_; + std::string Diagnostics_; +}; + +//! Represents result of DescribeTable call +class TDescribeTableResult : public NScheme::TDescribePathResult { +public: + TDescribeTableResult(TStatus&& status, Ydb::Table::DescribeTableResult&& desc, + const TDescribeTableSettings& describeSettings); + + TTableDescription GetTableDescription() const; + +private: + TTableDescription TableDescription_; +}; + +class TDataQueryResult : public TStatus { +public: + TDataQueryResult(TStatus&& status, std::vector&& resultSets, const std::optional& transaction, + const std::optional& dataQuery, bool fromCache, const std::optional& queryStats); + + const std::vector& GetResultSets() const; + std::vector ExtractResultSets() &&; + TResultSet GetResultSet(size_t resultIndex) const; + + TResultSetParser GetResultSetParser(size_t resultIndex) const; + + std::optional GetTransaction() const; + + std::optional GetQuery() const; + bool IsQueryFromCache() const; + + const std::optional& GetStats() const; + + const std::string GetQueryPlan() const; + +private: + std::optional Transaction_; + std::vector ResultSets_; + std::optional DataQuery_; + bool FromCache_; + std::optional QueryStats_; +}; + +class TReadTableSnapshot { +public: + TReadTableSnapshot(uint64_t step, uint64_t txId) + : Step_(step) + , TxId_(txId) + {} + + uint64_t GetStep() const { return Step_; } + uint64_t GetTxId() const { return TxId_; } + +private: + uint64_t Step_; + uint64_t TxId_; +}; + +template +class TSimpleStreamPart : public TStreamPartStatus { +public: + const TPart& GetPart() const { return Part_; } + + TPart ExtractPart() { return std::move(Part_); } + + TSimpleStreamPart(TPart&& part, TStatus&& status, + std::optional snapshot = std::nullopt) + : TStreamPartStatus(std::move(status)) + , Part_(std::move(part)) + , Snapshot_(std::move(snapshot)) + {} + + const std::optional& GetSnapshot() { return Snapshot_; } + +private: + TPart Part_; + std::optional Snapshot_; +}; + +template +using TAsyncSimpleStreamPart = NThreading::TFuture>; + +class TTablePartIterator : public TStatus { + friend class TSession; +public: + TAsyncSimpleStreamPart ReadNext(); + class TReaderImpl; +private: + TTablePartIterator( + std::shared_ptr impl, + TPlainStatus&& status + ); + std::shared_ptr ReaderImpl_; +}; + +using TReadTableResultPart = TSimpleStreamPart; + +class TScanQueryPart : public TStreamPartStatus { +public: + bool HasResultSet() const { return ResultSet_.has_value(); } + const TResultSet& GetResultSet() const { return *ResultSet_; } + TResultSet ExtractResultSet() { return std::move(*ResultSet_); } + + bool HasQueryStats() const { return QueryStats_.has_value(); } + const TQueryStats& GetQueryStats() const { return *QueryStats_; } + TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } + + bool HasDiagnostics() const { return Diagnostics_.has_value(); } + const std::string& GetDiagnostics() const { return *Diagnostics_; } + std::string&& ExtractDiagnostics() { return std::move(*Diagnostics_); } + + TScanQueryPart(TStatus&& status) + : TStreamPartStatus(std::move(status)) + {} + + TScanQueryPart(TStatus&& status, const std::optional& queryStats, const std::optional& diagnostics) + : TStreamPartStatus(std::move(status)) + , QueryStats_(queryStats) + , Diagnostics_(diagnostics) + {} + + TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const std::optional& queryStats, const std::optional& diagnostics) + : TStreamPartStatus(std::move(status)) + , ResultSet_(std::move(resultSet)) + , QueryStats_(queryStats) + , Diagnostics_(diagnostics) + {} + +private: + std::optional ResultSet_; + std::optional QueryStats_; + std::optional Diagnostics_; +}; + +using TAsyncScanQueryPart = NThreading::TFuture; + +class TScanQueryPartIterator : public TStatus { + friend class TTableClient; +public: + TAsyncScanQueryPart ReadNext(); + class TReaderImpl; +private: + TScanQueryPartIterator( + std::shared_ptr impl, + TPlainStatus&& status + ); + std::shared_ptr ReaderImpl_; +}; + +class TBeginTransactionResult : public TStatus { +public: + TBeginTransactionResult(TStatus&& status, TTransaction transaction); + + const TTransaction& GetTransaction() const; + +private: + TTransaction Transaction_; +}; + +class TCommitTransactionResult : public TStatus { +public: + TCommitTransactionResult(TStatus&& status, const std::optional& queryStats); + + const std::optional& GetStats() const; + +private: + std::optional QueryStats_; +}; + +class TCreateSessionResult: public TStatus { + friend class TSession::TImpl; +public: + TCreateSessionResult(TStatus&& status, TSession&& session); + TSession GetSession() const; + +private: + TSession Session_; +}; + +enum class ESessionStatus { + Unspecified = 0, + Ready = 1, + Busy = 2 +}; + +class TKeepAliveResult : public TStatus { +public: + TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus); + ESessionStatus GetSessionStatus() const; +private: + ESessionStatus SessionStatus; +}; + +class TBulkUpsertResult : public TStatus { +public: + explicit TBulkUpsertResult(TStatus&& status); +}; + +class TReadRowsResult : public TStatus { + TResultSet ResultSet; + + public: + explicit TReadRowsResult(TStatus&& status, TResultSet&& resultSet); + + TResultSet GetResultSet() { + return std::move(ResultSet); + } +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table_enum.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table_enum.h new file mode 100644 index 000000000000..7c5197a84dbf --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table_enum.h @@ -0,0 +1,63 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +//! Column family compression codec +enum class EColumnFamilyCompression { + None, + LZ4, +}; + +//! State of build index operation +enum class EBuildIndexState { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + Applying = 3, + Done = 4, + Cancellation = 5, + Cancelled = 6, + Rejection = 7, + Rejected = 8, +}; + +enum class EIndexType { + GlobalSync, + GlobalAsync, + GlobalUnique, + GlobalVectorKMeansTree, + + Unknown = std::numeric_limits::max() +}; + +enum class EChangefeedMode { + KeysOnly /* "KEYS_ONLY" */, + Updates /* "UPDATES" */, + NewImage /* "NEW_IMAGE" */, + OldImage /* "OLD_IMAGE" */, + NewAndOldImages /* "NEW_AND_OLD_IMAGES" */, + + Unknown = std::numeric_limits::max() +}; + +enum class EChangefeedFormat { + Json /* "JSON" */, + DynamoDBStreamsJson /* "DYNAMODB_STREAMS_JSON" */, + DebeziumJson /* "DEBEZIUM_JSON" */, + + Unknown = std::numeric_limits::max() +}; + +enum class EChangefeedState { + Enabled, + Disabled, + InitialScan, + + Unknown = std::numeric_limits::max() +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h new file mode 100644 index 000000000000..1b1ea70c8079 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h @@ -0,0 +1,64 @@ +#pragma once + +#include "control_plane.h" +#include "read_session.h" +#include "write_session.h" + +namespace NYdb::inline V3::NTopic { + +struct TTopicClientSettings : public TCommonClientSettingsBase { + using TSelf = TTopicClientSettings; + + //! Default executor for compression tasks. + FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); + + //! Default executor for callbacks. + FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); +}; + +// Topic client. +class TTopicClient { +public: + class TImpl; + + TTopicClient(const TDriver& driver, const TTopicClientSettings& settings = TTopicClientSettings()); + + void ProvideCodec(ECodec codecId, std::unique_ptr&& codecImpl); + + // Create a new topic. + TAsyncStatus CreateTopic(const std::string& path, const TCreateTopicSettings& settings = {}); + + // Update a topic. + TAsyncStatus AlterTopic(const std::string& path, const TAlterTopicSettings& settings = {}); + + // Delete a topic. + TAsyncStatus DropTopic(const std::string& path, const TDropTopicSettings& settings = {}); + + // Describe a topic. + TAsyncDescribeTopicResult DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings = {}); + + // Describe a topic consumer. + TAsyncDescribeConsumerResult DescribeConsumer(const std::string& path, const std::string& consumer, const TDescribeConsumerSettings& settings = {}); + + // Describe a topic partition + TAsyncDescribePartitionResult DescribePartition(const std::string& path, int64_t partitionId, const TDescribePartitionSettings& settings = {}); + + //! Create read session. + std::shared_ptr CreateReadSession(const TReadSessionSettings& settings); + + //! Create write session. + std::shared_ptr CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TWriteSessionSettings& settings); + + // Commit offset + TAsyncStatus CommitOffset(const std::string& path, uint64_t partitionId, const std::string& consumerName, uint64_t offset, + const TCommitOffsetSettings& settings = {}); + +protected: + void OverrideCodec(ECodec codecId, std::unique_ptr&& codecImpl); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/codecs.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/codecs.h new file mode 100644 index 000000000000..a91c87afee11 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/codecs.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace NYdb::inline V3::NTopic { + +enum class ECodec : uint32_t { + RAW = 1, + GZIP = 2, + LZOP = 3, + ZSTD = 4, + CUSTOM = 10000, +}; + +inline const std::string& GetCodecId(const ECodec codec) { + static std::unordered_map idByCodec{ + {ECodec::RAW, std::string(1, '\0')}, + {ECodec::GZIP, "\1"}, + {ECodec::LZOP, "\2"}, + {ECodec::ZSTD, "\3"} + }; + Y_ABORT_UNLESS(idByCodec.contains(codec)); + return idByCodec[codec]; +} + +class ICodec { +public: + virtual ~ICodec() = default; + virtual std::string Decompress(const std::string& data) const = 0; + virtual std::unique_ptr CreateCoder(TBuffer& result, int quality) const = 0; +}; + +class TGzipCodec final : public ICodec { + std::string Decompress(const std::string& data) const override; + + std::unique_ptr CreateCoder(TBuffer& result, int quality) const override; +}; + +class TZstdCodec final : public ICodec { + std::string Decompress(const std::string& data) const override; + + std::unique_ptr CreateCoder(TBuffer& result, int quality) const override; +}; + +class TUnsupportedCodec final : public ICodec { + std::string Decompress(const std::string&) const override; + + std::unique_ptr CreateCoder(TBuffer&, int) const override; +}; + +class TCodecMap { +public: + static TCodecMap& GetTheCodecMap() { + static TCodecMap instance; + return instance; + } + + void Set(uint32_t codecId, std::unique_ptr&& codecImpl) { + with_lock(Lock) { + Codecs[codecId] = std::move(codecImpl); + } + } + + const ICodec* GetOrThrow(uint32_t codecId) const { + with_lock(Lock) { + if (!Codecs.contains(codecId)) { + throw yexception() << "codec with id " << uint32_t(codecId) << " not provided"; + } + return Codecs.at(codecId).get(); + } + } + + + TCodecMap(const TCodecMap&) = delete; + TCodecMap(TCodecMap&&) = delete; + TCodecMap& operator=(const TCodecMap&) = delete; + TCodecMap& operator=(TCodecMap&&) = delete; + +private: + TCodecMap() = default; + +private: + std::unordered_map> Codecs; + TAdaptiveLock Lock; +}; + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h new file mode 100644 index 000000000000..c9a96da3fe4e --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h @@ -0,0 +1,759 @@ +#pragma once + +#include "codecs.h" + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { + class TProtoAccessor; + + namespace NScheme { + struct TPermissions; + } +} + +namespace NYdb::inline V3::NTopic { + +enum class EMeteringMode : uint32_t { + Unspecified = 0, + ReservedCapacity = 1, + RequestUnits = 2, + + Unknown = std::numeric_limits::max(), +}; + +enum class EAutoPartitioningStrategy: uint32_t { + Unspecified = 0, + Disabled = 1, + ScaleUp = 2, + ScaleUpAndDown = 3, + Paused = 4, +}; + +class TConsumer { +public: + TConsumer(const Ydb::Topic::Consumer&); + + const std::string& GetConsumerName() const; + bool GetImportant() const; + const TInstant& GetReadFrom() const; + const std::vector& GetSupportedCodecs() const; + const std::map& GetAttributes() const; + +private: + std::string ConsumerName_; + bool Important_; + TInstant ReadFrom_; + std::map Attributes_; + std::vector SupportedCodecs_; + +}; + + +class TTopicStats { +public: + TTopicStats(const Ydb::Topic::DescribeTopicResult::TopicStats& topicStats); + + uint64_t GetStoreSizeBytes() const; + TDuration GetMaxWriteTimeLag() const; + TInstant GetMinLastWriteTime() const; + uint64_t GetBytesWrittenPerMinute() const; + uint64_t GetBytesWrittenPerHour() const; + uint64_t GetBytesWrittenPerDay() const; + +private: + uint64_t StoreSizeBytes_; + TInstant MinLastWriteTime_; + TDuration MaxWriteTimeLag_; + uint64_t BytesWrittenPerMinute_; + uint64_t BytesWrittenPerHour_; + uint64_t BytesWrittenPerDay_; +}; + + +class TPartitionStats { +public: + TPartitionStats(const Ydb::Topic::PartitionStats& partitionStats); + + uint64_t GetStartOffset() const; + uint64_t GetEndOffset() const; + uint64_t GetStoreSizeBytes() const; + TDuration GetMaxWriteTimeLag() const; + TInstant GetLastWriteTime() const; + uint64_t GetBytesWrittenPerMinute() const; + uint64_t GetBytesWrittenPerHour() const; + uint64_t GetBytesWrittenPerDay() const; + +private: + uint64_t StartOffset_; + uint64_t EndOffset_; + uint64_t StoreSizeBytes_; + TInstant LastWriteTime_; + TDuration MaxWriteTimeLag_; + uint64_t BytesWrittenPerMinute_; + uint64_t BytesWrittenPerHour_; + uint64_t BytesWrittenPerDay_; +}; + +class TPartitionConsumerStats { +public: + TPartitionConsumerStats(const Ydb::Topic::DescribeConsumerResult::PartitionConsumerStats& partitionStats); + uint64_t GetCommittedOffset() const; + uint64_t GetLastReadOffset() const; + std::string GetReaderName() const; + std::string GetReadSessionId() const; + const TInstant& GetLastReadTime() const; + const TDuration& GetMaxReadTimeLag() const; + const TDuration& GetMaxWriteTimeLag() const; + +private: + uint64_t CommittedOffset_; + int64_t LastReadOffset_; + std::string ReaderName_; + std::string ReadSessionId_; + TInstant LastReadTime_; + TDuration MaxReadTimeLag_; + TDuration MaxWriteTimeLag_; +}; + +// Topic partition location +class TPartitionLocation { +public: + TPartitionLocation(const Ydb::Topic::PartitionLocation& partitionLocation); + int32_t GetNodeId() const; + int64_t GetGeneration() const; + +private: + // Node identificator. + int32_t NodeId_ = 1; + + // Partition generation. + int64_t Generation_ = 2; +}; + +class TPartitionInfo { +public: + TPartitionInfo(const Ydb::Topic::DescribeTopicResult::PartitionInfo& partitionInfo); + TPartitionInfo(const Ydb::Topic::DescribeConsumerResult::PartitionInfo& partitionInfo); + + uint64_t GetPartitionId() const; + bool GetActive() const; + const std::vector GetChildPartitionIds() const; + const std::vector GetParentPartitionIds() const; + + const std::optional& GetPartitionStats() const; + const std::optional& GetPartitionConsumerStats() const; + const std::optional& GetPartitionLocation() const; + + const std::optional& GetFromBound() const; + const std::optional& GetToBound() const; + +private: + uint64_t PartitionId_; + bool Active_; + std::vector ChildPartitionIds_; + std::vector ParentPartitionIds_; + + std::optional PartitionStats_; + std::optional PartitionConsumerStats_; + std::optional PartitionLocation_; + + std::optional FromBound_; + std::optional ToBound_; +}; + +struct TAlterPartitioningSettings; +struct TAlterTopicSettings; + +struct TAutoPartitioningSettings { +friend struct TAutoPartitioningSettingsBuilder; +public: + TAutoPartitioningSettings() + : Strategy_(EAutoPartitioningStrategy::Disabled) + , StabilizationWindow_(TDuration::Seconds(0)) + , DownUtilizationPercent_(0) + , UpUtilizationPercent_(0) { + } + TAutoPartitioningSettings(const Ydb::Topic::AutoPartitioningSettings& settings); + TAutoPartitioningSettings(EAutoPartitioningStrategy strategy, TDuration stabilizationWindow, ui64 downUtilizationPercent, ui64 upUtilizationPercent) + : Strategy_(strategy) + , StabilizationWindow_(stabilizationWindow) + , DownUtilizationPercent_(downUtilizationPercent) + , UpUtilizationPercent_(upUtilizationPercent) {} + + EAutoPartitioningStrategy GetStrategy() const; + TDuration GetStabilizationWindow() const; + ui32 GetDownUtilizationPercent() const; + ui32 GetUpUtilizationPercent() const; +private: + EAutoPartitioningStrategy Strategy_; + TDuration StabilizationWindow_; + ui32 DownUtilizationPercent_; + ui32 UpUtilizationPercent_; +}; + +struct TAlterAutoPartitioningSettings { + using TSelf = TAlterAutoPartitioningSettings; +public: + TAlterAutoPartitioningSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} + + FLUENT_SETTING_OPTIONAL(EAutoPartitioningStrategy, Strategy); + FLUENT_SETTING_OPTIONAL(TDuration, StabilizationWindow); + FLUENT_SETTING_OPTIONAL(ui64, DownUtilizationPercent); + FLUENT_SETTING_OPTIONAL(ui64, UpUtilizationPercent); + + TAlterPartitioningSettings& EndAlterAutoPartitioningSettings() { return Parent_; }; + +private: + TAlterPartitioningSettings& Parent_; +}; + +class TPartitioningSettings { + using TSelf = TPartitioningSettings; + friend struct TPartitioningSettingsBuilder; +public: + TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), PartitionCountLimit_(0), AutoPartitioningSettings_(){} + TPartitioningSettings(const Ydb::Topic::PartitioningSettings& settings); + TPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions, TAutoPartitioningSettings autoPartitioning = {}) + : MinActivePartitions_(minActivePartitions) + , MaxActivePartitions_(maxActivePartitions) + , PartitionCountLimit_(0) + , AutoPartitioningSettings_(autoPartitioning) + { + } + + uint64_t GetMinActivePartitions() const; + uint64_t GetMaxActivePartitions() const; + uint64_t GetPartitionCountLimit() const; + TAutoPartitioningSettings GetAutoPartitioningSettings() const; +private: + uint64_t MinActivePartitions_; + uint64_t MaxActivePartitions_; + uint64_t PartitionCountLimit_; + TAutoPartitioningSettings AutoPartitioningSettings_; +}; + +struct TAlterTopicSettings; + +struct TAlterPartitioningSettings { + using TSelf = TAlterPartitioningSettings; +public: + TAlterPartitioningSettings(TAlterTopicSettings& parent): Parent_(parent) {} + + FLUENT_SETTING_OPTIONAL(uint64_t, MinActivePartitions); + FLUENT_SETTING_OPTIONAL(uint64_t, MaxActivePartitions); + + TAlterTopicSettings& EndAlterTopicPartitioningSettings() { return Parent_; }; + + TAlterAutoPartitioningSettings& BeginAlterAutoPartitioningSettings() { + AutoPartitioningSettings_.emplace(*this); + return *AutoPartitioningSettings_; + } + + std::optional AutoPartitioningSettings_; + +private: + TAlterTopicSettings& Parent_; +}; + +class TTopicDescription { + friend class NYdb::V3::TProtoAccessor; + +public: + TTopicDescription(Ydb::Topic::DescribeTopicResult&& desc); + + const std::string& GetOwner() const; + + const NScheme::TVirtualTimestamp& GetCreationTimestamp() const; + + const std::vector& GetPermissions() const; + + const std::vector& GetEffectivePermissions() const; + + const TPartitioningSettings& GetPartitioningSettings() const; + + uint32_t GetTotalPartitionsCount() const; + + const std::vector& GetPartitions() const; + + const std::vector& GetSupportedCodecs() const; + + const TDuration& GetRetentionPeriod() const; + + std::optional GetRetentionStorageMb() const; + + uint64_t GetPartitionWriteSpeedBytesPerSecond() const; + + uint64_t GetPartitionWriteBurstBytes() const; + + const std::map& GetAttributes() const; + + const std::vector& GetConsumers() const; + + EMeteringMode GetMeteringMode() const; + + const TTopicStats& GetTopicStats() const; + + void SerializeTo(Ydb::Topic::CreateTopicRequest& request) const; +private: + + const Ydb::Topic::DescribeTopicResult& GetProto() const; + + const Ydb::Topic::DescribeTopicResult Proto_; + std::vector Partitions_; + std::vector SupportedCodecs_; + TPartitioningSettings PartitioningSettings_; + TDuration RetentionPeriod_; + std::optional RetentionStorageMb_; + uint64_t PartitionWriteSpeedBytesPerSecond_; + uint64_t PartitionWriteBurstBytes_; + EMeteringMode MeteringMode_; + std::map Attributes_; + std::vector Consumers_; + + TTopicStats TopicStats_; + + std::string Owner_; + NScheme::TVirtualTimestamp CreationTimestamp_; + std::vector Permissions_; + std::vector EffectivePermissions_; +}; + +class TConsumerDescription { + friend class NYdb::V3::TProtoAccessor; + +public: + TConsumerDescription(Ydb::Topic::DescribeConsumerResult&& desc); + + const std::vector& GetPartitions() const; + + const TConsumer& GetConsumer() const; + +private: + + const Ydb::Topic::DescribeConsumerResult& GetProto() const; + + + const Ydb::Topic::DescribeConsumerResult Proto_; + std::vector Partitions_; + TConsumer Consumer_; +}; + +class TPartitionDescription { + friend class NYdb::V3::TProtoAccessor; + +public: + TPartitionDescription(Ydb::Topic::DescribePartitionResult&& desc); + + const TPartitionInfo& GetPartition() const; +private: + const Ydb::Topic::DescribePartitionResult& GetProto() const; + + const Ydb::Topic::DescribePartitionResult Proto_; + TPartitionInfo Partition_; +}; + +// Result for describe topic request. +struct TDescribeTopicResult : public TStatus { + friend class NYdb::V3::TProtoAccessor; + + TDescribeTopicResult(TStatus&& status, Ydb::Topic::DescribeTopicResult&& result); + + const TTopicDescription& GetTopicDescription() const; + +private: + TTopicDescription TopicDescription_; +}; + +// Result for describe consumer request. +struct TDescribeConsumerResult : public TStatus { + friend class NYdb::V3::TProtoAccessor; + + TDescribeConsumerResult(TStatus&& status, Ydb::Topic::DescribeConsumerResult&& result); + + const TConsumerDescription& GetConsumerDescription() const; + +private: + TConsumerDescription ConsumerDescription_; +}; + +// Result for describe partition request. +struct TDescribePartitionResult: public TStatus { + friend class NYdb::V3::TProtoAccessor; + + TDescribePartitionResult(TStatus&& status, Ydb::Topic::DescribePartitionResult&& result); + + const TPartitionDescription& GetPartitionDescription() const; + +private: + TPartitionDescription PartitionDescription_; +}; + +using TAsyncDescribeTopicResult = NThreading::TFuture; +using TAsyncDescribeConsumerResult = NThreading::TFuture; +using TAsyncDescribePartitionResult = NThreading::TFuture; + +template +class TAlterAttributesBuilderImpl { +public: + TAlterAttributesBuilderImpl(TSettings& parent) + : Parent_(parent) + { } + + TAlterAttributesBuilderImpl& Alter(const std::string& key, const std::string& value) { + Parent_.AlterAttributes_[key] = value; + return *this; + } + + TAlterAttributesBuilderImpl& Add(const std::string& key, const std::string& value) { + return Alter(key, value); + } + + TAlterAttributesBuilderImpl& Drop(const std::string& key) { + return Alter(key, ""); + } + + TSettings& EndAlterAttributes() { return Parent_; } + +private: + TSettings& Parent_; +}; + +struct TAlterConsumerSettings; +struct TAlterTopicSettings; + +using TAlterConsumerAttributesBuilder = TAlterAttributesBuilderImpl; +using TAlterTopicAttributesBuilder = TAlterAttributesBuilderImpl; + +template +struct TConsumerSettings { + using TSelf = TConsumerSettings; + + using TAttributes = std::map; + + TConsumerSettings(TSettings& parent): Parent_(parent) {} + TConsumerSettings(TSettings& parent, const std::string& name) : ConsumerName_(name), Parent_(parent) {} + + FLUENT_SETTING(std::string, ConsumerName); + FLUENT_SETTING_DEFAULT(bool, Important, false); + FLUENT_SETTING_DEFAULT(TInstant, ReadFrom, TInstant::Zero()); + + FLUENT_SETTING_VECTOR(ECodec, SupportedCodecs); + + FLUENT_SETTING(TAttributes, Attributes); + + TConsumerSettings& AddAttribute(const std::string& key, const std::string& value) { + Attributes_[key] = value; + return *this; + } + + TConsumerSettings& SetAttributes(std::map&& attributes) { + Attributes_ = std::move(attributes); + return *this; + } + + TConsumerSettings& SetAttributes(const std::map& attributes) { + Attributes_ = attributes; + return *this; + } + + TConsumerSettings& SetSupportedCodecs(std::vector&& codecs) { + SupportedCodecs_ = std::move(codecs); + return *this; + } + + TConsumerSettings& SetSupportedCodecs(const std::vector& codecs) { + SupportedCodecs_ = codecs; + return *this; + } + + TConsumerSettings& SetImportant(bool isImportant) { + Important_ = isImportant; + return *this; + } + + TSettings& EndAddConsumer() { return Parent_; }; + +private: + TSettings& Parent_; +}; + +struct TAlterConsumerSettings { + using TSelf = TAlterConsumerSettings; + + using TAlterAttributes = std::map; + + TAlterConsumerSettings(TAlterTopicSettings& parent): Parent_(parent) {} + TAlterConsumerSettings(TAlterTopicSettings& parent, const std::string& name) : ConsumerName_(name), Parent_(parent) {} + + FLUENT_SETTING(std::string, ConsumerName); + FLUENT_SETTING_OPTIONAL(bool, SetImportant); + FLUENT_SETTING_OPTIONAL(TInstant, SetReadFrom); + + FLUENT_SETTING_OPTIONAL_VECTOR(ECodec, SetSupportedCodecs); + + FLUENT_SETTING(TAlterAttributes, AlterAttributes); + + TAlterConsumerAttributesBuilder BeginAlterAttributes() { + return TAlterConsumerAttributesBuilder(*this); + } + + TAlterConsumerSettings& SetSupportedCodecs(std::vector&& codecs) { + SetSupportedCodecs_ = std::move(codecs); + return *this; + } + + TAlterConsumerSettings& SetSupportedCodecs(const std::vector& codecs) { + SetSupportedCodecs_ = codecs; + return *this; + } + + TAlterTopicSettings& EndAlterConsumer() { return Parent_; }; + +private: + TAlterTopicSettings& Parent_; +}; + +struct TPartitioningSettingsBuilder; +struct TCreateTopicSettings : public TOperationRequestSettings { + + using TSelf = TCreateTopicSettings; + using TAttributes = std::map; + + FLUENT_SETTING(TPartitioningSettings, PartitioningSettings); + + FLUENT_SETTING_DEFAULT(TDuration, RetentionPeriod, TDuration::Hours(24)); + + FLUENT_SETTING_VECTOR(ECodec, SupportedCodecs); + + FLUENT_SETTING_DEFAULT(uint64_t, RetentionStorageMb, 0); + FLUENT_SETTING_DEFAULT(EMeteringMode, MeteringMode, EMeteringMode::Unspecified); + + FLUENT_SETTING_DEFAULT(uint64_t, PartitionWriteSpeedBytesPerSecond, 0); + FLUENT_SETTING_DEFAULT(uint64_t, PartitionWriteBurstBytes, 0); + + FLUENT_SETTING_VECTOR(TConsumerSettings, Consumers); + + FLUENT_SETTING(TAttributes, Attributes); + + TCreateTopicSettings& SetSupportedCodecs(std::vector&& codecs) { + SupportedCodecs_ = std::move(codecs); + return *this; + } + + TCreateTopicSettings& SetSupportedCodecs(const std::vector& codecs) { + SupportedCodecs_ = codecs; + return *this; + } + + TConsumerSettings& BeginAddConsumer() { + Consumers_.push_back({*this}); + return Consumers_.back(); + } + + TConsumerSettings& BeginAddConsumer(const std::string& name) { + Consumers_.push_back({*this, name}); + return Consumers_.back(); + } + + TCreateTopicSettings& AddAttribute(const std::string& key, const std::string& value) { + Attributes_[key] = value; + return *this; + } + + TCreateTopicSettings& SetAttributes(std::map&& attributes) { + Attributes_ = std::move(attributes); + return *this; + } + + TCreateTopicSettings& SetAttributes(const std::map& attributes) { + Attributes_ = attributes; + return *this; + } + + TCreateTopicSettings& PartitioningSettings(ui64 minActivePartitions, ui64 maxActivePartitions, TAutoPartitioningSettings autoPartitioningSettings = {}) { + PartitioningSettings_ = TPartitioningSettings(minActivePartitions, maxActivePartitions, autoPartitioningSettings); + return *this; + } + + TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); +}; + +struct TAutoPartitioningSettingsBuilder { + using TSelf = TAutoPartitioningSettingsBuilder; +public: + TAutoPartitioningSettingsBuilder(TPartitioningSettingsBuilder& parent, TAutoPartitioningSettings& settings): Parent_(parent), Settings_(settings) {} + + TSelf Strategy(EAutoPartitioningStrategy value) { + Settings_.Strategy_ = value; + return *this; + } + + TSelf StabilizationWindow(TDuration value) { + Settings_.StabilizationWindow_ = value; + return *this; + } + + TSelf DownUtilizationPercent(ui32 value) { + Settings_.DownUtilizationPercent_ = value; + return *this; + } + + TSelf UpUtilizationPercent(ui32 value) { + Settings_.UpUtilizationPercent_ = value; + return *this; + } + + TPartitioningSettingsBuilder& EndConfigureAutoPartitioningSettings() { + return Parent_; + } + +private: + TPartitioningSettingsBuilder& Parent_; + TAutoPartitioningSettings& Settings_; +}; + +struct TPartitioningSettingsBuilder { + using TSelf = TPartitioningSettingsBuilder; +public: + TPartitioningSettingsBuilder(TCreateTopicSettings& parent): Parent_(parent) {} + + TSelf MinActivePartitions(uint64_t value) { + Parent_.PartitioningSettings_.MinActivePartitions_ = value; + return *this; + } + + TSelf MaxActivePartitions(uint64_t value) { + Parent_.PartitioningSettings_.MaxActivePartitions_ = value; + return *this; + } + + TAutoPartitioningSettingsBuilder BeginConfigureAutoPartitioningSettings() { + return {*this, Parent_.PartitioningSettings_.AutoPartitioningSettings_}; + } + + TCreateTopicSettings& EndConfigurePartitioningSettings() { + return Parent_; + } + +private: + TCreateTopicSettings& Parent_; +}; + +struct TAlterTopicSettings : public TOperationRequestSettings { + + using TSelf = TAlterTopicSettings; + using TAlterAttributes = std::map; + + FLUENT_SETTING_OPTIONAL(TDuration, SetRetentionPeriod); + + FLUENT_SETTING_OPTIONAL_VECTOR(ECodec, SetSupportedCodecs); + + FLUENT_SETTING_OPTIONAL(uint64_t, SetRetentionStorageMb); + + FLUENT_SETTING_OPTIONAL(uint64_t, SetPartitionWriteSpeedBytesPerSecond); + FLUENT_SETTING_OPTIONAL(uint64_t, SetPartitionWriteBurstBytes); + + FLUENT_SETTING_OPTIONAL(EMeteringMode, SetMeteringMode); + + FLUENT_SETTING_VECTOR(TConsumerSettings, AddConsumers); + FLUENT_SETTING_VECTOR(std::string, DropConsumers); + FLUENT_SETTING_VECTOR(TAlterConsumerSettings, AlterConsumers); + + FLUENT_SETTING(TAlterAttributes, AlterAttributes); + + TAlterTopicAttributesBuilder BeginAlterAttributes() { + return TAlterTopicAttributesBuilder(*this); + } + + TAlterTopicSettings& SetSupportedCodecs(std::vector&& codecs) { + SetSupportedCodecs_ = std::move(codecs); + return *this; + } + + TAlterTopicSettings& SetSupportedCodecs(const std::vector& codecs) { + SetSupportedCodecs_ = codecs; + return *this; + } + + TConsumerSettings& BeginAddConsumer() { + AddConsumers_.push_back({*this}); + return AddConsumers_.back(); + } + + TConsumerSettings& BeginAddConsumer(const std::string& name) { + AddConsumers_.push_back({*this, name}); + return AddConsumers_.back(); + } + + TAlterConsumerSettings& BeginAlterConsumer() { + AlterConsumers_.push_back({*this}); + return AlterConsumers_.back(); + } + + TAlterConsumerSettings& BeginAlterConsumer(const std::string& name) { + AlterConsumers_.push_back({*this, name}); + return AlterConsumers_.back(); + } + + TAlterPartitioningSettings& BeginAlterPartitioningSettings() { + AlterPartitioningSettings_.emplace(*this); + return *AlterPartitioningSettings_; + } + + TAlterTopicSettings& AlterPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions) { + AlterPartitioningSettings_.emplace(*this); + AlterPartitioningSettings_->MinActivePartitions(minActivePartitions); + AlterPartitioningSettings_->MaxActivePartitions(maxActivePartitions); + return *this; + } + + std::optional AlterPartitioningSettings_; +}; + +inline TPartitioningSettingsBuilder TCreateTopicSettings::BeginConfigurePartitioningSettings() { + return {*this}; +} + +// Settings for drop resource request. +struct TDropTopicSettings : public TOperationRequestSettings { + using TOperationRequestSettings::TOperationRequestSettings; +}; + +// Settings for describe topic request. +struct TDescribeTopicSettings : public TOperationRequestSettings { + using TSelf = TDescribeTopicSettings; + + FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + + FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); +}; + +// Settings for describe consumer request. +struct TDescribeConsumerSettings : public TOperationRequestSettings { + using TSelf = TDescribeConsumerSettings; + + FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + + FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); +}; + +// Settings for describe partition request. +struct TDescribePartitionSettings: public TOperationRequestSettings { + using TSelf = TDescribePartitionSettings; + + FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); + + FLUENT_SETTING_DEFAULT(bool, IncludeLocation, false); +}; + +// Settings for commit offset request. +struct TCommitOffsetSettings : public TOperationRequestSettings {}; + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/counters.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/counters.h new file mode 100644 index 000000000000..69568c3e1772 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/counters.h @@ -0,0 +1,172 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NTopic { + +using TCounterPtr = ::NMonitoring::TDynamicCounters::TCounterPtr; + +#define TOPIC_COUNTERS_HISTOGRAM_SETUP ::NMonitoring::ExplicitHistogram({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}) + +struct TWriterCounters : public TThrRefBase { + using TSelf = TWriterCounters; + using TPtr = TIntrusivePtr; + + explicit TWriterCounters(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) { + Errors = counters->GetCounter("errors", true); + CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false); + BytesWritten = counters->GetCounter("bytesWritten", true); + MessagesWritten = counters->GetCounter("messagesWritten", true); + BytesWrittenCompressed = counters->GetCounter("bytesWrittenCompressed", true); + BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false); + BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false); + BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false); + MessagesInflight = counters->GetCounter("messagesInflight", false); + + TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + } + + TCounterPtr Errors; + TCounterPtr CurrentSessionLifetimeMs; + + TCounterPtr BytesWritten; + TCounterPtr MessagesWritten; + TCounterPtr BytesWrittenCompressed; + + TCounterPtr BytesInflightUncompressed; + TCounterPtr BytesInflightCompressed; + TCounterPtr BytesInflightTotal; + TCounterPtr MessagesInflight; + + //! Histograms reporting % usage of memory limit in time. + //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms + //! Which means that < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms + //! Used to monitor if the writer successfully deals with data flow provided. Larger values in higher buckets + //! mean that writer is close to overflow (or being overflown) for major periods of time + //! 3 histograms stand for: + //! Total memory usage: + ::NMonitoring::THistogramPtr TotalBytesInflightUsageByTime; + //! Memory usage by messages waiting for comression: + ::NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime; + //! Memory usage by compressed messages pending for write: + ::NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime; +}; + +struct TReaderCounters: public TThrRefBase { + using TSelf = TReaderCounters; + using TPtr = TIntrusivePtr; + + TReaderCounters() = default; + explicit TReaderCounters(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) { + Errors = counters->GetCounter("errors", true); + CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false); + BytesRead = counters->GetCounter("bytesRead", true); + MessagesRead = counters->GetCounter("messagesRead", true); + BytesReadCompressed = counters->GetCounter("bytesReadCompressed", true); + BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false); + BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false); + BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false); + MessagesInflight = counters->GetCounter("messagesInflight", false); + + TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", TOPIC_COUNTERS_HISTOGRAM_SETUP); + } + + TCounterPtr Errors; + TCounterPtr CurrentSessionLifetimeMs; + + TCounterPtr BytesRead; + TCounterPtr MessagesRead; + TCounterPtr BytesReadCompressed; + + TCounterPtr BytesInflightUncompressed; + TCounterPtr BytesInflightCompressed; + TCounterPtr BytesInflightTotal; + TCounterPtr MessagesInflight; + + //! Histograms reporting % usage of memory limit in time. + //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms + //! Which means < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms. + //! Used to monitor if the read session successfully deals with data flow provided. Larger values in higher buckets + //! mean that read session is close to overflow (or being overflown) for major periods of time. + //! + //! Total memory usage. + ::NMonitoring::THistogramPtr TotalBytesInflightUsageByTime; + //! Memory usage by messages waiting that are ready to be received by user. + ::NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime; + //! Memory usage by compressed messages pending for decompression. + ::NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime; +}; + +inline void MakeCountersNotNull(TReaderCounters& counters) { + if (!counters.Errors) { + counters.Errors = MakeIntrusive<::NMonitoring::TCounterForPtr>(true); + } + + if (!counters.CurrentSessionLifetimeMs) { + counters.CurrentSessionLifetimeMs = MakeIntrusive<::NMonitoring::TCounterForPtr>(false); + } + + if (!counters.BytesRead) { + counters.BytesRead = MakeIntrusive<::NMonitoring::TCounterForPtr>(true); + } + + if (!counters.MessagesRead) { + counters.MessagesRead = MakeIntrusive<::NMonitoring::TCounterForPtr>(true); + } + + if (!counters.BytesReadCompressed) { + counters.BytesReadCompressed = MakeIntrusive<::NMonitoring::TCounterForPtr>(true); + } + + if (!counters.BytesInflightUncompressed) { + counters.BytesInflightUncompressed = MakeIntrusive<::NMonitoring::TCounterForPtr>(false); + } + + if (!counters.BytesInflightCompressed) { + counters.BytesInflightCompressed = MakeIntrusive<::NMonitoring::TCounterForPtr>(false); + } + + if (!counters.BytesInflightTotal) { + counters.BytesInflightTotal = MakeIntrusive<::NMonitoring::TCounterForPtr>(false); + } + + if (!counters.MessagesInflight) { + counters.MessagesInflight = MakeIntrusive<::NMonitoring::TCounterForPtr>(false); + } + + + if (!counters.TotalBytesInflightUsageByTime) { + counters.TotalBytesInflightUsageByTime = MakeIntrusive<::NMonitoring::THistogramCounter>(TOPIC_COUNTERS_HISTOGRAM_SETUP); + } + + if (!counters.UncompressedBytesInflightUsageByTime) { + counters.UncompressedBytesInflightUsageByTime = MakeIntrusive<::NMonitoring::THistogramCounter>(TOPIC_COUNTERS_HISTOGRAM_SETUP); + } + + if (!counters.CompressedBytesInflightUsageByTime) { + counters.CompressedBytesInflightUsageByTime = MakeIntrusive<::NMonitoring::THistogramCounter>(TOPIC_COUNTERS_HISTOGRAM_SETUP); + } +} + +inline bool HasNullCounters(TReaderCounters& counters) { + return !counters.Errors + || !counters.CurrentSessionLifetimeMs + || !counters.BytesRead + || !counters.MessagesRead + || !counters.BytesReadCompressed + || !counters.BytesInflightUncompressed + || !counters.BytesInflightCompressed + || !counters.BytesInflightTotal + || !counters.MessagesInflight + || !counters.TotalBytesInflightUsageByTime + || !counters.UncompressedBytesInflightUsageByTime + || !counters.CompressedBytesInflightUsageByTime; +} + +#undef TOPIC_COUNTERS_HISTOGRAM_SETUP + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/errors.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/errors.h new file mode 100644 index 000000000000..33565d3a8b91 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/errors.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +#include + +namespace NYdb::inline V3::NTopic { + +ERetryErrorClass GetRetryErrorClass(EStatus status); +ERetryErrorClass GetRetryErrorClassV2(EStatus status); + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/events_common.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/events_common.h new file mode 100644 index 000000000000..075565924158 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/events_common.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include +#include + +namespace NYdb::inline V3::NTopic { + +template +class TPrintable { +public: + std::string DebugString(bool printData = false) const { + TStringBuilder b; + static_cast(this)->DebugString(b, printData); + return b; + } + + // implemented in template specializations + void DebugString(TStringBuilder& ret, bool printData = false) const = delete; +}; + +//! Session metainformation. +struct TWriteSessionMeta: public TThrRefBase { + using TPtr = TIntrusivePtr; + + //! User defined fields. + std::unordered_map Fields; +}; + +struct TMessageMeta: public TThrRefBase { + using TPtr = TIntrusivePtr; + + //! User defined fields. + std::vector> Fields; +}; + +//! Event that is sent to client during session destruction. +struct TSessionClosedEvent: public TStatus, public TPrintable { + using TStatus::TStatus; +}; + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const; + +using TSessionClosedHandler = std::function; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/executor.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/executor.h new file mode 100644 index 000000000000..cb9201d3c6ee --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/executor.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3::NTopic { + +class IExecutor: public TThrRefBase { +public: + using TPtr = TIntrusivePtr; + using TFunction = std::function; + + // Is executor asynchronous. + virtual bool IsAsync() const = 0; + + // Post function to execute. + virtual void Post(TFunction&& f) = 0; + + // Start method. + // This method is idempotent. + // It can be called many times. Only the first one has effect. + void Start() { + std::lock_guard guard(StartLock); + if (!Started) { + DoStart(); + Started = true; + } + } + +private: + virtual void DoStart() = 0; + +private: + bool Started = false; + TAdaptiveLock StartLock; +}; +IExecutor::TPtr CreateThreadPoolExecutorAdapter( + std::shared_ptr threadPool); // Thread pool is expected to have been started. +IExecutor::TPtr CreateThreadPoolExecutor(size_t threads); + +IExecutor::TPtr CreateSyncExecutor(); + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h new file mode 100644 index 000000000000..81f0d88dd15b --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h @@ -0,0 +1,440 @@ +#pragma once + +#include "codecs.h" +#include "events_common.h" + +#include + +namespace NYdb::inline V3::NTopic { + +//! Partition session. +struct TPartitionSession: public TThrRefBase, public TPrintable { + using TPtr = TIntrusivePtr; + +public: + //! Request partition session status. + //! Result will come to TPartitionSessionStatusEvent. + virtual void RequestStatus() = 0; + + //! + //! Properties. + //! + + //! Unique identifier of partition session. + //! It is unique within one read session. + uint64_t GetPartitionSessionId() const { + return PartitionSessionId; + } + + //! Topic path. + const std::string& GetTopicPath() const { + return TopicPath; + } + + //! Partition id. + uint64_t GetPartitionId() const { + return PartitionId; + } + +protected: + uint64_t PartitionSessionId; + std::string TopicPath; + uint64_t PartitionId; +}; + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const; + +//! Events for read session. +struct TReadSessionEvent { + class TPartitionSessionAccessor { + public: + TPartitionSessionAccessor(TPartitionSession::TPtr partitionSession); + + virtual ~TPartitionSessionAccessor() = default; + + virtual const TPartitionSession::TPtr& GetPartitionSession() const; + + protected: + TPartitionSession::TPtr PartitionSession; + }; + + //! Event with new data. + //! Contains batch of messages from single partition session. + struct TDataReceivedEvent : public TPartitionSessionAccessor, public TPrintable { + struct TMessageInformation { + TMessageInformation(uint64_t offset, + std::string producerId, + uint64_t seqNo, + TInstant createTime, + TInstant writeTime, + TWriteSessionMeta::TPtr meta, + TMessageMeta::TPtr messageMeta, + uint64_t uncompressedSize, + std::string messageGroupId); + uint64_t Offset; + std::string ProducerId; + uint64_t SeqNo; + TInstant CreateTime; + TInstant WriteTime; + TWriteSessionMeta::TPtr Meta; + TMessageMeta::TPtr MessageMeta; + uint64_t UncompressedSize; + std::string MessageGroupId; + }; + + class TMessageBase : public TPrintable { + public: + TMessageBase(const std::string& data, TMessageInformation info); + + virtual ~TMessageBase() = default; + + virtual const std::string& GetData() const; + + virtual void Commit() = 0; + + //! Message offset. + uint64_t GetOffset() const; + + //! Producer id. + const std::string& GetProducerId() const; + + //! Message group id. + const std::string& GetMessageGroupId() const; + + //! Sequence number. + uint64_t GetSeqNo() const; + + //! Message creation timestamp. + TInstant GetCreateTime() const; + + //! Message write timestamp. + TInstant GetWriteTime() const; + + //! Metainfo. + const TWriteSessionMeta::TPtr& GetMeta() const; + + //! Message level meta info. + const TMessageMeta::TPtr& GetMessageMeta() const; + + protected: + std::string Data; + TMessageInformation Information; + }; + + //! Single message. + struct TMessage: public TMessageBase, public TPartitionSessionAccessor, public TPrintable { + using TPrintable::DebugString; + + TMessage(const std::string& data, std::exception_ptr decompressionException, TMessageInformation information, + TPartitionSession::TPtr partitionSession); + + //! User data. + //! Throws decompressor exception if decompression failed. + const std::string& GetData() const override; + + //! Commits single message. + void Commit() override; + + bool HasException() const; + + private: + std::exception_ptr DecompressionException; + }; + + struct TCompressedMessage: public TMessageBase, + public TPartitionSessionAccessor, + public TPrintable { + using TPrintable::DebugString; + + TCompressedMessage(ECodec codec, const std::string& data, TMessageInformation information, + TPartitionSession::TPtr partitionSession); + + virtual ~TCompressedMessage() { + } + + //! Message codec + ECodec GetCodec() const; + + //! Uncompressed size. + uint64_t GetUncompressedSize() const; + + //! Commits all offsets in compressed message. + void Commit() override; + + private: + ECodec Codec; + }; + + public: + TDataReceivedEvent(std::vector messages, std::vector compressedMessages, + TPartitionSession::TPtr partitionSession); + + bool HasCompressedMessages() const { + return !CompressedMessages.empty(); + } + + size_t GetMessagesCount() const { + return Messages.size() + CompressedMessages.size(); + } + + //! Get messages. + std::vector& GetMessages() { + CheckMessagesFilled(false); + return Messages; + } + + const std::vector& GetMessages() const { + CheckMessagesFilled(false); + return Messages; + } + + //! Get compressed messages. + std::vector& GetCompressedMessages() { + CheckMessagesFilled(true); + return CompressedMessages; + } + + const std::vector& GetCompressedMessages() const { + CheckMessagesFilled(true); + return CompressedMessages; + } + + void SetReadInTransaction() { + ReadInTransaction = true; + } + + //! Commits all messages in batch. + void Commit(); + + private: + void CheckMessagesFilled(bool compressed) const { + Y_ABORT_UNLESS(!Messages.empty() || !CompressedMessages.empty()); + if (compressed && CompressedMessages.empty()) { + ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session"; + } + if (!compressed && Messages.empty()) { + ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session"; + } + } + + private: + std::vector Messages; + std::vector CompressedMessages; + std::vector> OffsetRanges; + bool ReadInTransaction = false; + }; + + //! Acknowledgement for commit request. + struct TCommitOffsetAcknowledgementEvent: public TPartitionSessionAccessor, + public TPrintable { + TCommitOffsetAcknowledgementEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset); + + //! Committed offset. + //! This means that from now the first available + //! message offset in current partition + //! for current consumer is this offset. + //! All messages before are committed and futher never be available. + uint64_t GetCommittedOffset() const { + return CommittedOffset; + } + + private: + uint64_t CommittedOffset; + }; + + //! Server command for creating and starting partition session. + struct TStartPartitionSessionEvent: public TPartitionSessionAccessor, + public TPrintable { + explicit TStartPartitionSessionEvent(TPartitionSession::TPtr, uint64_t committedOffset, uint64_t endOffset); + + //! Current committed offset in partition session. + uint64_t GetCommittedOffset() const { + return CommittedOffset; + } + + //! Offset of first not existing message in partition session. + uint64_t GetEndOffset() const { + return EndOffset; + } + + //! Confirm partition session creation. + //! This signals that user is ready to receive data from this partition session. + //! If maybe is empty then no rewinding + void Confirm(std::optional readOffset = std::nullopt, std::optional commitOffset = std::nullopt); + + private: + uint64_t CommittedOffset; + uint64_t EndOffset; + }; + + //! Server command for stopping and destroying partition session. + //! Server can destroy partition session gracefully + //! for rebalancing among all topic clients. + struct TStopPartitionSessionEvent: public TPartitionSessionAccessor, public TPrintable { + TStopPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset); + + //! Last offset of the partition session that was committed. + uint64_t GetCommittedOffset() const { + return CommittedOffset; + } + + //! Confirm partition session destruction. + //! Confirm has no effect if TPartitionSessionClosedEvent for same partition session with is received. + void Confirm(); + + private: + uint64_t CommittedOffset; + }; + + //! Server command for ending partition session. + //! This is a hint that all messages from the partition have been read and will no longer appear, and that the client must commit offsets. + struct TEndPartitionSessionEvent: public TPartitionSessionAccessor, public TPrintable { + TEndPartitionSessionEvent(TPartitionSession::TPtr partitionSession, std::vector&& adjacentPartitionIds, std::vector&& childPartitionIds); + + //! A list of the partition IDs that also participated in the partition's merge. + const std::vector& GetAdjacentPartitionIds() const { + return AdjacentPartitionIds; + } + + //! A list of partition IDs that were obtained as a result of merging or splitting this partition. + const std::vector& GetChildPartitionIds() const { + return ChildPartitionIds; + } + + //! Confirm partition session destruction. + //! Confirm has no effect if TPartitionSessionClosedEvent for same partition session with is received. + void Confirm(); + + private: + std::vector AdjacentPartitionIds; + std::vector ChildPartitionIds; + }; + + //! Status for partition session requested via TPartitionSession::RequestStatus() + struct TPartitionSessionStatusEvent: public TPartitionSessionAccessor, + public TPrintable { + TPartitionSessionStatusEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset, uint64_t readOffset, + uint64_t endOffset, TInstant writeTimeHighWatermark); + + //! Committed offset. + uint64_t GetCommittedOffset() const { + return CommittedOffset; + } + + //! Offset of next message (that is not yet read by session). + uint64_t GetReadOffset() const { + return ReadOffset; + } + + //! Offset of first not existing message in partition. + uint64_t GetEndOffset() const { + return EndOffset; + } + + //! Write time high watermark. + //! Write timestamp of next message written to this partition will be no less than this. + TInstant GetWriteTimeHighWatermark() const { + return WriteTimeHighWatermark; + } + + private: + uint64_t CommittedOffset = 0; + uint64_t ReadOffset = 0; + uint64_t EndOffset = 0; + TInstant WriteTimeHighWatermark; + }; + + //! Event that signals user about + //! partition session death. + //! This could be after graceful stop of partition session + //! or when connection with partition was lost. + struct TPartitionSessionClosedEvent: public TPartitionSessionAccessor, + public TPrintable { + enum class EReason { + StopConfirmedByUser, + Lost, + ConnectionLost, + }; + + public: + TPartitionSessionClosedEvent(TPartitionSession::TPtr partitionSession, EReason reason); + + EReason GetReason() const { + return Reason; + } + + private: + EReason Reason; + }; + + using TEvent = std::variant; +}; + +//! Set of offsets to commit. +//! Class that could store offsets in order to commit them later. +//! This class is not thread safe. +class TDeferredCommit { +public: + //! Add message to set. + void Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message); + + //! Add all messages from dataReceivedEvent to set. + void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); + + //! Add offsets range to set. + void Add(const TPartitionSession::TPtr& partitionSession, uint64_t startOffset, uint64_t endOffset); + + //! Add offset to set. + void Add(const TPartitionSession::TPtr& partitionSession, uint64_t offset); + + //! Commit all added offsets. + void Commit(); + + TDeferredCommit(); + TDeferredCommit(const TDeferredCommit&) = delete; + TDeferredCommit(TDeferredCommit&&); + TDeferredCommit& operator=(const TDeferredCommit&) = delete; + TDeferredCommit& operator=(TDeferredCommit&&); + + ~TDeferredCommit(); + +private: + class TImpl; + std::unique_ptr Impl; +}; + +//! Events debug strings. +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; + +std::string DebugString(const TReadSessionEvent::TEvent& event); + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_session.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_session.h new file mode 100644 index 000000000000..696d3f01f02d --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_session.h @@ -0,0 +1,256 @@ +#pragma once + +#include "counters.h" +#include "executor.h" +#include "read_events.h" +#include "retry_policy.h" + +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NTable { + class TTransaction; +} + +namespace NYdb::inline V3::NTopic { + +//! Read settings for single topic. +struct TTopicReadSettings { + using TSelf = TTopicReadSettings; + + TTopicReadSettings() = default; + TTopicReadSettings(const TTopicReadSettings&) = default; + TTopicReadSettings(TTopicReadSettings&&) = default; + TTopicReadSettings(const std::string& path) { + Path(path); + } + + TTopicReadSettings& operator=(const TTopicReadSettings&) = default; + TTopicReadSettings& operator=(TTopicReadSettings&&) = default; + + //! Path of topic to read. + FLUENT_SETTING(std::string, Path); + + //! Start reading from this timestamp. + FLUENT_SETTING_OPTIONAL(TInstant, ReadFromTimestamp); + + //! Partition ids to read. + //! 0-based. + FLUENT_SETTING_VECTOR(uint64_t, PartitionIds); + + //! Max message time lag. All messages older that now - MaxLag will be ignored. + FLUENT_SETTING_OPTIONAL(TDuration, MaxLag); +}; + +//! Settings for read session. +struct TReadSessionSettings: public TRequestSettings { + using TSelf = TReadSessionSettings; + + struct TEventHandlers { + using TSelf = TEventHandlers; + + //! Set simple handler with data processing and also + //! set other handlers with default behaviour. + //! They automatically commit data after processing + //! and confirm partition session events. + //! + //! Sets the following handlers: + //! DataReceivedHandler: sets DataReceivedHandler to handler that calls dataHandler and (if + //! commitDataAfterProcessing is set) then calls Commit(). CommitAcknowledgementHandler to handler that does + //! nothing. StartPartitionSessionHandler to handler that confirms event. StopPartitionSessionHandler to + //! handler that confirms event. PartitionSessionStatusHandler to handler that does nothing. + //! PartitionSessionClosedHandler to handler that does nothing. + //! + //! dataHandler: handler of data event. + //! commitDataAfterProcessing: automatically commit data after calling of dataHandler. + //! gracefulReleaseAfterCommit: wait for commit acknowledgements for all inflight data before confirming + //! partition session stop. + TSelf& SimpleDataHandlers(std::function dataHandler, + bool commitDataAfterProcessing = false, bool gracefulStopAfterCommit = true); + + //! Data size limit for the DataReceivedHandler handler. + //! The data size may exceed this limit. + FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, std::numeric_limits::max()); + + //! Function to handle data events. + //! If this handler is set, data events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, DataReceivedHandler); + + //! Function to handle commit ack events. + //! If this handler is set, commit ack events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + CommitOffsetAcknowledgementHandler); + + //! Function to handle start partition session events. + //! If this handler is set, create partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + StartPartitionSessionHandler); + + //! Function to handle stop partition session events. + //! If this handler is set, destroy partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + StopPartitionSessionHandler); + + //! Function to handle end partition session events. + //! If this handler is set, end partition session events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + EndPartitionSessionHandler); + + //! Function to handle partition session status events. + //! If this handler is set, partition session status events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + PartitionSessionStatusHandler); + + //! Function to handle partition session closed events. + //! If this handler is set, partition session closed events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, + PartitionSessionClosedHandler); + + //! Function to handle session closed events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + + //! Function to handle all event types. + //! If event with current type has no handler for this type of event, + //! this handler (if specified) will be used. + //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. + FLUENT_SETTING(std::function, CommonHandler); + + //! Executor for handlers. + //! If not set, default single threaded executor will be used. + FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + }; + + + std::string ConsumerName_ = ""; + //! Consumer. + TSelf& ConsumerName(const std::string& name) { + ConsumerName_ = name; + WithoutConsumer_ = false; + return static_cast(*this); + } + + bool WithoutConsumer_ = false; + //! Read without consumer. + TSelf& WithoutConsumer() { + WithoutConsumer_ = true; + ConsumerName_ = ""; + return static_cast(*this); + } + + //! Topics. + FLUENT_SETTING_VECTOR(TTopicReadSettings, Topics); + + //! Maximum memory usage for read session. + FLUENT_SETTING_DEFAULT(size_t, MaxMemoryUsageBytes, 100_MB); + + //! Max message time lag. All messages older that now - MaxLag will be ignored. + FLUENT_SETTING_OPTIONAL(TDuration, MaxLag); + + //! Start reading from this timestamp. + FLUENT_SETTING_OPTIONAL(TInstant, ReadFromTimestamp); + + //! Policy for reconnections. + //! IRetryPolicy::GetDefaultPolicy() if null (not set). + FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + + //! Event handlers. + //! See description in TEventHandlers class. + FLUENT_SETTING(TEventHandlers, EventHandlers); + + //! Decompress messages + FLUENT_SETTING_DEFAULT(bool, Decompress, true); + + //! Executor for decompression tasks. + //! If not set, default executor will be used. + FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor); + + //! Counters. + //! If counters are not provided explicitly, + //! they will be created inside session (without link with parent counters). + FLUENT_SETTING(TReaderCounters::TPtr, Counters); + + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + + //! AutoPartitioningSupport. + FLUENT_SETTING_DEFAULT(bool, AutoPartitioningSupport, false); + + //! Log. + FLUENT_SETTING_OPTIONAL(TLog, Log); +}; + +struct TReadSessionGetEventSettings : public TCommonClientSettingsBase { + using TSelf = TReadSessionGetEventSettings; + + FLUENT_SETTING_DEFAULT(bool, Block, false); + FLUENT_SETTING_OPTIONAL(size_t, MaxEventsCount); + FLUENT_SETTING_DEFAULT(size_t, MaxByteSize, std::numeric_limits::max()); + FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); +}; + +class IReadSession { +public: + //! Main reader loop. + //! Wait for next reader event. + virtual NThreading::TFuture WaitEvent() = 0; + + //! Main reader loop. + //! Get read session events. + //! Blocks until event occurs if "block" is set. + //! + //! maxEventsCount: maximum events count in batch. + //! maxByteSize: total size limit of data messages in batch. + //! block: block until event occurs. + //! + //! If maxEventsCount is not specified, + //! read session chooses event batch size automatically. + virtual std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt, + size_t maxByteSize = std::numeric_limits::max()) = 0; + + virtual std::vector GetEvents(const TReadSessionGetEventSettings& settings) = 0; + + //! Get single event. + virtual std::optional GetEvent(bool block = false, + size_t maxByteSize = std::numeric_limits::max()) = 0; + + virtual std::optional GetEvent(const TReadSessionGetEventSettings& settings) = 0; + + //! Close read session. + //! Waits for all commit acknowledgments to arrive. + //! Force close after timeout. + //! This method is blocking. + //! When session is closed, + //! TSessionClosedEvent arrives. + virtual bool Close(TDuration timeout = TDuration::Max()) = 0; + + //! Reader counters with different stats (see TReaderConuters). + virtual TReaderCounters::TPtr GetCounters() const = 0; + + //! Get unique identifier of read session. + virtual std::string GetSessionId() const = 0; + + virtual ~IReadSession() = default; +}; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/retry_policy.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/retry_policy.h new file mode 100644 index 000000000000..1e7519f64064 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/retry_policy.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +namespace NYdb::inline V3::NTopic { + +//! Retry policy. +//! Calculates delay before next retry. +//! Has several default implementations: +//! - exponential backoff policy; +//! - retries with fixed interval; +//! - no retries. + +struct IRetryPolicy: ::IRetryPolicy { + //! + //! Default implementations. + //! + + static TPtr GetDefaultPolicy(); // Exponential backoff with infinite retry attempts. + static TPtr GetNoRetryPolicy(); // Denies all kind of retries. + + //! Randomized exponential backoff policy. + static TPtr GetExponentialBackoffPolicy( + TDuration minDelay = TDuration::MilliSeconds(10), + // Delay for statuses that require waiting before retry (such as OVERLOADED). + TDuration minLongRetryDelay = TDuration::MilliSeconds(200), TDuration maxDelay = TDuration::Seconds(30), + size_t maxRetries = std::numeric_limits::max(), TDuration maxTime = TDuration::Max(), + double scaleFactor = 2.0, std::function customRetryClassFunction = {}); + + //! Randomized fixed interval policy. + static TPtr GetFixedIntervalPolicy(TDuration delay = TDuration::MilliSeconds(100), + // Delay for statuses that require waiting before retry (such as OVERLOADED). + TDuration longRetryDelay = TDuration::MilliSeconds(300), + size_t maxRetries = std::numeric_limits::max(), + TDuration maxTime = TDuration::Max(), + std::function customRetryClassFunction = {}); +}; + +} // namespace NYdb::V3::NTopic diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_events.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_events.h new file mode 100644 index 000000000000..49b7bb1ded24 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_events.h @@ -0,0 +1,111 @@ +#pragma once + +#include "events_common.h" + +#include +#include + +#include + +namespace NYdb::inline V3::NTopic { + +struct TWriteStat : public TThrRefBase { + TDuration WriteTime; + TDuration MinTimeInPartitionQueue; + TDuration MaxTimeInPartitionQueue; + TDuration PartitionQuotedTime; + TDuration TopicQuotedTime; + using TPtr = TIntrusivePtr; +}; + +class TContinuationToken : public TNonCopyable { + friend class TContinuationTokenIssuer; + + bool Valid = true; + +public: + TContinuationToken& operator=(TContinuationToken&& other) { + if (!other.Valid) { + ythrow TContractViolation("Cannot move invalid token"); + } + Valid = std::exchange(other.Valid, false); + return *this; + } + + TContinuationToken(TContinuationToken&& other) { + *this = std::move(other); + } + +private: + TContinuationToken() = default; +}; + +class TContinuationTokenIssuer { +protected: + static auto IssueContinuationToken() { + return TContinuationToken{}; + } +}; + +//! Events for write session. +struct TWriteSessionEvent { + + //! Event with acknowledge for written messages. + struct TWriteAck { + //! Write result. + enum EEventState { + EES_WRITTEN, //! Successfully written. + EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication. + EES_DISCARDED, //! In case of destruction of writer or retry policy discarded future retries in this writer. + EES_WRITTEN_IN_TX, //! Successfully written in tx. + }; + //! Details of successfully written message. + struct TWrittenMessageDetails { + uint64_t Offset; + uint64_t PartitionId; + }; + //! Same SeqNo as provided on write. + uint64_t SeqNo; + EEventState State; + //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED. + std::optional Details; + //! Write stats from server. See TWriteStat. nullptr for DISCARDED event. + TWriteStat::TPtr Stat; + }; + + struct TAcksEvent : public TPrintable { + //! Acks could be batched from several Write requests. + //! They are provided to client as soon as possible. + std::vector Acks; + }; + + //! Indicates that a writer is ready to accept new message(s). + //! Continuation token should be kept and then used in write methods. + struct TReadyToAcceptEvent : public TPrintable { + mutable TContinuationToken ContinuationToken; + + TReadyToAcceptEvent() = delete; + TReadyToAcceptEvent(TContinuationToken&& t) : ContinuationToken(std::move(t)) { + } + TReadyToAcceptEvent(TReadyToAcceptEvent&&) = default; + TReadyToAcceptEvent(const TReadyToAcceptEvent& other) : ContinuationToken(std::move(other.ContinuationToken)) { + } + TReadyToAcceptEvent& operator=(TReadyToAcceptEvent&&) = default; + TReadyToAcceptEvent& operator=(const TReadyToAcceptEvent& other) { + ContinuationToken = std::move(other.ContinuationToken); + return *this; + } + }; + + using TEvent = std::variant; +}; + +//! Events debug strings. +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const; + +std::string DebugString(const TWriteSessionEvent::TEvent& event); + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_session.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_session.h new file mode 100644 index 000000000000..585e32f13c0b --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_session.h @@ -0,0 +1,284 @@ +#pragma once + +#include "codecs.h" +#include "counters.h" +#include "executor.h" +#include "retry_policy.h" +#include "write_events.h" + +#include +#include + +#include + +namespace NYdb::inline V3::NTable { + class TTransaction; +} + +namespace NYdb::inline V3::NTopic { + +using TTransaction = NTable::TTransaction; + +//! Settings for write session. +struct TWriteSessionSettings : public TRequestSettings { + using TSelf = TWriteSessionSettings; + + TWriteSessionSettings() = default; + TWriteSessionSettings(const TWriteSessionSettings&) = default; + TWriteSessionSettings(TWriteSessionSettings&&) = default; + TWriteSessionSettings(const std::string& path, const std::string& producerId, const std::string& messageGroupId) { + Path(path); + ProducerId(producerId); + MessageGroupId(messageGroupId); + } + + TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default; + TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; + + //! Path of topic to write. + FLUENT_SETTING(std::string, Path); + + //! ProducerId (aka SourceId) to use. + FLUENT_SETTING(std::string, ProducerId); + + //! MessageGroupId to use. + FLUENT_SETTING(std::string, MessageGroupId); + + //! Explicitly enables or disables deduplication for this write session. + //! If ProducerId option is defined deduplication will always be enabled. + //! If ProducerId option is empty, but deduplication is enable, a random ProducerId is generated. + FLUENT_SETTING_OPTIONAL(bool, DeduplicationEnabled); + + //! Write to an exact partition. Generally server assigns partition automatically by message_group_id. + //! Using this option is not recommended unless you know for sure why you need it. + FLUENT_SETTING_OPTIONAL(uint32_t, PartitionId); + + //! Direct write to the partition host. + //! If both PartitionId and DirectWriteToPartition are set, write session goes directly to the partition host. + //! If DirectWriteToPartition set without PartitionId, the write session is established in three stages: + //! 1. Get a partition ID. + //! 2. Find out the location of the partition by its ID. + //! 3. Connect directly to the partition host. + FLUENT_SETTING_DEFAULT(bool, DirectWriteToPartition, true); + + //! codec and level to use for data compression prior to write. + FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); + FLUENT_SETTING_DEFAULT(int32_t, CompressionLevel, 4); + + //! Writer will not accept new messages if memory usage exceeds this limit. + //! Memory usage consists of raw data pending compression and compressed messages being sent. + FLUENT_SETTING_DEFAULT(uint64_t, MaxMemoryUsage, 20_MB); + + //! Maximum messages accepted by writer but not written (with confirmation from server). + //! Writer will not accept new messages after reaching the limit. + FLUENT_SETTING_DEFAULT(uint32_t, MaxInflightCount, 100000); + + //! Retry policy enables automatic retries for non-fatal errors. + //! IRetryPolicy::GetDefaultPolicy() if null (not set). + FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + + //! User metadata that may be attached to write session. + TWriteSessionSettings& AppendSessionMeta(const std::string& key, const std::string& value) { + Meta_.Fields[key] = value; + return *this; + }; + + TWriteSessionMeta Meta_; + + //! Writer will accumulate messages until reaching up to BatchFlushSize bytes + //! but for no longer than BatchFlushInterval. + //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch. + //! Greatly increases performance for small messages. + //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients + //! sending small messages at high rate). + FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); + FLUENT_SETTING_OPTIONAL(uint64_t, BatchFlushSizeBytes); + + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + + FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); + + //! Executor for compression tasks. + //! If not set, default executor will be used. + FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); + + struct TEventHandlers { + using TSelf = TEventHandlers; + using TWriteAckHandler = std::function; + using TReadyToAcceptHandler = std::function; + + //! Function to handle Acks events. + //! If this handler is set, write ack events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TWriteAckHandler, AcksHandler); + + //! Function to handle ReadyToAccept event. + //! If this handler is set, write these events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHandler); + + //! Function to handle close session events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + + //! Function to handle all event types. + //! If event with current type has no handler for this type of event, + //! this handler (if specified) will be used. + //! If this handler is not specified, event can be received with TWriteSession::GetEvent() method. + std::function CommonHandler_; + TSelf& CommonHandler(std::function&& handler) { + CommonHandler_ = std::move(handler); + return static_cast(*this); + } + + //! Executor for handlers. + //! If not set, default single threaded executor will be used. + FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + + [[deprecated("Typo in name. Use ReadyToAcceptHandler instead.")]] + TSelf& ReadyToAcceptHander(const TReadyToAcceptHandler& value) { + return ReadyToAcceptHandler(value); + } + }; + + //! Event handlers. + FLUENT_SETTING(TEventHandlers, EventHandlers); + + //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception. + FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true); +}; + +//! Contains the message to write and all the options. +struct TWriteMessage { + using TSelf = TWriteMessage; + using TMessageMeta = std::vector>; +public: + TWriteMessage() = delete; + TWriteMessage(std::string_view data) + : Data(data) + {} + + //! A message that is already compressed by codec. Codec from WriteSessionSettings does not apply to this message. + //! Compression will not be performed in SDK for such messages. + static TWriteMessage CompressedMessage(const std::string_view& data, ECodec codec, uint32_t originalSize) { + TWriteMessage result{data}; + result.Codec = codec; + result.OriginalSize = originalSize; + return result; + } + + bool Compressed() const { + return Codec.has_value(); + } + + //! Message body. + std::string_view Data; + + //! Codec and original size for compressed message. + //! Do not specify or change these options directly, use CompressedMessage() + //! method to create an object for compressed message. + std::optional Codec; + uint32_t OriginalSize = 0; + + //! Message SeqNo, optional. If not provided SDK core will calculate SeqNo automatically. + //! NOTICE: Either all messages within one write session must have SeqNo provided or none of them. + FLUENT_SETTING_OPTIONAL(uint64_t, SeqNo); + + //! Message creation timestamp. If not provided, Now() will be used. + FLUENT_SETTING_OPTIONAL(TInstant, CreateTimestamp); + + //! Message metadata. Limited to 4096 characters overall (all keys and values combined). + FLUENT_SETTING(TMessageMeta, MessageMeta); + + //! Transaction id + FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); + + TTransaction* GetTxPtr() const + { + return Tx_ ? &Tx_->get() : nullptr; + } +}; + +//! Simple write session. Does not need event handlers. Does not provide Events, ContinuationTokens, write Acks. +class ISimpleBlockingWriteSession : public TThrRefBase { +public: + //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded; + //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout. + //! no Ack is provided. + virtual bool Write(TWriteMessage&& message, + NTable::TTransaction* tx = nullptr, + const TDuration& blockTimeout = TDuration::Max()) = 0; + + + //! Write single message. Deprecated method with only basic message options. + virtual bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, + const TDuration& blockTimeout = TDuration::Max()) = 0; + + //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init. + virtual uint64_t GetInitSeqNo() = 0; + + //! Complete all active writes, wait for ack from server and close. + //! closeTimeout - max time to wait. Empty Maybe means infinity. + //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. + + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Returns true if write session is alive and acitve. False if session was closed. + virtual bool IsAlive() const = 0; + + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close immediately and destroy, don't wait for anything. + virtual ~ISimpleBlockingWriteSession() = default; +}; + +//! Generic write session with all capabilities. +class IWriteSession { +public: + //! Future that is set when next event is available. + virtual NThreading::TFuture WaitEvent() = 0; + + //! Wait and return next event. Use WaitEvent() for non-blocking wait. + virtual std::optional GetEvent(bool block = false) = 0; + + //! Get several events in one call. + //! If blocking = false, instantly returns up to maxEventsCount available events. + //! If blocking = true, blocks till maxEventsCount events are available. + //! If maxEventsCount is unset, write session decides the count to return itself. + virtual std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt) = 0; + + //! Future that is set when initial SeqNo is available. + virtual NThreading::TFuture GetInitSeqNo() = 0; + + //! Write single message. + //! continuationToken - a token earlier provided to client with ReadyToAccept event. + virtual void Write(TContinuationToken&& continuationToken, TWriteMessage&& message, + NTable::TTransaction* tx = nullptr) = 0; + + //! Write single message. Old method with only basic message options. + virtual void Write(TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, + std::optional createTimestamp = std::nullopt) = 0; + + //! Write single message that is already coded by codec. + //! continuationToken - a token earlier provided to client with ReadyToAccept event. + virtual void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& params, + NTable::TTransaction* tx = nullptr) = 0; + + //! Write single message that is already compressed by codec. Old method with only basic message options. + virtual void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, uint32_t originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) = 0; + + + //! Wait for all writes to complete (no more that closeTimeout()), then close. + //! Return true if all writes were completed and acked, false if timeout was reached and some writes were aborted. + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Writer counters with different stats (see TWriterConuters). + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close() with timeout = 0 and destroy everything instantly. + virtual ~IWriteSession() = default; +}; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/ya.make b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/ya.make new file mode 100644 index 000000000000..4d625cfee9d1 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/ya.make @@ -0,0 +1,32 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/codecs.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/write_events.h) + +SRCS( + client.h + codecs.h + control_plane.h + counters.h + errors.h + events_common.h + executor.h + read_events.h + read_session.h + retry_policy.h + write_events.h + write_session.h +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + + library/cpp/streams/zstd +) + +END() diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h new file mode 100644 index 000000000000..75d51f465490 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include + +namespace NYdb::inline V3 { + +class ICredentialsProvider { +public: + virtual ~ICredentialsProvider() = default; + virtual std::string GetAuthInfo() const = 0; + virtual bool IsValid() const = 0; +}; + +using TCredentialsProviderPtr = std::shared_ptr; + +class ICoreFacility; +class ICredentialsProviderFactory { +public: + virtual ~ICredentialsProviderFactory() = default; + virtual TCredentialsProviderPtr CreateProvider() const = 0; + // !!!Experimental!!! + virtual TCredentialsProviderPtr CreateProvider([[maybe_unused]] std::weak_ptr facility) const { + return CreateProvider(); + } + virtual std::string GetClientIdentity() const; +}; + +using TCredentialsProviderFactoryPtr = std::shared_ptr; + +std::shared_ptr CreateInsecureCredentialsProviderFactory(); +std::shared_ptr CreateOAuthCredentialsProviderFactory(const std::string& token); + +struct TLoginCredentialsParams { + std::string User; + std::string Password; +}; + +std::shared_ptr CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h new file mode 100644 index 000000000000..78be18a8021f --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h @@ -0,0 +1,64 @@ +#pragma once + +#include + +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +class ITokenSource { +public: + struct TToken { + std::string Token; + + // token type according to OAuth 2.0 token exchange protocol + // https://www.rfc-editor.org/rfc/rfc8693#TokenTypeIdentifiers + // for example urn:ietf:params:oauth:token-type:jwt + std::string TokenType; + }; + + virtual ~ITokenSource() = default; + virtual TToken GetToken() const = 0; +}; + +std::shared_ptr CreateFixedTokenSource(const std::string& token, const std::string& tokenType); + +#define FLUENT_SETTING_VECTOR_OR_SINGLE(type, name) \ + FLUENT_SETTING_VECTOR(type, name); \ + TSelf& name(const type& value) { \ + name##_.resize(1); \ + name##_[0] = value; \ + return static_cast(*this); \ + } + +struct TOauth2TokenExchangeParams { + using TSelf = TOauth2TokenExchangeParams; + + FLUENT_SETTING(std::string, TokenEndpoint); + + FLUENT_SETTING_DEFAULT(std::string, GrantType, "urn:ietf:params:oauth:grant-type:token-exchange"); + + FLUENT_SETTING_VECTOR_OR_SINGLE(std::string, Resource); + FLUENT_SETTING_VECTOR_OR_SINGLE(std::string, Audience); + FLUENT_SETTING_VECTOR_OR_SINGLE(std::string, Scope); + + FLUENT_SETTING_DEFAULT(std::string, RequestedTokenType, "urn:ietf:params:oauth:token-type:access_token"); + + FLUENT_SETTING(std::shared_ptr, SubjectTokenSource); + FLUENT_SETTING(std::shared_ptr, ActorTokenSource); + + FLUENT_SETTING_DEFAULT(TDuration, SocketTimeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + FLUENT_SETTING_DEFAULT(TDuration, SyncUpdateTimeout, TDuration::Seconds(20)); +}; + +// Creates OAuth 2.0 token exchange credentials provider factory that exchanges token using standard protocol +// https://www.rfc-editor.org/rfc/rfc8693 +std::shared_ptr CreateOauth2TokenExchangeCredentialsProviderFactory(const TOauth2TokenExchangeParams& params); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/from_file.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/from_file.h new file mode 100644 index 000000000000..242da20a88dd --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/from_file.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace NYdb::inline V3 { + +// Lists supported algorithms for creation of OAuth 2.0 token exchange provider via config file +std::vector GetSupportedOauth2TokenExchangeJwtAlgorithms(); + +// Creates OAuth 2.0 token exchange credentials provider factory that exchanges token using standard protocol +// https://www.rfc-editor.org/rfc/rfc8693 +// +// Config file must be a valid json file +// +// Fields of json file +// grant-type: [string] Grant type option (default: see TOauth2TokenExchangeParams) +// res: [string | list of strings] Resource option (optional) +// aud: [string | list of strings] Audience option for token exchange request (optional) +// scope: [string | list of strings] Scope option (optional) +// requested-token-type: [string] Requested token type option (default: see TOauth2TokenExchangeParams) +// subject-credentials: [creds_json] Subject credentials options (optional) +// actor-credentials: [creds_json] Actor credentials options (optional) +// token-endpoint: [string] Token endpoint. Can be overritten with tokenEndpoint param (if it is not empty) +// +// Fields of creds_json (JWT): +// type: [string] Token source type. Set JWT +// alg: [string] Algorithm for JWT signature. Supported algorithms can be listed with GetSupportedOauth2TokenExchangeJwtAlgorithms() +// private-key: [string] (Private) key in PEM format for JWT signature +// kid: [string] Key id JWT standard claim (optional) +// iss: [string] Issuer JWT standard claim (optional) +// sub: [string] Subject JWT standard claim (optional) +// aud: [string | list of strings] Audience JWT standard claim (optional) +// jti: [string] JWT ID JWT standard claim (optional) +// ttl: [string] Token TTL (default: see TJwtTokenSourceParams) +// +// Fields of creds_json (FIXED): +// type: [string] Token source type. Set FIXED +// token: [string] Token value +// token-type: [string] Token type value. It will become subject_token_type/actor_token_type parameter in token exchange request (https://www.rfc-editor.org/rfc/rfc8693) +// +std::shared_ptr CreateOauth2TokenExchangeFileCredentialsProviderFactory(const std::string& configFilePath, const std::string& tokenEndpoint = {}); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h new file mode 100644 index 000000000000..99a485acb5fa --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h @@ -0,0 +1,80 @@ +#pragma once + +#include + +#include "credentials.h" + +#include + +#include + +namespace NYdb::inline V3 { + +constexpr TDuration DEFAULT_JWT_TOKEN_TTL = TDuration::Hours(1); + +struct TJwtTokenSourceParams { + using TSelf = TJwtTokenSourceParams; + + FLUENT_SETTING(std::string, KeyId); + + template + TSelf& SigningAlgorithm(T&&... args) { + SigningAlgorithm_ = std::make_shared>(std::forward(args)...); + return *this; + } + + // JWT Claims + FLUENT_SETTING(std::string, Issuer); + FLUENT_SETTING(std::string, Subject); + FLUENT_SETTING(std::string, Id); + FLUENT_SETTING_VECTOR_OR_SINGLE(std::string, Audience); + + FLUENT_SETTING_DEFAULT(TDuration, TokenTtl, DEFAULT_JWT_TOKEN_TTL); + + + // Helpers + class ISigningAlgorithm { + public: + virtual ~ISigningAlgorithm() = default; +#ifdef YDB_SDK_USE_NEW_JWT + virtual std::string sign(const std::string& data, std::error_code& ec) const = 0; +#else + virtual std::string sign(const std::string& data) const = 0; +#endif + virtual std::string name() const = 0; + }; + + // Interface implementation for jwt-cpp algorithm classes + template + class TJwtSigningAlgorithm: public ISigningAlgorithm { + public: + template + explicit TJwtSigningAlgorithm(T&&... args) + : Alg(std::forward(args)...) + { + } + +#ifdef YDB_SDK_USE_NEW_JWT + std::string sign(const std::string& data, std::error_code& ec) const override { + return Alg.sign(data, ec); + } +#else + std::string sign(const std::string& data) const override { + return Alg.sign(data); + } +#endif + + std::string name() const override { + return Alg.name(); + } + + private: + TAlg Alg; + }; + + std::shared_ptr SigningAlgorithm_; +}; + +std::shared_ptr CreateJwtTokenSource(const TJwtTokenSourceParams& params); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/exceptions/exceptions.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/exceptions/exceptions.h new file mode 100644 index 000000000000..bd96f8b530d5 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/exceptions/exceptions.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +class TYdbException : public yexception { +public: + using yexception::yexception; + TYdbException(const std::string& reason); +}; + +class TContractViolation : public TYdbException { +public: + TContractViolation(const std::string& reason); +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fatal_error_handlers/handlers.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fatal_error_handlers/handlers.h new file mode 100644 index 000000000000..a36e083895bd --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fatal_error_handlers/handlers.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +void ThrowFatalError(const std::string& str); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fluent_settings_helpers.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fluent_settings_helpers.h new file mode 100644 index 000000000000..a29f98321c26 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fluent_settings_helpers.h @@ -0,0 +1,50 @@ +#pragma once + +#define FLUENT_SETTING(type, name) \ + type name##_; \ + TSelf& name(const type& value) { \ + name##_ = value; \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_OPTIONAL(type, name) \ + std::optional name##_; \ + TSelf& name(const std::optional& value) { \ + name##_ = value; \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \ + type name##_ = defaultValue; \ + TSelf& name(const type& value) { \ + name##_ = value; \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_FLAG(name) \ + bool name##_ = false; \ + TSelf& name(bool value = true) { \ + name##_ = value; \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \ + TSelf& name() { \ + other##_ = value; \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_VECTOR(type, name) \ + std::vector name##_; \ + TSelf& Append##name(const type& value) { \ + name##_.push_back(value); \ + return static_cast(*this); \ + } + +#define FLUENT_SETTING_OPTIONAL_VECTOR(type, name) \ + std::optional> name##_; \ + TSelf& Append##name(const type& value) { \ + if (!name##_) name##_ = std::vector{}; \ + name##_->push_back(value); \ + return static_cast(*this); \ + } diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fwd.h new file mode 100644 index 000000000000..6168920659e1 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fwd.h @@ -0,0 +1,34 @@ +#pragma once + +namespace NYdb::inline V3 { + +template +struct TRequestSettings; + +template +struct TSimpleRequestSettings; + +template +struct TOperationRequestSettings; + +template +struct TS3Settings; + +class TStatus; +class TStreamPartStatus; + +class TOperation; + +class TYdbException; +class TContractViolation; + +class ICredentialsProvider; +class ICredentialsProviderFactory; + +class ITokenSource; + +namespace NStatusHelpers { +class TYdbErrorException; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/operation/operation.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/operation/operation.h new file mode 100644 index 000000000000..2909ffbaca91 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/operation/operation.h @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include + +#include + +#include +#include +#include + +namespace Ydb { +namespace Operations { + +class Operation; + +} // namespace Operations +} // namespace Ydb + +namespace NYdb::inline V3 { + +class TOperation { +public: + using TOperationId = NKikimr::NOperationId::TOperationId; + +public: + TOperation(TStatus&& status); + TOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + virtual ~TOperation() = default; + + const TOperationId& Id() const; + bool Ready() const; + const TStatus& Status() const; + TInstant CreateTime() const; + TInstant EndTime() const; + const std::string& CreatedBy() const; + + std::string ToString() const; + std::string ToJsonString() const; + void Out(IOutputStream& o) const; + +protected: + const Ydb::Operations::Operation& GetProto() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +using TAsyncOperation = NThreading::TFuture; + +TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h new file mode 100644 index 000000000000..2f2d4dca8a03 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h @@ -0,0 +1,77 @@ +#pragma once + +#include "fwd.h" + +#include "fluent_settings_helpers.h" + +#include + +#include +#include + +namespace NYdb::inline V3 { + +template +struct TRequestSettings { + using TSelf = TDerived; + using THeader = std::vector>; + + FLUENT_SETTING(std::string, TraceId); + FLUENT_SETTING(std::string, RequestType); + FLUENT_SETTING(THeader, Header); + FLUENT_SETTING(TDuration, ClientTimeout); + + TRequestSettings() = default; + + template + explicit TRequestSettings(const TRequestSettings& other) + : TraceId_(other.TraceId_) + , RequestType_(other.RequestType_) + , Header_(other.Header_) + , ClientTimeout_(other.ClientTimeout_) + {} +}; + +template +struct TSimpleRequestSettings : public TRequestSettings { + using TSelf = TDerived; + + TSimpleRequestSettings() = default; + + template + explicit TSimpleRequestSettings(const TSimpleRequestSettings& other) + : TRequestSettings(other) + {} +}; + +template +struct TOperationRequestSettings : public TSimpleRequestSettings { + using TSelf = TDerived; + + /* Cancel/timeout operation settings available from 18-8 YDB server version */ + FLUENT_SETTING(TDuration, OperationTimeout); + FLUENT_SETTING(TDuration, CancelAfter); + FLUENT_SETTING(TDuration, ForgetAfter); + FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); + FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); + + TOperationRequestSettings() = default; + + template + explicit TOperationRequestSettings(const TOperationRequestSettings& other) + : TSimpleRequestSettings(other) + , OperationTimeout_(other.OperationTimeout_) + , CancelAfter_(other.CancelAfter_) + , ForgetAfter_(other.ForgetAfter_) + , UseClientTimeoutForOperation_(other.UseClientTimeoutForOperation_) + , ReportCostInfo_(other.ReportCostInfo_) + {} + + TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) { + CancelAfter_ = cancelAfter; + OperationTimeout_ = operationTimeout; + return static_cast(*this); + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/s3_settings.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/s3_settings.h new file mode 100644 index 000000000000..8079a7eee906 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/s3_settings.h @@ -0,0 +1,28 @@ +#pragma once + +#include "fwd.h" +#include "fluent_settings_helpers.h" + +#include + +namespace NYdb::inline V3 { + +enum class ES3Scheme { + HTTP = 1 /* "http" */, + HTTPS = 2 /* "https" */, +}; + +template +struct TS3Settings { + using TSelf = TDerived; + + FLUENT_SETTING(std::string, Endpoint); + FLUENT_SETTING_DEFAULT(ES3Scheme, Scheme, ES3Scheme::HTTPS); + FLUENT_SETTING(std::string, Bucket); + FLUENT_SETTING(std::string, AccessKey); + FLUENT_SETTING(std::string, SecretKey); + // true by default for backward compatibility + FLUENT_SETTING_DEFAULT(bool, UseVirtualAddressing, true); +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h new file mode 100644 index 000000000000..09ab1a6699ea --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h @@ -0,0 +1,74 @@ +#pragma once + +#include + +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +//! Internal status representation +struct TPlainStatus; + +//! Represents status of call +class TStatus { +public: + TStatus(EStatus statusCode, NYdb::NIssue::TIssues&& issues); + TStatus(TPlainStatus&& plain); + + EStatus GetStatus() const; + const NYdb::NIssue::TIssues& GetIssues() const; + bool IsSuccess() const; + bool IsTransportError() const; + const std::string& GetEndpoint() const; + const std::multimap& GetResponseMetadata() const; + float GetConsumedRu() const; + + void Out(IOutputStream& out) const; + friend IOutputStream& operator<<(IOutputStream& out, const TStatus& st); + +protected: + void CheckStatusOk(const std::string& str) const; + void RaiseError(const std::string& str) const; +private: + class TImpl; + std::shared_ptr Impl_; +}; + +using TAsyncStatus = NThreading::TFuture; + +class TStreamPartStatus : public TStatus { +public: + TStreamPartStatus(TStatus&& status); + bool EOS() const; +}; + +namespace NStatusHelpers { + +class TYdbErrorException : public TYdbException { +public: + TYdbErrorException(TStatus status) + : Status_(std::move(status)) + { + *this << status; + } + + friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { + return out << e.Status_; + } + +private: + TStatus Status_; +}; + +void ThrowOnError(TStatus status, std::function onSuccess = [](TStatus) {}); +void ThrowOnErrorOrPrintIssues(TStatus status); + +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h new file mode 100644 index 000000000000..e6047b7db60e --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h @@ -0,0 +1,56 @@ +#pragma once + +#include "fwd.h" + +#include + +namespace NYdb::inline V3 { + +constexpr size_t TRANSPORT_STATUSES_FIRST = 401000; +constexpr size_t TRANSPORT_STATUSES_LAST = 401999; +constexpr size_t INTERNAL_CLIENT_FIRST = 402000; + +enum class EStatus : size_t { + // Server statuses + STATUS_UNDEFINED = 0, + SUCCESS = 400000, + BAD_REQUEST = 400010, + UNAUTHORIZED = 400020, + INTERNAL_ERROR = 400030, + ABORTED = 400040, + UNAVAILABLE = 400050, + OVERLOADED = 400060, + SCHEME_ERROR = 400070, + GENERIC_ERROR = 400080, + TIMEOUT = 400090, + BAD_SESSION = 400100, + PRECONDITION_FAILED = 400120, + ALREADY_EXISTS = 400130, + NOT_FOUND = 400140, + SESSION_EXPIRED = 400150, + CANCELLED = 400160, + UNDETERMINED = 400170, + UNSUPPORTED = 400180, + SESSION_BUSY = 400190, + EXTERNAL_ERROR = 400200, + + // Client statuses + // Cannot connect or unrecoverable network error. (map from gRPC UNAVAILABLE) + TRANSPORT_UNAVAILABLE = TRANSPORT_STATUSES_FIRST + 10, + // No more resources to accept RPC call + CLIENT_RESOURCE_EXHAUSTED = TRANSPORT_STATUSES_FIRST + 20, + // Network layer does not receive response in given time + CLIENT_DEADLINE_EXCEEDED = TRANSPORT_STATUSES_FIRST + 30, + // Unknown client error + CLIENT_INTERNAL_ERROR = TRANSPORT_STATUSES_FIRST + 50, + CLIENT_CANCELLED = TRANSPORT_STATUSES_FIRST + 60, + CLIENT_UNAUTHENTICATED = TRANSPORT_STATUSES_FIRST + 70, + // Unknown gRPC call + CLIENT_CALL_UNIMPLEMENTED = TRANSPORT_STATUSES_FIRST + 80, + // Attempt to read out of stream + CLIENT_OUT_OF_RANGE = TRANSPORT_STATUSES_FIRST + 90, + CLIENT_DISCOVERY_FAILED = INTERNAL_CLIENT_FIRST + 10, // Not used + CLIENT_LIMITS_REACHED = INTERNAL_CLIENT_FIRST + 20 +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/ydb.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/ydb.h new file mode 100644 index 000000000000..3957f1fedc8b --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/ydb.h @@ -0,0 +1,30 @@ +#pragma once + +#include "fwd.h" +#include "status_codes.h" + +namespace NYdb::inline V3 { + +enum class EDiscoveryMode { + //! Block in ctor (driver or client if database and/or auth token is overridden by client settings) + //! until we get list of endpoints, if list of endpoints become empty during executing requests + //! corresponding error will be returned. + //! Note: Even in Sync mode SDK will perform lazy async update of endpoints list + Sync, + //! Do not block in ctor to get endpoint list. + //! If list of endpoints is empty in time of request (or becomes empty during execution) the request will be pending until + //! we got endpoint list. The error will be returned if the endpoint list + //! is empty and discovery failed + //! This method is a bit more "user friendly" but can produce additional hidden latency + Async +}; + +enum class EBalancingPolicy { + //! Use all available cluster nodes regardless datacenter locality + UseAllNodes, + //! Use preferable location, + //! params is a name of location (VLA, MAN), if params is empty local datacenter is used + UsePreferableLocation +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/fwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/fwd.h new file mode 100644 index 000000000000..41fcc5635f8a --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/fwd.h @@ -0,0 +1,15 @@ +#pragma once + +namespace NYdb::inline V3 { + +class TType; +class TTypeParser; +class TTypeBuilder; +class TValue; +class TValueParser; +class TValueBuilder; + +template +class TValueBuilderBase; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h new file mode 100644 index 000000000000..3c2787090c83 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h @@ -0,0 +1,541 @@ +#pragma once + +#include "fwd.h" + +#include + +#include +#include + +namespace Ydb { + class Type; + class Value; +} + +namespace NYdb::inline V3 { + +class TResultSetParser; + +//! Representation of YDB type. +class TType { + friend class TProtoAccessor; +public: + TType(const Ydb::Type& typeProto); + TType(Ydb::Type&& typeProto); + + std::string ToString() const; + void Out(IOutputStream& o) const; + + const Ydb::Type& GetProto() const; + Ydb::Type& GetProto(); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +enum class EPrimitiveType { + Bool = 0x0006, + Int8 = 0x0007, + Uint8 = 0x0005, + Int16 = 0x0008, + Uint16 = 0x0009, + Int32 = 0x0001, + Uint32 = 0x0002, + Int64 = 0x0003, + Uint64 = 0x0004, + Float = 0x0021, + Double = 0x0020, + Date = 0x0030, + Datetime = 0x0031, + Timestamp = 0x0032, + Interval = 0x0033, + Date32 = 0x0040, + Datetime64 = 0x0041, + Timestamp64 = 0x0042, + Interval64 = 0x0043, + TzDate = 0x0034, + TzDatetime = 0x0035, + TzTimestamp = 0x0036, + String = 0x1001, + Utf8 = 0x1200, + Yson = 0x1201, + Json = 0x1202, + Uuid = 0x1203, + JsonDocument = 0x1204, + DyNumber = 0x1302, +}; + +struct TDecimalType { + uint8_t Precision; + uint8_t Scale; + + TDecimalType(uint8_t precision, uint8_t scale) + : Precision(precision) + , Scale(scale) {} +}; + +struct TPgType { + std::string TypeName; + std::string TypeModifier; + + uint32_t Oid = 0; + int16_t Typlen = 0; + int32_t Typmod = 0; + + TPgType(const std::string& typeName, const std::string& typeModifier = {}) + : TypeName(typeName) + , TypeModifier(typeModifier) + {} +}; + +//! Types can be complex, so TTypeParser allows to traverse through this hierarchies. +class TTypeParser : public TMoveOnly { + friend class TValueParser; +public: + enum class ETypeKind { + Primitive, + Decimal, + Optional, + List, + Tuple, + Struct, + Dict, + Variant, + Void, + Null, + EmptyList, + EmptyDict, + Tagged, + Pg + }; + +public: + TTypeParser(TTypeParser&&); + TTypeParser(const TType& type); + + ~TTypeParser(); + + ETypeKind GetKind() const; + + EPrimitiveType GetPrimitive() const; + TDecimalType GetDecimal() const; + TPgType GetPg() const; + + // Optional + void OpenOptional(); + void CloseOptional(); + + // List + void OpenList(); + void CloseList(); + + // Struct + void OpenStruct(); + bool TryNextMember(); + const std::string& GetMemberName(); + void CloseStruct(); + + // Tuple + void OpenTuple(); + bool TryNextElement(); + void CloseTuple(); + + // Dict + void OpenDict(); + void DictKey(); + void DictPayload(); + void CloseDict(); + + // Variant + void OpenVariant(size_t index); + void OpenVariant(); + void CloseVariant(); + + // Tagged + void OpenTagged(); + const std::string& GetTag(); + void CloseTagged(); + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +bool TypesEqual(const TType& t1, const TType& t2); + +std::string FormatType(const TType& type); + +//! Used to create arbitrary type. +//! To create complex type, corresponding scope should be opened by Begin*/End* calls +//! To create complex repeated type, Add* should be called at least once +class TTypeBuilder : public TMoveOnly { + friend class TValueBuilderImpl; +public: + TTypeBuilder(TTypeBuilder&&); + TTypeBuilder(); + + ~TTypeBuilder(); + + TTypeBuilder& Primitive(const EPrimitiveType& primitiveType); + TTypeBuilder& Decimal(const TDecimalType& decimalType); + TTypeBuilder& Pg(const TPgType& pgType); + + // Optional + TTypeBuilder& BeginOptional(); + TTypeBuilder& EndOptional(); + TTypeBuilder& Optional(const TType& itemType); + + // List + TTypeBuilder& BeginList(); + TTypeBuilder& EndList(); + TTypeBuilder& List(const TType& itemType); + + // Struct + TTypeBuilder& BeginStruct(); + TTypeBuilder& AddMember(const std::string& memberName); + TTypeBuilder& AddMember(const std::string& memberName, const TType& memberType); + TTypeBuilder& EndStruct(); + + // Tuple + TTypeBuilder& BeginTuple(); + TTypeBuilder& AddElement(); + TTypeBuilder& AddElement(const TType& elementType); + TTypeBuilder& EndTuple(); + + // Dict + TTypeBuilder& BeginDict(); + TTypeBuilder& DictKey(); + TTypeBuilder& DictKey(const TType& keyType); + TTypeBuilder& DictPayload(); + TTypeBuilder& DictPayload(const TType& payloadType); + TTypeBuilder& EndDict(); + + // Tagged + TTypeBuilder& BeginTagged(const std::string& tag); + TTypeBuilder& EndTagged(); + TTypeBuilder& Tagged(const std::string& tag, const TType& itemType); + + TType Build(); + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +struct TDecimalValue { + std::string ToString() const; + TDecimalValue(const Ydb::Value& decimalValueProto, const TDecimalType& decimalType); + TDecimalValue(const std::string& decimalString, uint8_t precision, uint8_t scale); + + TDecimalType DecimalType_; + uint64_t Low_; + int64_t Hi_; +}; + +struct TPgValue { + enum EPgValueKind { + VK_NULL, + VK_TEXT, + VK_BINARY + }; + + TPgValue(const Ydb::Value& pgValueProto, const TPgType& pgType); + TPgValue(EPgValueKind kind, const std::string& content, const TPgType& pgType); + bool IsNull() const; + bool IsText() const; + + TPgType PgType_; + EPgValueKind Kind_ = VK_NULL; + std::string Content_; +}; + +struct TUuidValue { + std::string ToString() const; + TUuidValue(uint64_t low_128, uint64_t high_128); + TUuidValue(const Ydb::Value& uuidValueProto); + TUuidValue(const std::string& uuidString); + + union { + char Bytes[16]; + uint64_t Halfs[2]; + } Buf_; +}; + +//! Representation of YDB value. +class TValue { + friend class TValueParser; + friend class TProtoAccessor; +public: + TValue(const TType& type, const Ydb::Value& valueProto); + TValue(const TType& type, Ydb::Value&& valueProto); + + const TType& GetType() const; + TType & GetType(); + + const Ydb::Value& GetProto() const; + Ydb::Value& GetProto(); + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +class TValueParser : public TMoveOnly { + friend class TResultSetParser; +public: + TValueParser(TValueParser&&); + TValueParser(const TValue& value); + + ~TValueParser(); + + TTypeParser::ETypeKind GetKind() const; + EPrimitiveType GetPrimitiveType() const; + + bool GetBool() const; + int8_t GetInt8() const; + uint8_t GetUint8() const; + int16_t GetInt16() const; + uint16_t GetUint16() const; + int32_t GetInt32() const; + uint32_t GetUint32() const; + int64_t GetInt64() const; + uint64_t GetUint64() const; + float GetFloat() const; + double GetDouble() const; + TInstant GetDate() const; + TInstant GetDatetime() const; + TInstant GetTimestamp() const; + int64_t GetInterval() const; + int32_t GetDate32() const; + int64_t GetDatetime64() const; + int64_t GetTimestamp64() const; + int64_t GetInterval64() const; + const std::string& GetTzDate() const; + const std::string& GetTzDatetime() const; + const std::string& GetTzTimestamp() const; + const std::string& GetString() const; + const std::string& GetUtf8() const; + const std::string& GetYson() const; + const std::string& GetJson() const; + TDecimalValue GetDecimal() const; + TPgValue GetPg() const; + TUuidValue GetUuid() const; + const std::string& GetJsonDocument() const; + const std::string& GetDyNumber() const; + + std::optional GetOptionalBool() const; + std::optional GetOptionalInt8() const; + std::optional GetOptionalUint8() const; + std::optional GetOptionalInt16() const; + std::optional GetOptionalUint16() const; + std::optional GetOptionalInt32() const; + std::optional GetOptionalUint32() const; + std::optional GetOptionalInt64() const; + std::optional GetOptionalUint64() const; + std::optional GetOptionalFloat() const; + std::optional GetOptionalDouble() const; + std::optional GetOptionalDate() const; + std::optional GetOptionalDatetime() const; + std::optional GetOptionalTimestamp() const; + std::optional GetOptionalInterval() const; + std::optional GetOptionalDate32() const; + std::optional GetOptionalDatetime64() const; + std::optional GetOptionalTimestamp64() const; + std::optional GetOptionalInterval64() const; + std::optional GetOptionalTzDate() const; + std::optional GetOptionalTzDatetime() const; + std::optional GetOptionalTzTimestamp() const; + std::optional GetOptionalString() const; + std::optional GetOptionalUtf8() const; + std::optional GetOptionalYson() const; + std::optional GetOptionalJson() const; + std::optional GetOptionalDecimal() const; + std::optional GetOptionalUuid() const; + std::optional GetOptionalJsonDocument() const; + std::optional GetOptionalDyNumber() const; + + // Optional + void OpenOptional(); + bool IsNull() const; + void CloseOptional(); + + // List + void OpenList(); + void CloseList(); + bool TryNextListItem(); + + // Struct + void OpenStruct(); + bool TryNextMember(); + const std::string& GetMemberName() const; + void CloseStruct(); + + // Tuple + void OpenTuple(); + bool TryNextElement(); + void CloseTuple(); + + // Dict + void OpenDict(); + bool TryNextDictItem(); + void DictKey(); + void DictPayload(); + void CloseDict(); + + // Variant + void OpenVariant(); + void CloseVariant(); + + // Tagged + void OpenTagged(); + const std::string& GetTag() const; + void CloseTagged(); + +private: + TValueParser(const TType& type); + void Reset(const Ydb::Value& value); + + class TImpl; + std::unique_ptr Impl_; +}; + +class TValueBuilderImpl; + +template +class TValueBuilderBase : public TMoveOnly { + friend TDerived; +public: + TDerived& Bool(bool value); + TDerived& Int8(int8_t value); + TDerived& Uint8(uint8_t value); + TDerived& Int16(int16_t value); + TDerived& Uint16(uint16_t value); + TDerived& Int32(int32_t value); + TDerived& Uint32(uint32_t value); + TDerived& Int64(int64_t value); + TDerived& Uint64(uint64_t value); + TDerived& Float(float value); + TDerived& Double(double value); + TDerived& Date(const TInstant& value); + TDerived& Datetime(const TInstant& value); + TDerived& Timestamp(const TInstant& value); + TDerived& Interval(int64_t value); + TDerived& TzDate(const std::string& value); + TDerived& TzDatetime(const std::string& value); + TDerived& TzTimestamp(const std::string& value); + TDerived& String(const std::string& value); + TDerived& Utf8(const std::string& value); + TDerived& Yson(const std::string& value); + TDerived& Json(const std::string& value); + TDerived& Decimal(const TDecimalValue& value); + TDerived& Pg(const TPgValue& value); + TDerived& Uuid(const TUuidValue& value); + TDerived& JsonDocument(const std::string& value); + TDerived& DyNumber(const std::string& value); + TDerived& Date32(const int32_t value); + TDerived& Datetime64(const int64_t value); + TDerived& Timestamp64(const int64_t value); + TDerived& Interval64(const int64_t value); + + TDerived& OptionalBool(const std::optional& value); + TDerived& OptionalInt8(const std::optional& value); + TDerived& OptionalUint8(const std::optional& value); + TDerived& OptionalInt16(const std::optional& value); + TDerived& OptionalUint16(const std::optional& value); + TDerived& OptionalInt32(const std::optional& value); + TDerived& OptionalUint32(const std::optional& value); + TDerived& OptionalInt64(const std::optional& value); + TDerived& OptionalUint64(const std::optional& value); + TDerived& OptionalFloat(const std::optional& value); + TDerived& OptionalDouble(const std::optional& value); + TDerived& OptionalDate(const std::optional& value); + TDerived& OptionalDatetime(const std::optional& value); + TDerived& OptionalTimestamp(const std::optional& value); + TDerived& OptionalInterval(const std::optional& value); + TDerived& OptionalTzDate(const std::optional& value); + TDerived& OptionalTzDatetime(const std::optional& value); + TDerived& OptionalTzTimestamp(const std::optional& value); + TDerived& OptionalString(const std::optional& value); + TDerived& OptionalUtf8(const std::optional& value); + TDerived& OptionalYson(const std::optional& value); + TDerived& OptionalJson(const std::optional& value); + TDerived& OptionalUuid(const std::optional& value); + TDerived& OptionalJsonDocument(const std::optional& value); + TDerived& OptionalDyNumber(const std::optional& value); + TDerived& OptionalDate32(const std::optional& value); + TDerived& OptionalDatetime64(const std::optional& value); + TDerived& OptionalTimestamp64(const std::optional& value); + TDerived& OptionalInterval64(const std::optional& value); + + // Optional + TDerived& BeginOptional(); + TDerived& EndOptional(); + TDerived& EmptyOptional(const TType& itemType); + TDerived& EmptyOptional(EPrimitiveType itemType); + TDerived& EmptyOptional(); + + // List + TDerived& BeginList(); + TDerived& AddListItem(); + TDerived& AddListItem(const TValue& itemValue); + TDerived& AddListItem(TValue&& itemValue); + TDerived& EndList(); + TDerived& EmptyList(const TType& itemType); + TDerived& EmptyList(); + + // Struct + TDerived& BeginStruct(); + TDerived& AddMember(const std::string& memberName); + TDerived& AddMember(const std::string& memberName, const TValue& memberValue); + TDerived& AddMember(const std::string& memberName, TValue&& memberValue); + TDerived& EndStruct(); + + // Tuple + TDerived& BeginTuple(); + TDerived& AddElement(); + TDerived& AddElement(const TValue& elementValue); + TDerived& EndTuple(); + + // Dict + TDerived& BeginDict(); + TDerived& AddDictItem(); + TDerived& DictKey(); + TDerived& DictKey(const TValue& keyValue); + TDerived& DictPayload(); + TDerived& DictPayload(const TValue& payloadValue); + TDerived& EndDict(); + TDerived& EmptyDict(const TType& keyType, const TType& payloadType); + TDerived& EmptyDict(); + + // Tagged + TDerived& BeginTagged(const std::string& tag); + TDerived& EndTagged(); + +protected: + TValueBuilderBase(TValueBuilderBase&&); + + TValueBuilderBase(); + + TValueBuilderBase(const TType& type); + + TValueBuilderBase(Ydb::Type& type, Ydb::Value& value); + + ~TValueBuilderBase(); + + void CheckValue(); + +private: + std::unique_ptr Impl_; +}; + +class TValueBuilder : public TValueBuilderBase { +public: + TValueBuilder(); + + TValueBuilder(const TType& type); + + TValue Build(); +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/issue/yql_issue.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/issue/yql_issue.h new file mode 100644 index 000000000000..01779537a2e4 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/issue/yql_issue.h @@ -0,0 +1,351 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace NYdb::NIssue { + +using TIssueCode = uint32_t; +constexpr TIssueCode DEFAULT_ERROR = 0; +constexpr TIssueCode UNEXPECTED_ERROR = 1; + +enum class ESeverity : uint32_t { + Fatal = 0, + Error = 1, + Warning = 2, + Info = 3, +}; + +std::string SeverityToString(ESeverity severity); + +void SanitizeNonAscii(std::string& s); + +/////////////////////////////////////////////////////////////////////////////// +// TPosition +/////////////////////////////////////////////////////////////////////////////// +struct TPosition { + uint32_t Column = 0U; + uint32_t Row = 0U; + std::string File; + + TPosition() = default; + + TPosition(uint32_t column, uint32_t row, const std::string& file = {}) + : Column(column) + , Row(row) + , File(file) + { + SanitizeNonAscii(File); + } + + explicit operator bool() const { + return HasValue(); + } + + inline bool HasValue() const { + return Row | Column; + } + + inline bool operator==(const TPosition& other) const { + return Column == other.Column && Row == other.Row && File == other.File; + } + + inline bool operator<(const TPosition& other) const { + return std::tie(Row, Column, File) < std::tie(other.Row, other.Column, other.File); + } +}; + +struct TRange { + TPosition Position; + TPosition EndPosition; + + TRange() = default; + + TRange(TPosition position) + : Position(position) + , EndPosition(position) + { + } + + TRange(TPosition position, TPosition endPosition) + : Position(position) + , EndPosition(endPosition) + { + } + + inline bool IsRange() const { + return !(Position == EndPosition); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// TIssue +/////////////////////////////////////////////////////////////////////////////// + +class TIssue; +using TIssuePtr = TIntrusivePtr; +class TIssue: public TThrRefBase { + std::vector> Children_; + std::string Message; +public: + TPosition Position; + TPosition EndPosition; + TIssueCode IssueCode = 0U; + ESeverity Severity = ESeverity::Error; + + TIssue() = default; + + template + explicit TIssue(const T& message) + : Message(message) + , Position(TPosition()) + , EndPosition(TPosition()) + { + SanitizeNonAscii(Message); + } + + template + TIssue(TPosition position, const T& message) + : Message(message) + , Position(position) + , EndPosition(position) + { + SanitizeNonAscii(Message); + } + + inline TRange Range() const { + return {Position, EndPosition}; + } + + template + TIssue(TPosition position, TPosition endPosition, const T& message) + : Message(message) + , Position(position) + , EndPosition(endPosition) + { + SanitizeNonAscii(Message); + } + + inline bool operator==(const TIssue& other) const { + return Position == other.Position && Message == other.Message + && IssueCode == other.IssueCode; + } + + uint64_t Hash() const noexcept; + + TIssue& SetCode(TIssueCode id, ESeverity severity) { + IssueCode = id; + Severity = severity; + return *this; + } + + TIssue& SetMessage(const std::string& msg) { + Message = msg; + SanitizeNonAscii(Message); + return *this; + } + + ESeverity GetSeverity() const { + return Severity; + } + + TIssueCode GetCode() const { + return IssueCode; + } + + const std::string& GetMessage() const { + return Message; + } + + TIssue& AddSubIssue(TIntrusivePtr issue) { + Severity = static_cast(std::min(static_cast(issue->GetSeverity()), + static_cast(Severity))); + Children_.push_back(issue); + return *this; + } + + const std::vector>& GetSubIssues() const { + return Children_; + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + + std::string ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + // Unsafe method. Doesn't call SanitizeNonAscii(Message) + std::string* MutableMessage() { + return &Message; + } + + TIssue& CopyWithoutSubIssues(const TIssue& src) { + Message = src.Message; + IssueCode = src.IssueCode; + Severity = src.Severity; + Position = src.Position; + EndPosition = src.EndPosition; + return *this; + } +}; + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function fn, std::function afterChildrenFn = {}); + +/////////////////////////////////////////////////////////////////////////////// +// TIssues +/////////////////////////////////////////////////////////////////////////////// +class TIssues { +public: + TIssues() = default; + + inline TIssues(const std::vector& issues) + : Issues_(issues) + { + } + + inline TIssues(const std::initializer_list& issues) + : TIssues(std::vector(issues)) + { + } + + inline TIssues(const TIssues& rhs) + : Issues_(rhs.Issues_) + { + } + + inline TIssues& operator=(const TIssues& rhs) { + Issues_ = rhs.Issues_; + return *this; + } + + inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) + { + } + + inline TIssues& operator=(TIssues&& rhs) { + Issues_ = std::move(rhs.Issues_); + return *this; + } + + template void AddIssue(Args&& ... args) { + Issues_.emplace_back(std::forward(args)...); + } + + inline void AddIssues(const TIssues& errors) { + Issues_.insert(Issues_.end(), + errors.Issues_.begin(), errors.Issues_.end()); + } + + inline void AddIssues(const TPosition& pos, const TIssues& errors) { + Issues_.reserve(Issues_.size() + errors.Size()); + for (const auto& e: errors) { + TIssue& issue = Issues_.emplace_back(); + *issue.MutableMessage() = e.GetMessage(); // No need to sanitize message, it has already been sanitized. + issue.Position = pos; + issue.SetCode(e.IssueCode, e.Severity); + } + } + + inline const TIssue* begin() const { + return Issues_.data(); + } + + inline const TIssue* end() const { + return Issues_.data() + Issues_.size(); + } + + inline TIssue& back() { + return Issues_.back(); + } + + inline const TIssue& back() const { + return Issues_.back(); + } + + inline bool Empty() const { + return Issues_.empty(); + } + + explicit operator bool() const noexcept { + return !Issues_.empty(); + } + + inline size_t Size() const { + return Issues_.size(); + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + void PrintWithProgramTo( + IOutputStream& out, + const std::string& programFilename, + const std::string& programText) const; + + inline std::string ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + std::string ToOneLineString() const { + return ToString(true); + } + + inline void Clear() { + Issues_.clear(); + } + + void Reserve(size_t capacity) { + Issues_.reserve(capacity); + } + +private: + std::vector Issues_; +}; + +class TErrorException : public yexception { + const TIssueCode Code_; +public: + explicit TErrorException(TIssueCode code) + : Code_(code) + {} + TIssueCode GetCode() const { + return Code_; + } +}; + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos = TPosition()); +std::optional TryParseTerminationMessage(std::string_view& message); + +} // namespace NYql + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TPosition& pos); + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TRange& pos); + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TIssue& error); + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TIssues& error); + +template <> +struct THash { + inline size_t operator()(const NYdb::NIssue::TIssue& err) const { + return err.Hash(); + } +}; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/jwt/jwt.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/jwt/jwt.h new file mode 100644 index 000000000000..4d3d90f31532 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/jwt/jwt.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace NYdb { + +struct TJwtParams { + std::string PrivKey; + std::string PubKey; + std::string AccountId; + std::string KeyId; +}; + +TJwtParams ParseJwtParams(const std::string& jsonParamsStr); +std::string MakeSignedJwt( + const TJwtParams& params, + const TDuration& lifetime = TDuration::Hours(1) +); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/operation_id/operation_id.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/operation_id/operation_id.h new file mode 100644 index 000000000000..ed509e16844a --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/operation_id/operation_id.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +namespace Ydb { + class TOperationId; +} + +namespace NKikimr { +namespace NOperationId { + +class TOperationId { +public: + enum EKind : int { + UNUSED = 0, + OPERATION_DDL = 1, + OPERATION_DML = 2, + SESSION_YQL = 3, + PREPARED_QUERY_ID = 4, + CMS_REQUEST = 5, + EXPORT = 6, + BUILD_INDEX = 7, + IMPORT = 8, + SCRIPT_EXECUTION = 9, + SS_BG_TASKS = 10, + }; + + struct TData { + std::string Key; + std::string Value; + }; + + TOperationId(); + explicit TOperationId(const std::string& string, bool allowEmpty = false); + + TOperationId(const TOperationId& other); + TOperationId(TOperationId&& other); + + TOperationId& operator=(const TOperationId& other); + TOperationId& operator=(TOperationId&& other); + + ~TOperationId(); + + EKind GetKind() const; + void SetKind(const EKind& kind); + + std::vector GetData() const; + + void AddOptionalValue(const std::string& key, const std::string& value); + const std::vector& GetValue(const std::string& key) const; + + std::string GetSubKind() const; + std::string ToString() const; + + const Ydb::TOperationId& GetProto() const; +private: + class TImpl; + std::unique_ptr Impl; +}; + +std::string ProtoToString(const Ydb::TOperationId& proto); + +void AddOptionalValue(Ydb::TOperationId& operarionId, const std::string& key, const std::string& value); + +TOperationId::EKind ParseKind(const std::string_view value); + +std::string FormatPreparedQueryIdCompat(const std::string& str); +bool DecodePreparedQueryIdCompat(const std::string& in, std::string& out); + +} // namespace NOperationId +} // namespace NKikimr diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/retry/retry_policy.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/retry/retry_policy.h new file mode 100644 index 000000000000..bc553cbd9c96 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/retry/retry_policy.h @@ -0,0 +1,289 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +//! Retry policy. +//! Calculates delay before next retry (if any). +//! Has several default implementations: +//! - exponential backoff policy; +//! - retries with fixed interval; +//! - no retries. + +enum class ERetryErrorClass { + // This error shouldn't be retried. + NoRetry, + + // This error could be retried in short period of time. + ShortRetry, + + // This error requires waiting before it could be retried. + LongRetry, +}; + +template +struct IRetryPolicy { + using TPtr = std::shared_ptr; + + using TRetryClassFunction = std::function::TFuncParam...)>; + + //! Retry state of single request. + struct IRetryState { + using TPtr = std::unique_ptr; + + virtual ~IRetryState() = default; + + //! Calculate delay before next retry if next retry is allowed. + //! Returns empty maybe if retry is not allowed anymore. + [[nodiscard]] virtual std::optional GetNextRetryDelay(typename TTypeTraits::TFuncParam... args) = 0; + }; + + virtual ~IRetryPolicy() = default; + + //! Function that is called after first error + //! to find out a futher retry behaviour. + //! Retry state is expected to be created for the whole single retry session. + [[nodiscard]] virtual typename IRetryState::TPtr CreateRetryState() const = 0; + + //! + //! Default implementations. + //! + + static TPtr GetNoRetryPolicy(); // Denies all kind of retries. + + //! Randomized exponential backoff policy. + static TPtr GetExponentialBackoffPolicy(TRetryClassFunction retryClassFunction, + TDuration minDelay = TDuration::MilliSeconds(10), + // Delay for statuses that require waiting before retry (such as OVERLOADED). + TDuration minLongRetryDelay = TDuration::MilliSeconds(200), + TDuration maxDelay = TDuration::Seconds(30), + size_t maxRetries = std::numeric_limits::max(), + TDuration maxTime = TDuration::Max(), + double scaleFactor = 2.0); + + //! Randomized fixed interval policy. + static TPtr GetFixedIntervalPolicy(TRetryClassFunction retryClassFunction, + TDuration delay = TDuration::MilliSeconds(100), + // Delay for statuses that require waiting before retry (such as OVERLOADED). + TDuration longRetryDelay = TDuration::MilliSeconds(300), + size_t maxRetries = std::numeric_limits::max(), + TDuration maxTime = TDuration::Max()); +}; + +template +struct TNoRetryPolicy : IRetryPolicy { + using IRetryState = typename IRetryPolicy::IRetryState; + + struct TNoRetryState : IRetryState { + std::optional GetNextRetryDelay(typename TTypeTraits::TFuncParam...) override { + return std::nullopt; + } + }; + + typename IRetryState::TPtr CreateRetryState() const override { + return std::make_unique(); + } +}; + +namespace NRetryDetails { +inline TDuration RandomizeDelay(TDuration baseDelay) { + const TDuration::TValue half = baseDelay.GetValue() / 2; + return TDuration::FromValue(half + RandomNumber(half)); +} +} // namespace NRetryDetails + +template +struct TExponentialBackoffPolicy : IRetryPolicy { + using IRetryPolicy = IRetryPolicy; + using IRetryState = typename IRetryPolicy::IRetryState; + + struct TExponentialBackoffState : IRetryState { + TExponentialBackoffState(typename IRetryPolicy::TRetryClassFunction retryClassFunction, + TDuration minDelay, + TDuration minLongRetryDelay, + TDuration maxDelay, + size_t maxRetries, + TDuration maxTime, + double scaleFactor) + : MinLongRetryDelay(minLongRetryDelay) + , MaxDelay(maxDelay) + , MaxRetries(maxRetries) + , MaxTime(maxTime) + , ScaleFactor(scaleFactor) + , StartTime(maxTime != TDuration::Max() ? TInstant::Now() : TInstant::Zero()) + , CurrentDelay(minDelay) + , AttemptsDone(0) + , RetryClassFunction(std::move(retryClassFunction)) + { + } + + std::optional GetNextRetryDelay(typename TTypeTraits::TFuncParam... args) override { + const ERetryErrorClass errorClass = RetryClassFunction(args...); + if (errorClass == ERetryErrorClass::NoRetry || AttemptsDone >= MaxRetries || StartTime && TInstant::Now() - StartTime >= MaxTime) { + return std::nullopt; + } + + if (errorClass == ERetryErrorClass::LongRetry) { + CurrentDelay = Max(CurrentDelay, MinLongRetryDelay); + } + + const TDuration delay = NRetryDetails::RandomizeDelay(CurrentDelay); + + if (CurrentDelay < MaxDelay) { + CurrentDelay = Min(CurrentDelay * ScaleFactor, MaxDelay); + } + + ++AttemptsDone; + return delay; + } + + const TDuration MinLongRetryDelay; + const TDuration MaxDelay; + const size_t MaxRetries; + const TDuration MaxTime; + const double ScaleFactor; + const TInstant StartTime; + TDuration CurrentDelay; + size_t AttemptsDone; + typename IRetryPolicy::TRetryClassFunction RetryClassFunction; + }; + + TExponentialBackoffPolicy(typename IRetryPolicy::TRetryClassFunction retryClassFunction, + TDuration minDelay, + TDuration minLongRetryDelay, + TDuration maxDelay, + size_t maxRetries, + TDuration maxTime, + double scaleFactor) + : MinDelay(minDelay) + , MinLongRetryDelay(minLongRetryDelay) + , MaxDelay(maxDelay) + , MaxRetries(maxRetries) + , MaxTime(maxTime) + , ScaleFactor(scaleFactor) + , RetryClassFunction(std::move(retryClassFunction)) + { + Y_ASSERT(RetryClassFunction); + Y_ASSERT(MinDelay < MaxDelay); + Y_ASSERT(MinLongRetryDelay < MaxDelay); + Y_ASSERT(MinLongRetryDelay >= MinDelay); + Y_ASSERT(ScaleFactor > 1.0); + Y_ASSERT(MaxRetries > 0); + Y_ASSERT(MaxTime > MinDelay); + } + + typename IRetryState::TPtr CreateRetryState() const override { + return std::make_unique(RetryClassFunction, MinDelay, MinLongRetryDelay, MaxDelay, MaxRetries, MaxTime, ScaleFactor); + } + + const TDuration MinDelay; + const TDuration MinLongRetryDelay; + const TDuration MaxDelay; + const size_t MaxRetries; + const TDuration MaxTime; + const double ScaleFactor; + typename IRetryPolicy::TRetryClassFunction RetryClassFunction; +}; + +template +struct TFixedIntervalPolicy : IRetryPolicy { + using IRetryPolicy = IRetryPolicy; + using IRetryState = typename IRetryPolicy::IRetryState; + + struct TFixedIntervalState : IRetryState { + TFixedIntervalState(typename IRetryPolicy::TRetryClassFunction retryClassFunction, + TDuration delay, + TDuration longRetryDelay, + size_t maxRetries, + TDuration maxTime) + : Delay(delay) + , LongRetryDelay(longRetryDelay) + , MaxRetries(maxRetries) + , MaxTime(maxTime) + , StartTime(maxTime != TDuration::Max() ? TInstant::Now() : TInstant::Zero()) + , AttemptsDone(0) + , RetryClassFunction(std::move(retryClassFunction)) + { + } + + std::optional GetNextRetryDelay(typename TTypeTraits::TFuncParam... args) override { + const ERetryErrorClass errorClass = RetryClassFunction(args...); + if (errorClass == ERetryErrorClass::NoRetry || AttemptsDone >= MaxRetries || StartTime && TInstant::Now() - StartTime >= MaxTime) { + return std::nullopt; + } + + const TDuration delay = NRetryDetails::RandomizeDelay(errorClass == ERetryErrorClass::LongRetry ? LongRetryDelay : Delay); + + ++AttemptsDone; + return delay; + } + + const TDuration Delay; + const TDuration LongRetryDelay; + const size_t MaxRetries; + const TDuration MaxTime; + const TInstant StartTime; + size_t AttemptsDone; + typename IRetryPolicy::TRetryClassFunction RetryClassFunction; + }; + + TFixedIntervalPolicy(typename IRetryPolicy::TRetryClassFunction retryClassFunction, + TDuration delay, + TDuration longRetryDelay, + size_t maxRetries, + TDuration maxTime) + : Delay(delay) + , LongRetryDelay(longRetryDelay) + , MaxRetries(maxRetries) + , MaxTime(maxTime) + , RetryClassFunction(std::move(retryClassFunction)) + { + Y_ASSERT(RetryClassFunction); + Y_ASSERT(MaxTime > Delay); + Y_ASSERT(MaxTime > LongRetryDelay); + Y_ASSERT(LongRetryDelay >= Delay); + } + + typename IRetryState::TPtr CreateRetryState() const override { + return std::make_unique(RetryClassFunction, Delay, LongRetryDelay, MaxRetries, MaxTime); + } + + const TDuration Delay; + const TDuration LongRetryDelay; + const size_t MaxRetries; + const TDuration MaxTime; + typename IRetryPolicy::TRetryClassFunction RetryClassFunction; +}; + +template +typename IRetryPolicy::TPtr IRetryPolicy::GetNoRetryPolicy() { + return std::make_shared>(); +} + +template +typename IRetryPolicy::TPtr IRetryPolicy::GetExponentialBackoffPolicy(TRetryClassFunction retryClassFunction, + TDuration minDelay, + TDuration minLongRetryDelay, + TDuration maxDelay, + size_t maxRetries, + TDuration maxTime, + double scaleFactor) +{ + return std::make_shared>(std::move(retryClassFunction), minDelay, minLongRetryDelay, maxDelay, maxRetries, maxTime, scaleFactor); +} + +template +typename IRetryPolicy::TPtr IRetryPolicy::GetFixedIntervalPolicy(TRetryClassFunction retryClassFunction, + TDuration delay, + TDuration longRetryDelay, + size_t maxRetries, + TDuration maxTime) +{ + return std::make_shared>(std::move(retryClassFunction), delay, longRetryDelay, maxRetries, maxTime); +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/helpers/helpers.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/helpers/helpers.h new file mode 100644 index 000000000000..09c31c959c2d --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/helpers/helpers.h @@ -0,0 +1,83 @@ +#pragma once + +#include + +namespace NUtils { + +char* ToLower(char* str) noexcept(noexcept(std::tolower(0))); +std::string ToLower(const std::string& str); +void ToLower(std::string& str); + +std::string ToTitle(const std::string& s); + +void RemoveAll(std::string& str, char ch); + +bool TrySplitOn(std::string_view src, std::string_view& l, std::string_view& r, size_t pos, size_t len); + +bool TrySplit(std::string_view src, std::string_view& l, std::string_view& r, char delim); +bool TrySplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim); +bool TryRSplit(std::string_view src, std::string_view& l, std::string_view& r, char delim); +bool TryRSplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim); + +void Split(std::string_view src, std::string_view& l, std::string_view& r, char delim); +void Split(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim); +void RSplit(std::string_view src, std::string_view& l, std::string_view& r, char delim); +void RSplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim); + +std::string_view NextTok(std::string_view& src, char delim); +std::string_view NextTok(std::string_view& src, std::string_view delim); +bool NextTok(std::string_view& src, std::string_view& tok, char delim); +bool NextTok(std::string_view& src, std::string_view& tok, std::string_view delim); + +std::string_view RNextTok(std::string_view& src, char delim); +std::string_view RNextTok(std::string_view& src, std::string_view delim); + +std::string_view After(std::string_view src, char c); +std::string_view Before(std::string_view src, char c); + +size_t SumLength() noexcept; + +template +size_t SumLength(const std::string_view s1, const R&... r) noexcept { + return s1.size() + SumLength(r...); +} + +/** + * Appends `src` to string dst of size `dsize` (unlike `strncat`, `dsize` is + * the full size of `dst`, not space left). At most `dsize-1` characters + * will be copied. Always NUL terminates (unless `dsize <= strlen(dst)`). + * + * @return strlen(src) + MIN(dsize, strlen(initial dst)); + * if retval >= dsize, truncation occurred. + */ +size_t Strlcat(char* dst, const char* src, size_t dsize) noexcept; + +/** + * Copy string `src` to buffer `dst` of size `dsize`. At most `dsize-1` + * chars will be copied. Always NUL terminates (unless `dsize == 0`). + * + * @return strlen(src); if retval >= dsize, truncation occurred. + */ +size_t Strlcpy(char* dst, const char* src, size_t dsize) noexcept; + +void CopyAll(char*) noexcept; + +template +void CopyAll(char* p, const std::string_view s, const R&... r) { + std::string::traits_type::copy(p, s.data(), s.size()); + CopyAll(p + s.size(), r...); +} + +template +std::string Join(const R&... r) { + std::string s(SumLength(r...), '\0'); + + CopyAll(s.data(), r...); + + return s; +} + +template +std::basic_string FromAscii(const std::string_view& s); + +} // namespace NUtils diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/misc/misc.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/misc/misc.h new file mode 100644 index 000000000000..20beb71f72c0 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/library/string_utils/misc/misc.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace NUtils { + +template +bool ContainerTransform(TContainer& str, T&& f, size_t pos = 0, size_t n = TContainer::npos) { + size_t len = str.size(); + if (pos > len) { + pos = len; + } + if (n > len - pos) { + n = len - pos; + } + + bool changed = false; + + for (size_t i = pos; i != pos + n; ++i) { + auto c = f(i, str[i]); + if (c != str[i]) { + changed = true; + str[i] = c; + } + } + return changed; +} + +template +void GetNext(std::string_view& s, D delim, P& param) { + std::string_view next; + Y_ENSURE(NUtils::NextTok(s, next, delim), "Split: number of fields less than number of Split output arguments"); + param = FromString

(next); +} + +template +void GetNext(std::string_view& s, D delim, std::optional

& param) { + std::string_view next; + if (NUtils::NextTok(s, next, delim)) { + param = FromString

(next); + } else { + param.reset(); + } +} + +template +void GetNext(std::string_view& s, D delim, std::optional& param) { + std::string_view next; + if (NUtils::NextTok(s, next, delim)) { + param = next; + } else { + param.reset(); + } +} + +} \ No newline at end of file diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/stlfwd.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/stlfwd.h new file mode 100644 index 000000000000..77b97233f956 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/stlfwd.h @@ -0,0 +1,15 @@ +#pragma once + +// STL "forwarding" for the poor + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h new file mode 100644 index 000000000000..107c259a39de --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h @@ -0,0 +1,17 @@ +#pragma once + +#ifdef YDB_SDK_USE_STD_STRING + +#include +namespace NYdb { +using TStringType = std::string; +} + +#else + +#include +namespace NYdb { +using TStringType = TString; +} + +#endif diff --git a/ydb/public/sdk/cpp/sdk_common.inc b/ydb/public/sdk/cpp/sdk_common.inc new file mode 100644 index 000000000000..d7f76f95c7ba --- /dev/null +++ b/ydb/public/sdk/cpp/sdk_common.inc @@ -0,0 +1,7 @@ +ADDINCL( + GLOBAL ydb/public/sdk/cpp/include +) + +ADDINCL( + ydb/public/sdk/cpp +) diff --git a/ydb/public/sdk/cpp/src/CMakeLists.txt b/ydb/public/sdk/cpp/src/CMakeLists.txt new file mode 100644 index 000000000000..3dff70940585 --- /dev/null +++ b/ydb/public/sdk/cpp/src/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(api) +add_subdirectory(client) +add_subdirectory(library) \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/CMakeLists.txt new file mode 100644 index 000000000000..65426eeaccde --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/CMakeLists.txt @@ -0,0 +1,32 @@ +add_subdirectory(bsconfig) +add_subdirectory(draft) +add_subdirectory(helpers) +add_subdirectory(iam) +add_subdirectory(iam_private) +add_subdirectory(impl) +add_subdirectory(resources) +add_subdirectory(common_client) +add_subdirectory(coordination) +add_subdirectory(datastreams) +add_subdirectory(debug) +add_subdirectory(discovery) +add_subdirectory(driver) +add_subdirectory(export) +add_subdirectory(extension_common) +add_subdirectory(extensions) +add_subdirectory(federated_topic) +add_subdirectory(import) +add_subdirectory(monitoring) +add_subdirectory(operation) +add_subdirectory(params) +add_subdirectory(persqueue_public) +add_subdirectory(proto) +add_subdirectory(query) +add_subdirectory(rate_limiter) +add_subdirectory(result) +add_subdirectory(scheme) +add_subdirectory(ss_tasks) +add_subdirectory(table) +add_subdirectory(topic) +add_subdirectory(types) +add_subdirectory(value) diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/bsconfig/CMakeLists.txt new file mode 100644 index 000000000000..961a3327a601 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/bsconfig/CMakeLists.txt @@ -0,0 +1,20 @@ +_ydb_sdk_add_library(client-bsconfig) + +target_link_libraries(client-bsconfig + PUBLIC + client-ydb_driver + client-ydb_common_client + client-ydb_types-status + client-ydb_types + PRIVATE + api-grpc + impl-ydb_internal-make_request + client-ydb_common_client-impl +) + +target_sources(client-bsconfig + PRIVATE + storage_config.cpp +) + +_ydb_sdk_make_client_component(BSConfig client-bsconfig) diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp new file mode 100644 index 000000000000..4fda3b385697 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include + +#include + +namespace NYdb::inline V3::NStorageConfig { + +class TStorageConfigClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncStatus ReplaceStorageConfig(const std::string& config) { + auto request = MakeRequest(); + request.set_yaml_config(config); + + return RunSimple( + std::move(request), + &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceStorageConfig); + } + + TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + NYdb::TStringType config; + if (Ydb::BSConfig::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { + config = result.yaml_config(); + } + + TFetchStorageConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchStorageConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + return promise.GetFuture(); + } + + TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID) { + auto request = MakeRequest(); + request.set_self_assembly_uuid(selfAssemblyUUID); + return RunSimple(std::move(request), + &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncBootstrapCluster); + } +}; + +TStorageConfigClient::TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TStorageConfigClient::TImpl(CreateInternalInterface(driver), settings)) +{} + +TStorageConfigClient::~TStorageConfigClient() = default; + +TAsyncStatus TStorageConfigClient::ReplaceStorageConfig(const std::string& config) { + return Impl_->ReplaceStorageConfig(config); +} + +TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(const TStorageConfigSettings& settings) { + return Impl_->FetchStorageConfig(settings); +} + +TAsyncStatus TStorageConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID) { + return Impl_->BootstrapCluster(selfAssemblyUUID); +} + + +} diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/ya.make b/ydb/public/sdk/cpp/src/client/bsconfig/ya.make new file mode 100644 index 000000000000..c329c69b0e18 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/bsconfig/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + storage_config.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/types/operation + ydb/public/sdk/cpp/src/client/value +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/common_client/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/common_client/CMakeLists.txt new file mode 100644 index 000000000000..468b6dc61fa3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/CMakeLists.txt @@ -0,0 +1,15 @@ +add_subdirectory(impl) + +_ydb_sdk_add_library(client-ydb_common_client) + +target_link_libraries(client-ydb_common_client PUBLIC + yutil + impl-ydb_internal-common + client-ydb_types-credentials +) + +target_sources(client-ydb_common_client PRIVATE + settings.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_common_client) diff --git a/ydb/public/sdk/cpp/src/client/common_client/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/common_client/impl/CMakeLists.txt new file mode 100644 index 000000000000..63889462767f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/impl/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(client-ydb_common_client-impl) + +target_link_libraries(client-ydb_common_client-impl PUBLIC + yutil + impl-ydb_internal-grpc_connections +) + +target_sources(client-ydb_common_client-impl PRIVATE + client.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_common_client-impl) diff --git a/ydb/public/sdk/cpp/src/client/common_client/impl/client.cpp b/ydb/public/sdk/cpp/src/client/common_client/impl/client.cpp new file mode 100644 index 000000000000..f679c0d5ec74 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/impl/client.cpp @@ -0,0 +1 @@ +#include "client.h" diff --git a/ydb/public/sdk/cpp/src/client/common_client/impl/client.h b/ydb/public/sdk/cpp/src/client/common_client/impl/client.h new file mode 100644 index 000000000000..a24667204cc7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/impl/client.h @@ -0,0 +1,133 @@ +#pragma once + +#include "iface.h" + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include + +namespace NYdb::inline V3 { + +template +class TClientImplCommon + : public IClientImplCommon + , public std::enable_shared_from_this { +public: + TClientImplCommon( + std::shared_ptr&& connections, + const std::optional& database, + const std::optional& discoveryEndpoint, + const std::optional& discoveryMode, + const std::optional& sslCredentials, + const std::optional>& credentialsProviderFactory) + : Connections_(std::move(connections)) + , DbDriverState_(Connections_->GetDriverState( + database, discoveryEndpoint, discoveryMode, sslCredentials, credentialsProviderFactory)) + { + Y_ABORT_UNLESS(DbDriverState_); + } + + TClientImplCommon( + std::shared_ptr&& connections, + const TCommonClientSettings& settings) + : Connections_(std::move(connections)) + , DbDriverState_( + Connections_->GetDriverState( + settings.Database_, + settings.DiscoveryEndpoint_, + settings.DiscoveryMode_, + settings.SslCredentials_, + settings.CredentialsProviderFactory_ + ) + ) + { + Y_ABORT_UNLESS(DbDriverState_); + } + + NThreading::TFuture DiscoveryCompleted() const { + return DbDriverState_->DiscoveryCompleted(); + } + + void ScheduleTask(const std::function& fn, TDuration timeout) override { + std::weak_ptr weak = this->shared_from_this(); + auto cbGuard = [weak, fn]() { + auto strongClient = weak.lock(); + if (strongClient) { + fn(); + } + }; + Connections_->ScheduleOneTimeTask(std::move(cbGuard), timeout); + } + +protected: + template + using TAsyncRequest = typename NYdbGrpc::TSimpleRequestProcessor< + typename TService::Stub, + TRequest, + TResponse>::TAsyncRequest; + + template + NThreading::TFuture RunSimple( + TRequest&& request, + TAsyncRequest rpc, + const TRpcRequestSettings& requestSettings = {}) + { + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any*, TPlainStatus status) mutable { + TStatus st(std::move(status)); + promise.SetValue(std::move(st)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + rpc, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + requestSettings); + + return promise.GetFuture(); + } + + template + NThreading::TFuture RunOperation( + TRequest&& request, + TAsyncRequest rpc, + const TRpcRequestSettings& requestSettings = {}) + { + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (Ydb::Operations::Operation* operation, TPlainStatus status) mutable { + TStatus st(std::move(status)); + if (!operation) { + promise.SetValue(TOp(std::move(st))); + } else { + promise.SetValue(TOp(std::move(st), std::move(*operation))); + } + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + rpc, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + requestSettings); + + return promise.GetFuture(); + } + +protected: + std::shared_ptr Connections_; + TDbDriverStatePtr DbDriverState_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/common_client/impl/iface.h b/ydb/public/sdk/cpp/src/client/common_client/impl/iface.h new file mode 100644 index 000000000000..87b6ad6472a6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/impl/iface.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { + +class IClientImplCommon { +public: + virtual ~IClientImplCommon() = default; + virtual void ScheduleTask(const std::function& fn, TDuration timeout) = 0; +}; + +} diff --git a/ydb/public/sdk/cpp/src/client/common_client/impl/ya.make b/ydb/public/sdk/cpp/src/client/common_client/impl/ya.make new file mode 100644 index 000000000000..9ccb37ee0d8d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/impl/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + client.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/common_client/settings.cpp b/ydb/public/sdk/cpp/src/client/common_client/settings.cpp new file mode 100644 index 000000000000..d8928a566f5f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/settings.cpp @@ -0,0 +1,20 @@ +#include + +#include + +namespace NYdb::inline V3 { + +TCommonClientSettings& TCommonClientSettings::AuthToken(const std::optional& token) { + return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.value())); +} + +TCommonClientSettings GetClientSettingsFromConnectionString(const std::string& connectionString) { + TCommonClientSettings settings; + auto connectionInfo = ParseConnectionString(connectionString); + settings.Database(connectionInfo.Database); + settings.DiscoveryEndpoint(connectionInfo.Endpoint); + settings.SslCredentials(TSslCredentials(connectionInfo.EnableSsl)); + return settings; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/common_client/ya.make b/ydb/public/sdk/cpp/src/client/common_client/ya.make new file mode 100644 index 000000000000..f4e1fa007f56 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/common_client/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + settings.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/common + ydb/public/sdk/cpp/src/client/types/credentials +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/coordination/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/coordination/CMakeLists.txt new file mode 100644 index 000000000000..94277bae8bca --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/coordination/CMakeLists.txt @@ -0,0 +1,27 @@ +_ydb_sdk_add_library(client-ydb_coordination) + +target_link_libraries(client-ydb_coordination PUBLIC + yutil + enum_serialization_runtime + api-grpc + impl-ydb_internal-make_request + client-ydb_common_client + client-ydb_common_client-impl + client-ydb_driver + client-ydb_proto + client-ydb_types + client-ydb_types-status +) + +target_sources(client-ydb_coordination PRIVATE + coordination.cpp + proto_accessor.cpp +) + +generate_enum_serilization(client-ydb_coordination + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/coordination/coordination.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/coordination/coordination.h +) + +_ydb_sdk_make_client_component(Coordination client-ydb_coordination) diff --git a/ydb/public/sdk/cpp/src/client/coordination/coordination.cpp b/ydb/public/sdk/cpp/src/client/coordination/coordination.cpp new file mode 100644 index 000000000000..54a37d1988bd --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/coordination/coordination.cpp @@ -0,0 +1,2096 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include +#include + +namespace NYdb::inline V3 { +namespace NCoordination { + +using NThreading::TFuture; +using NThreading::TPromise; +using NThreading::NewPromise; +using NYdbGrpc::TQueueClientFixedEvent; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template +using TResultPromise = TPromise>; + +template +inline TResultPromise NewResultPromise() { + return NewPromise>(); + + +} + +//////////////////////////////////////////////////////////////////////////////// + +template +void ConvertSettingsToProtoConfig( + const Settings& settings, + Ydb::Coordination::Config* config) +{ + if (settings.SelfCheckPeriod_) { + config->set_self_check_period_millis(settings.SelfCheckPeriod_->MilliSeconds()); + } + if (settings.SessionGracePeriod_) { + config->set_session_grace_period_millis(settings.SessionGracePeriod_->MilliSeconds()); + } + if (settings.ReadConsistencyMode_ != EConsistencyMode::UNSET) { + config->set_read_consistency_mode(static_cast(settings.ReadConsistencyMode_)); + } + if (settings.AttachConsistencyMode_ != EConsistencyMode::UNSET) { + config->set_attach_consistency_mode(static_cast(settings.AttachConsistencyMode_)); + } + if (settings.RateLimiterCountersMode_ != ERateLimiterCountersMode::UNSET) { + config->set_rate_limiter_counters_mode(static_cast(settings.RateLimiterCountersMode_)); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +std::string GenerateProtectionKey(size_t size) { + std::string key; + if (size > 0) { + key.resize(size); + EntropyPool().Read(key.data(), size); + } + return key; +} + +} + +//////////////////////////////////////////////////////////////////////////////// + +struct TNodeDescription::TImpl { + TImpl(const Ydb::Coordination::DescribeNodeResult& desc) { + auto& config = desc.config(); + if (config.self_check_period_millis()) { + SelfCheckPeriod_ = TDuration::MilliSeconds(config.self_check_period_millis()); + } + if (config.session_grace_period_millis()) { + SessionGracePeriod_ = TDuration::MilliSeconds(config.session_grace_period_millis()); + } + ReadConsistencyMode_ = static_cast(config.read_consistency_mode()); + AttachConsistencyMode_ = static_cast(config.attach_consistency_mode()); + RateLimiterCountersMode_ = static_cast(config.rate_limiter_counters_mode()); + Owner_ = desc.self().owner(); + PermissionToSchemeEntry(desc.self().effective_permissions(), &EffectivePermissions_); + Proto_ = desc; + } + + std::optional SelfCheckPeriod_; + std::optional SessionGracePeriod_; + EConsistencyMode ReadConsistencyMode_; + EConsistencyMode AttachConsistencyMode_; + ERateLimiterCountersMode RateLimiterCountersMode_; + std::string Owner_; + std::vector EffectivePermissions_; + Ydb::Coordination::DescribeNodeResult Proto_; +}; + +TNodeDescription::TNodeDescription( + const Ydb::Coordination::DescribeNodeResult& desc) + : Impl_(new TImpl(desc)) +{ } + +const std::optional& TNodeDescription::GetSelfCheckPeriod() const { + return Impl_->SelfCheckPeriod_; +} + +const std::optional& TNodeDescription::GetSessionGracePeriod() const { + return Impl_->SessionGracePeriod_; +} + +EConsistencyMode TNodeDescription::GetReadConsistencyMode() const { + return Impl_->ReadConsistencyMode_; +} + +EConsistencyMode TNodeDescription::GetAttachConsistencyMode() const { + return Impl_->AttachConsistencyMode_; +} + +ERateLimiterCountersMode TNodeDescription::GetRateLimiterCountersMode() const { + return Impl_->RateLimiterCountersMode_; +} + +const std::string& TNodeDescription::GetOwner() const { + return Impl_->Owner_; +} + +const std::vector& TNodeDescription::GetEffectivePermissions() const { + return Impl_->EffectivePermissions_; +} + +const Ydb::Coordination::DescribeNodeResult& TNodeDescription::GetProto() const { + return Impl_->Proto_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSemaphoreSession::TSemaphoreSession() { + OrderId_ = 0; + SessionId_ = 0; + Count_ = 0; +} + +TSemaphoreSession::TSemaphoreSession( + const Ydb::Coordination::SemaphoreSession& desc) +{ + OrderId_ = desc.order_id(); + SessionId_ = desc.session_id(); + Timeout_ = desc.timeout_millis() == Max() ? TDuration::Max() : TDuration::MilliSeconds(desc.timeout_millis()); + Count_ = desc.count(); + Data_ = desc.data(); +} + +TSemaphoreDescription::TSemaphoreDescription() { + Count_ = 0; + Limit_ = 0; + IsEphemeral_ = false; +} + +TSemaphoreDescription::TSemaphoreDescription( + const Ydb::Coordination::SemaphoreDescription& desc) +{ + Name_ = desc.name(); + Data_ = desc.data(); + Count_ = desc.count(); + Limit_ = desc.limit(); + Owners_.reserve(desc.owners_size()); + for (const auto& owner : desc.owners()) { + Owners_.emplace_back(owner); + } + Waiters_.reserve(desc.waiters_size()); + for (const auto& waiter : desc.waiters()) { + Waiters_.emplace_back(waiter); + } + IsEphemeral_ = desc.ephemeral(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TSessionContext : public TThrRefBase { + using TPtr = TIntrusivePtr; + using TService = Ydb::Coordination::V1::CoordinationService; + using TRequest = Ydb::Coordination::SessionRequest; + using TResponse = Ydb::Coordination::SessionResponse; + using TGrpcStatus = NYdbGrpc::TGrpcStatus; + using IProcessor = NYdbGrpc::IStreamRequestReadWriteProcessor; + + friend class TSession::TImpl; + +public: + TSessionContext( + TGRpcConnectionsImpl* connections, + TDbDriverStatePtr dbState, + const std::string& path, + const TSessionSettings& settings) + : Connections_(connections) + , DbDriverState_(dbState) + , Path_(path) + , Settings_(settings) + , ProtectionKey_(GenerateProtectionKey(8)) + , SessionPromise(NewPromise()) + , SessionFuture(SessionPromise.GetFuture()) + { } + + TAsyncSessionResult TakeStartResult() { + TAsyncSessionResult result; + result.Swap(SessionFuture); + Y_ABORT_UNLESS(result.Initialized(), "TakeStartResult cannot be called more than once"); + return result; + } + + void Start(IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + auto status = MakeStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"); + auto promise = std::move(SessionPromise); + promise.SetValue(TSessionResult(std::move(status))); + return; + } + { + std::lock_guard guard(Lock); + LocalContext = context; + ConnectionState = EConnectionState::CONNECTING; + } + DoConnectRequest(context); + context->SubscribeStop([self = TPtr(this)] { + self->Stop(); + }); + } + + void Stop() { + std::lock_guard guard(Lock); + IsStopping = true; + if (!CurrentFailure) { + SetCurrentFailure( + TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped")); + } + if (Processor) { + Processor->Cancel(); + } + if (SleepContext) { + SleepContext->Cancel(); + } + } + +private: + enum ESemaphoreOpType { + SEM_OP_ACQUIRE, + SEM_OP_RELEASE, + }; + + struct TSemaphoreOp : public TSimpleRefCount { + TInstant SendTimestamp; + uint64_t ReqId = 0; + TResultPromise Promise = NewResultPromise(); + const ESemaphoreOpType OpType; + + TSemaphoreOp(ESemaphoreOpType opType) + : OpType(opType) + {} + + virtual ~TSemaphoreOp() = default; + virtual void FillRequest(TRequest& req, const std::string& name) const = 0; + }; + + struct TSemaphoreAcquireOp : public TSemaphoreOp { + TAcquireSemaphoreSettings Settings; + TInstant Deadline; + + explicit TSemaphoreAcquireOp(const TAcquireSemaphoreSettings& settings) + : TSemaphoreOp(SEM_OP_ACQUIRE) + , Settings(settings) + , Deadline(Settings.Timeout_.ToDeadLine()) + { } + + uint64_t GetTimeoutMillisLeft() const { + if (Deadline == TInstant::Max()) { + return Max(); + } + return (Deadline - TInstant::Now()).MilliSeconds(); + } + + void FillRequest(TRequest& req, const std::string& name) const override { + auto& inner = *req.mutable_acquire_semaphore(); + inner.set_req_id(ReqId); + inner.set_name(TStringType{name}); + inner.set_count(Settings.Count_); + inner.set_timeout_millis(GetTimeoutMillisLeft()); + inner.set_data(TStringType{Settings.Data_}); + inner.set_ephemeral(Settings.Ephemeral_); + } + }; + + struct TSemaphoreReleaseOp : public TSemaphoreOp { + TSemaphoreReleaseOp() + : TSemaphoreOp(SEM_OP_RELEASE) + {} + + void FillRequest(TRequest& req, const std::string& name) const override { + auto& inner = *req.mutable_release_semaphore(); + inner.set_req_id(ReqId); + inner.set_name(TStringType{name}); + } + }; + + struct TSemaphoreState { + std::string Name; + TIntrusivePtr LastSentOp; + TIntrusivePtr LastAckedOp; + std::unordered_map> WaitingOps; + std::deque> OpQueue; + bool Restoring = false; + + bool IsEmpty() const { + return !LastSentOp + && !LastAckedOp + && WaitingOps.empty() + && OpQueue.empty(); + } + }; + + struct TSimpleOp { + TInstant SendTimestamp; + + virtual ~TSimpleOp() = default; + virtual void FillRequest(TRequest& req, uint64_t reqId) const = 0; + virtual void SetFailure(const TStatus& status) = 0; + }; + + struct TPingOp : public TSimpleOp { + TResultPromise Promise = NewResultPromise(); + + TPingOp() = default; + + void FillRequest(TRequest& req, uint64_t reqId) const override { + auto& inner = *req.mutable_ping(); + inner.set_opaque(reqId); + } + + void SetFailure(const TStatus& status) override { + Promise.SetValue(TResult(status)); + } + }; + + struct TDescribeSemaphoreOp : public TSimpleOp { + const std::string Name; + TDescribeSemaphoreSettings Settings; + TPromise Promise = NewPromise(); + + TDescribeSemaphoreOp(const std::string& name, const TDescribeSemaphoreSettings& settings) + : Name(name) + , Settings(settings) + {} + + void FillRequest(TRequest& req, uint64_t reqId) const override { + auto& inner = *req.mutable_describe_semaphore(); + inner.set_req_id(reqId); + inner.set_name(TStringType{Name}); + inner.set_include_owners(Settings.IncludeOwners_); + inner.set_include_waiters(Settings.IncludeWaiters_); + inner.set_watch_data(Settings.WatchData_); + inner.set_watch_owners(Settings.WatchOwners_); + } + + void SetFailure(const TStatus& status) override { + if (Promise.Initialized()) { + Promise.SetValue(TDescribeSemaphoreResult(status)); + } else if (Settings.OnChanged_) { + std::function callback; + callback.swap(Settings.OnChanged_); + callback(false); + } + } + }; + + struct TCreateSemaphoreOp : public TSimpleOp { + const std::string Name; + const uint64_t Limit; + const std::string Data; + TResultPromise Promise = NewResultPromise(); + + TCreateSemaphoreOp(const std::string& name, uint64_t limit, const std::string& data) + : Name(name) + , Limit(limit) + , Data(data) + { } + + void FillRequest(TRequest& req, uint64_t reqId) const override { + auto& inner = *req.mutable_create_semaphore(); + inner.set_req_id(reqId); + inner.set_name(TStringType{Name}); + inner.set_limit(Limit); + inner.set_data(TStringType{Data}); + } + + void SetFailure(const TStatus& status) override { + Promise.SetValue(TResult(status)); + } + }; + + struct TUpdateSemaphoreOp : public TSimpleOp { + const std::string Name; + const std::string Data; + TResultPromise Promise = NewResultPromise(); + + TUpdateSemaphoreOp(const std::string& name, const std::string& data) + : Name(name) + , Data(data) + {} + + void FillRequest(TRequest& req, uint64_t reqId) const override { + auto& inner = *req.mutable_update_semaphore(); + inner.set_req_id(reqId); + inner.set_name(TStringType{Name}); + inner.set_data(TStringType{Data}); + } + + void SetFailure(const TStatus& status) override { + Promise.SetValue(TResult(status)); + } + }; + + struct TDeleteSemaphoreOp : public TSimpleOp { + const std::string Name; + const bool Force; + TResultPromise Promise = NewResultPromise(); + + TDeleteSemaphoreOp(const std::string& name, bool force) + : Name(name) + , Force(force) + {} + + void FillRequest(TRequest& req, uint64_t reqId) const override { + auto& inner = *req.mutable_delete_semaphore(); + inner.set_req_id(reqId); + inner.set_name(TStringType{Name}); + inner.set_force(Force); + } + + void SetFailure(const TStatus& status) override { + Promise.SetValue(TResult(status)); + } + }; + +private: + ESessionState DoGetSessionState() { + std::lock_guard guard(Lock); + return SessionState; + } + + EConnectionState DoGetConnectionState() { + std::lock_guard guard(Lock); + return ConnectionState; + } + + TAsyncResult DoClose(bool isAbort) { + std::lock_guard guard(Lock); + if (!ClosedPromise.Initialized()) { + ClosedPromise = NewResultPromise(); + if (IsWriteAllowed()) { + SendCloseRequest(); + } + } + if (isAbort) { + // Move to the stopping state, but without cancelling the context + IsStopping = true; + if (!CurrentFailure) { + SetCurrentFailure( + TPlainStatus(EStatus::UNAVAILABLE, "Session is aborted")); + } + if (SleepContext) { + SleepContext->Cancel(); + } + } + return ClosedPromise; + } + + TAsyncResult DoPing() { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = std::make_unique(); + auto future = op->Promise.GetFuture(); + if (IsWriteAllowed()) { + DoSendSimpleOp(std::move(op)); + } else { + PendingRequests.emplace_back(std::move(op)); + } + return future; + } + + TAsyncResult DoReconnect() { + std::lock_guard guard(Lock); + SessionReconnectDelay = TDuration::Zero(); + if (Processor) { + Processor->Cancel(); + } + if (SleepContext) { + SleepContext->Cancel(); + } + if (IsClosed()) { + return MakeClosedResult(); + } + if (!ReconnectPromise.Initialized()) { + ReconnectPromise = NewResultPromise(); + } + return ReconnectPromise; + } + + TAsyncResult DoAcquireSemaphore( + const std::string& name, + const TAcquireSemaphoreSettings& settings) + { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = MakeIntrusive(settings); + DoSemaphoreEnqueueOp(name, op); + return op->Promise; + } + + TAsyncResult DoReleaseSemaphore(const std::string& name) { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = MakeIntrusive(); + DoSemaphoreEnqueueOp(name, op); + return op->Promise; + } + + TAsyncDescribeSemaphoreResult DoDescribeSemaphore( + const std::string& name, + const TDescribeSemaphoreSettings& settings) + { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = std::make_unique(name, settings); + auto future = op->Promise.GetFuture(); + if (IsWriteAllowed()) { + DoSendSimpleOp(std::move(op)); + } else { + PendingRequests.emplace_back(std::move(op)); + } + return future; + } + + TAsyncResult DoCreateSemaphore(const std::string& name, uint64_t limit, const std::string& data) { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = std::make_unique(name, limit, data); + auto future = op->Promise.GetFuture(); + if (IsWriteAllowed()) { + DoSendSimpleOp(std::move(op)); + } else { + PendingRequests.emplace_back(std::move(op)); + } + return future; + } + + TAsyncResult DoUpdateSemaphore(const std::string& name, const std::string& data) { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = std::make_unique(name, data); + auto future = op->Promise.GetFuture(); + if (IsWriteAllowed()) { + DoSendSimpleOp(std::move(op)); + } else { + PendingRequests.emplace_back(std::move(op)); + } + return future; + } + + TAsyncResult DoDeleteSemaphore(const std::string& name, bool force) { + std::lock_guard guard(Lock); + if (IsClosed()) { + return MakeClosedResult(); + } + auto op = std::make_unique(name, force); + auto future = op->Promise.GetFuture(); + if (IsWriteAllowed()) { + DoSendSimpleOp(std::move(op)); + } else { + PendingRequests.emplace_back(std::move(op)); + } + return future; + } + + TDuration GetConnectTimeout() { + // Use a separate connect timeout if available + if (Settings_.ConnectTimeout_) { + return Settings_.ConnectTimeout_; + } + + // Use session timeout otherwise + return Settings_.Timeout_; + } + + void DoConnectRequest(std::shared_ptr context) { + auto connectContext = context->CreateContext(); + + auto connectTimeout = GetConnectTimeout(); + auto connectTimeoutContext = connectTimeout != TDuration::Max() + ? connectContext->CreateContext() + : nullptr; + + IQueueClientContextPtr prevConnectContext; + IQueueClientContextPtr prevConnectTimeoutContext; + + { + std::lock_guard guard(Lock); + prevConnectContext = std::exchange(ConnectContext, connectContext); + prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); + } + + // Cancel previous connection contexts (if they still exist) + if (prevConnectTimeoutContext) { + prevConnectTimeoutContext->Cancel(); + } + if (prevConnectContext) { + prevConnectContext->Cancel(); + } + + Connections_->StartBidirectionalStream( + [self = TPtr(this)] (auto status, auto processor) { + self->OnConnect(std::move(status), std::move(processor)); + }, + &TService::Stub::AsyncSession, + DbDriverState_, + TRpcRequestSettings::Make(Settings_), + std::move(connectContext)); + + if (connectTimeoutContext) { + // N.B. to avoid deadlocks we must schedule timeout outside the lock + auto timerHandler = [self = TPtr(this), connectTimeoutContext] (bool ok) mutable { + if (ok) { + self->OnConnectTimeout(std::move(connectTimeoutContext)); + } + }; + Connections_->ScheduleCallback( + connectTimeout, + std::move(timerHandler), + std::move(connectTimeoutContext)); + } + } + +private: + TPlainStatus MakePlainStatus( + Ydb::StatusIds::StatusCode protoStatus, + const google::protobuf::RepeatedPtrField& protoIssues) const + { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(protoIssues, issues); + return TPlainStatus(static_cast(protoStatus), std::move(issues)); + } + + TStatus MakeStatus() const { + return TStatus(TPlainStatus()); + } + + template + TStatus MakeStatus(TSource&& source) const { + return TStatus(std::forward(source)); + } + + TStatus MakeStatus(EStatus status, NYdb::NIssue::TIssues&& issues) const { + return TStatus(TPlainStatus(status, std::move(issues))); + } + + TStatus MakeStatus(EStatus status, const std::string& message) const { + return TStatus(TPlainStatus(status, message)); + } + +private: + bool IsClosed() const { + return IsStopping || ClosedPromise.Initialized(); + } + + bool IsWriteAllowed() const { + return Processor && + SessionState == ESessionState::ATTACHED && + ConnectionState == EConnectionState::CONNECTED; + } + + template + void SetCurrentFailure(TSource&& source) { + Y_ABORT_UNLESS(!CurrentFailure); + CurrentFailure = std::make_unique(std::forward(source)); + } + + template + TAsyncResult MakeClosedResult() const { + auto promise = NewResultPromise(); + if (CurrentFailure) { + promise.SetValue(TResult(*CurrentFailure)); + } else { + auto status = MakeStatus(EStatus::CLIENT_CANCELLED, "Session is closed"); + promise.SetValue(TResult(std::move(status))); + } + return promise; + } + + void SendCloseRequest() { + Y_ABORT_UNLESS(Processor); + Y_ABORT_UNLESS(ClosedPromise.Initialized()); + TRequest req; + auto* stop = req.mutable_session_stop(); + Y_UNUSED(stop); + Processor->Write(std::move(req)); + } + + void SessionAttached() { + Y_ABORT_UNLESS(SessionState != ESessionState::ATTACHED); + Y_ABORT_UNLESS(ConnectionState == EConnectionState::ATTACHING); + SessionState = ESessionState::ATTACHED; + ConnectionState = EConnectionState::CONNECTED; + SessionExpireDeadline.reset(); + SessionReconnectDelay = TDuration::Zero(); + if (ClosedPromise.Initialized()) { + // Send the delayed close request, will fail if cancelled + SendCloseRequest(); + } + if (IsStopping) { + // Either cancelled (via shutdown) or aborted (implicit close) + return; + } + for (auto& kv : Semaphores) { + TSemaphoreState* state = &kv.second; + if (state->LastSentOp) { + // Repeat the last sent op + DoSemaphoreSendOp(state, state->LastSentOp); + state->Restoring = true; + } else if (state->LastAckedOp) { + // Repeat the last acknowledged op + DoSemaphoreSendOp(state, state->LastAckedOp); + state->Restoring = true; + } else { + state->Restoring = false; + DoSemaphoreProcessQueue(state); + } + } + for (auto& op : PendingRequests) { + DoSendSimpleOp(std::move(op)); + } + PendingRequests.clear(); + } + + void DoSemaphoreSendOp(TSemaphoreState* state, TIntrusivePtr op) { + Y_ABORT_UNLESS(IsWriteAllowed()); + Y_ABORT_UNLESS(op->ReqId > 0); + Y_ABORT_UNLESS(!state->WaitingOps.contains(op->ReqId)); + SemaphoreByReqId[op->ReqId] = state; + state->WaitingOps[op->ReqId] = op; + state->LastSentOp = op; + TRequest req; + op->FillRequest(req, state->Name); + op->SendTimestamp = TInstant::Now(); + Processor->Write(std::move(req)); + } + + void DoSemaphoreProcessQueue(TSemaphoreState* state) { + if (!state->OpQueue.empty() && !state->LastSentOp && IsWriteAllowed()) { + auto op = std::move(state->OpQueue.front()); + state->OpQueue.pop_front(); + op->ReqId = NextReqId++; + DoSemaphoreSendOp(state, std::move(op)); + } + } + + bool DoSemaphoreProcessResult( + uint64_t reqId, ESemaphoreOpType opType, EStatus status, + TResultPromise* supersededPromise, + TResultPromise* resultPromise) + { + auto statePtr = MapFindPtr(SemaphoreByReqId, reqId); + TSessionContext::TSemaphoreState* state = nullptr; + if (statePtr) { + state = *statePtr; + } + if (!state) { + return false; + } + auto op = std::move(state->WaitingOps[reqId]); + Y_ABORT_UNLESS(op, "SemaphoreByReqId and WaitingOps desync"); + state->WaitingOps.erase(reqId); + SemaphoreByReqId.erase(reqId); + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + + if (op->OpType != opType) { + return false; + } + + bool isError = status != EStatus::SUCCESS; + + if (state->LastSentOp == op) { + // Result received without getting a wait ack first + state->LastSentOp = {}; + if (state->LastAckedOp == op || !isError) { + // This was a post-reconnect retry, or the op succeeded + if (state->Restoring && state->LastAckedOp && state->LastAckedOp != op) { + // The last acknowledged op has never been sent, remember its promise + supersededPromise->Swap(state->LastAckedOp->Promise); + } + state->LastAckedOp = {}; + } + if (state->Restoring && state->LastAckedOp) { + // Unconfirmed op failed, retry the last confirmed op + if (IsWriteAllowed()) { + DoSemaphoreSendOp(state, state->LastAckedOp); + } + } else { + // Try sending whatever else is ready in the queue + DoSemaphoreProcessQueue(state); + } + } else if (state->LastAckedOp == op) { + // Result received after a wait ack + state->LastAckedOp = {}; + } + + state->Restoring = false; + resultPromise->Swap(op->Promise); + + if (state->IsEmpty()) { + // Forget useless semaphore entries + std::string name = state->Name; + Semaphores.erase(name); + } + + return true; + } + + void DoSemaphoreEnqueueOp(const std::string& name, TIntrusivePtr op) { + TSemaphoreState* state = MapFindPtr(Semaphores, name); + if (!state) { + state = &Semaphores[name]; + state->Name = name; + } + state->OpQueue.emplace_back(op); + DoSemaphoreProcessQueue(state); + } + + uint64_t DoSendSimpleOp(std::unique_ptr&& op) { + Y_ABORT_UNLESS(IsWriteAllowed()); + uint64_t reqId = NextReqId++; + TRequest req; + op->FillRequest(req, reqId); + op->SendTimestamp = TInstant::Now(); + Processor->Write(std::move(req)); + SentRequests[reqId] = std::move(op); + return reqId; + } + + template + TOperation* FindSentRequest(uint64_t reqId) const { + auto it = SentRequests.find(reqId); + if (it == SentRequests.end()) { + return nullptr; + } + return dynamic_cast(it->second.get()); + } + +private: + void OnProcessorStatus(TStatus status) { + std::shared_ptr context; + TDbDriverStatePtr dbDriverState; + + bool stopped; + bool expired = false; + bool notifyExpired = false; + TPromise sessionPromise; + TResultPromise reconnectPromise; + std::deque> abortedSemaphoreOps; + std::deque> failedSemaphoreOps; + std::deque> failedSimpleOps; + TResultPromise closePromise; + + { + std::lock_guard guard(Lock); + Processor.Reset(); + + Y_ABORT_UNLESS(LocalContext, "Processing event without a valid context"); + + // We must have detached by the time status is processed + Y_ABORT_UNLESS(SessionState != ESessionState::ATTACHED); + + stopped = IsStopping; + + if (SessionPromise.Initialized()) { + // Session was never established, no retries + sessionPromise.Swap(SessionPromise); + stopped = true; + } + + if (!stopped) { + if (SessionTimeout) { + // No retries after timeout + stopped = true; + } else if (!SessionExpireDeadline) { + // We will only keep retrying until this deadline expires + auto deadline = SessionLastKnownGoodTimestamp + + Settings_.Timeout_ * Settings_.ReconnectSessionTimeoutMultiplier_; + SessionExpireDeadline = deadline; + } else if (*SessionExpireDeadline <= TInstant::Now()) { + // Stop retrying right now + SessionTimeout = true; + stopped = true; + } + } + + if (stopped) { + context = std::move(LocalContext); + dbDriverState = std::move(DbDriverState_); + expired = SessionState == ESessionState::EXPIRED; + IsStopping = true; + ConnectionState = EConnectionState::STOPPED; + if (!ClosedPromise.Initialized()) { + ClosedPromise = NewResultPromise(); + } + closePromise = ClosedPromise; + if (ReconnectPromise.Initialized()) { + ReconnectPromise.Swap(reconnectPromise); + } + if (SessionId != 0 && SessionTimeout && !expired) { + // N.B. looking at common usage patterns users expect that + // session eventually reaches the expired state, and write + // buggy code that wouldn't work otherwise. Even though we + // don't really know if session has expired or not, we move + // to expired state as a cortesy to users. + SessionState = ESessionState::EXPIRED; + notifyExpired = true; + } + } else { + context = LocalContext; + ConnectionState = EConnectionState::DISCONNECTED; + } + + // Replace status with a previously known failure + if (CurrentFailure) { + if (stopped) { + status = *CurrentFailure; + } else { + status = std::move(*CurrentFailure); + CurrentFailure.reset(); + } + } + + // Handle all waiting ops + for (auto& kv : Semaphores) { + TSemaphoreState* state = &kv.second; + for (auto& wkv : state->WaitingOps) { + auto op = wkv.second; + if (stopped) { + if (state->LastSentOp == op) { + state->LastSentOp = {}; + } + if (state->LastAckedOp == op) { + state->LastAckedOp = {}; + } + failedSemaphoreOps.emplace_back(std::move(op->Promise)); + } else if (op != state->LastSentOp && op != state->LastAckedOp) { + // This operation was acknowledged, but will never be retried + // We already know it was superseded, we just haven't got a reply yet + abortedSemaphoreOps.emplace_back(std::move(op->Promise)); + } + } + state->WaitingOps.clear(); + if (stopped) { + if (state->LastAckedOp && state->LastAckedOp != state->LastSentOp) { + failedSemaphoreOps.emplace_back(std::move(state->LastAckedOp->Promise)); + state->LastAckedOp = {}; + } + if (state->LastSentOp) { + failedSemaphoreOps.emplace_back(std::move(state->LastSentOp->Promise)); + state->LastSentOp = {}; + } + for (auto& op : state->OpQueue) { + failedSemaphoreOps.emplace_back(std::move(op->Promise)); + } + state->OpQueue.clear(); + } + } + SemaphoreByReqId.clear(); + if (stopped) { + Semaphores.clear(); + } + + for (auto& kv : SentRequests) { + failedSimpleOps.emplace_back(std::move(kv.second)); + } + SentRequests.clear(); + + if (stopped) { + for (auto& op : PendingRequests) { + failedSimpleOps.emplace_back(std::move(op)); + } + PendingRequests.clear(); + } + } + + if (status.IsSuccess()) { + status = MakeStatus(EStatus::CLIENT_INTERNAL_ERROR, "Unexpected session request success"); + } + + if (sessionPromise.Initialized()) { + sessionPromise.SetValue(TSessionResult(status)); + } + + if (!abortedSemaphoreOps.empty()) { + auto aborted = MakeStatus(EStatus::ABORTED, + "Operation superseded by another request, true result has been lost"); + for (auto& promise : abortedSemaphoreOps) { + promise.SetValue(TResult(aborted, false)); + } + } + + for (auto& promise : failedSemaphoreOps) { + promise.SetValue(TResult(status, false)); + } + + for (auto& op : failedSimpleOps) { + op->SetFailure(status); + } + + if (closePromise.Initialized()) { + closePromise.SetValue(TResult(expired ? MakeStatus() : status)); + } + + if (stopped) { + if (notifyExpired && Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + } + if (Settings_.OnStopped_) { + Settings_.OnStopped_(); + } + return; + } + + TDuration sleepDuration; + IQueueClientContextPtr sleepContext; + { + std::lock_guard guard(Lock); + if (!IsStopping) { + // Session object still active, calculate sleep duration + sleepDuration = Min(SessionReconnectDelay, *SessionExpireDeadline - TInstant::Now()); + if (SessionReconnectDelay == TDuration::Zero()) { + SessionReconnectDelay = Settings_.ReconnectBackoffDelay_; + } else { + SessionReconnectDelay = Min( + SessionReconnectDelay * Settings_.ReconnectBackoffMultiplier_, + Settings_.Timeout_); + } + } else { + // IsStopping == true means session was stopped by the client + // during all the processing above. We use a zero-sleep duration + // to reschedule this error processing handler. + sleepDuration = TDuration::Zero(); + } + + Y_ABORT_UNLESS(!SleepContext, "Unexpected multiple concurrent sleeps scheduled"); + SleepContext = sleepContext = context->CreateContext(); + } + + auto sleepHandler = [self = TPtr(this)] (bool ok) mutable { + // N.B. OnSleepFinished is always called + self->OnSleepFinished(ok); + }; + Connections_->ScheduleCallback( + sleepDuration, + std::move(sleepHandler), + std::move(sleepContext)); + } + + void OnConnectTimeout(IQueueClientContextPtr context) { + Y_ABORT_UNLESS(context, "Connection timeout context is unexpectedly empty"); + + std::lock_guard guard(Lock); + if (ConnectTimeoutContext != context) { + // Context changed or dropped, ignore + return; + } + + ConnectTimeoutContext.reset(); + + // Timer succeeded and still current, cancel connection attempt + Y_ABORT_UNLESS(ConnectContext, "Connection context is unexpectedly empty"); + ConnectContext->Cancel(); + ConnectContext.reset(); + + SessionTimeout = true; + if (!CurrentFailure) { + SetCurrentFailure( + TPlainStatus(EStatus::TIMEOUT, "Connection request timed out")); + } + } + + void OnConnect(TPlainStatus plain, IProcessor::TPtr processor) { + bool hadTimeout = false; + + { + std::lock_guard guard(Lock); + if (SessionTimeout) { + // OnConnectTimeout was called first + hadTimeout = true; + } + + // Reset context pointers, no longer interested in these + ConnectTimeoutContext.reset(); + ConnectContext.reset(); + } + + if (hadTimeout) { + // Even if connection succeeded it's likely to break soon, as + // the context used for stream creation has been cancelled already + if (processor) { + processor->Cancel(); + } + OnProcessorStatus(MakeStatus(EStatus::TIMEOUT, "Connection request timed out")); + return; + } + + if (!plain.Ok()) { + OnProcessorStatus(MakeStatus(std::move(plain))); + return; + } + + uint64_t seqNo; + uint64_t sessionId; + + TDuration timeout = Settings_.Timeout_; + IQueueClientContextPtr timeoutContext; + + TResultPromise reconnectPromise; + { + std::lock_guard guard(Lock); + Processor = processor; + ConnectionState = EConnectionState::ATTACHING; + if (ReconnectPromise.Initialized()) { + ReconnectPromise.Swap(reconnectPromise); + } + + seqNo = ++SessionSeqNo; + sessionId = SessionId; + + SessionStartTimestamp = TInstant::Now(); + + if (timeout != TDuration::Max()) { + // If this is not the first time we start a session, assume worst + // case that after 2x session timeout there's no point in trying + // to restore it. + if (SessionLastKnownGoodTimestamp) { + auto maxDeadline = SessionLastKnownGoodTimestamp + + Settings_.Timeout_ * Settings_.ReconnectSessionTimeoutMultiplier_; + auto maxTimeout = maxDeadline - SessionStartTimestamp; + if (timeout > maxTimeout) { + timeout = maxTimeout; + } + } + + SessionStartTimeoutContext = timeoutContext = LocalContext->CreateContext(); + } + } + + if (reconnectPromise.Initialized()) { + // Set promise before issuing any requests on the connection + // This way there will be no races between promise and state changes + // However we must be ready for session to become aborted during this callback + reconnectPromise.SetValue(TResult(MakeStatus())); + } + + // Prepare and write the initial request + // It may be ignored if Close is called concurrently + { + TRequest req; + auto* start = req.mutable_session_start(); + start->set_seq_no(seqNo); + start->set_session_id(sessionId); + start->set_path(TStringType{Path_}); + start->set_timeout_millis(Settings_.Timeout_ != TDuration::Max() ? + Settings_.Timeout_.MilliSeconds() : Max()); + start->set_description(TStringType{Settings_.Description_}); + start->set_protection_key(TStringType{ProtectionKey_}); + processor->Write(std::move(req)); + } + + // Start reading responses + Response = std::make_unique(); + processor->Read( + Response.get(), + [self = TPtr(this)] (auto status) { + self->OnRead(std::move(status)); + }); + + if (timeoutContext) { + // N.B. to avoid deadlocks we must schedule timeout outside locks + auto timeoutHandler = [self = TPtr(this), timeoutContext] (bool ok) mutable { + if (ok) { + self->OnSessionStartTimeout(std::move(timeoutContext)); + } + }; + Connections_->ScheduleCallback( + timeout, + std::move(timeoutHandler), + std::move(timeoutContext)); + } + } + + enum class ESessionDetachResult { + Ok, + Timeout, + }; + + ESessionDetachResult HandleSessionDetach() { + ESessionDetachResult result = ESessionDetachResult::Ok; + + IQueueClientContextPtr sessionStartTimeoutContext; + IQueueClientContextPtr sessionSelfPingContext; + + bool detached = false; + { + std::lock_guard guard(Lock); + if (SessionState == ESessionState::ATTACHED) { + SessionState = ESessionState::DETACHED; + detached = true; + } + + sessionStartTimeoutContext = std::exchange(SessionStartTimeoutContext, nullptr); + sessionSelfPingContext = std::exchange(SessionSelfPingContext, nullptr); + + if (SessionTimeout) { + result = ESessionDetachResult::Timeout; + } + } + + if (detached && Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::DETACHED); + } + + if (sessionStartTimeoutContext) { + // Session start timeout no longer relevant + sessionStartTimeoutContext->Cancel(); + } + + if (sessionSelfPingContext) { + // Session pings are no longer relevant + sessionSelfPingContext->Cancel(); + } + + return result; + } + + void OnRead(TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok()) { + Response.reset(); + switch (HandleSessionDetach()) { + case ESessionDetachResult::Ok: + // Report grpc status to client + OnProcessorStatus(MakeStatus(TPlainStatus(std::move(grpcStatus)))); + break; + case ESessionDetachResult::Timeout: + // Original grpc status is irrelevant due to timeout + OnProcessorStatus(MakeStatus(EStatus::TIMEOUT, "Session timed out")); + break; + } + return; + } + + IProcessor::TPtr processor; + { + std::lock_guard guard(Lock); + processor = Processor; + } + Y_ABORT_UNLESS(processor, "Processor is missing for some reason"); + + if (ProcessResponse(processor)) { + // Start reading the next response + Response = std::make_unique(); + processor->Read( + Response.get(), + [self = TPtr(this)] (auto status) { + self->OnRead(std::move(status)); + }); + } else { + // Stop reading responses + Response.reset(); + processor->Finish([self = TPtr(this)] (auto status) { + self->OnFinish(std::move(status)); + }); + } + } + + void OnFinish(TGrpcStatus grpcStatus) { + OnProcessorStatus(MakeStatus(TPlainStatus(std::move(grpcStatus)))); + } + + void OnSleepFinished(bool ok) { + // N.B. even if sleep was cancelled (ok == false), we must still find + // the correct next step to move to. Either reconnect immediately + // without waiting, or finalize the session state. + Y_UNUSED(ok); + + IQueueClientContextPtr sleepContext; + + bool stopping; + std::shared_ptr context; + + { + std::lock_guard guard(Lock); + sleepContext.swap(SleepContext); + Y_ABORT_UNLESS(sleepContext, "Unexpected missing SleepContext in OnSleepFinished"); + if (!(stopping = IsStopping)) { + ConnectionState = EConnectionState::CONNECTING; + context = LocalContext; + Y_ABORT_UNLESS(context); + } + } + + // Call destructor as soon as possible + sleepContext.reset(); + + if (stopping) { + OnProcessorStatus(MakeStatus(EStatus::CLIENT_INTERNAL_ERROR, "Session stopped without an error")); + } else { + DoConnectRequest(std::move(context)); + } + } + +private: + void UpdateLastKnownGoodTimestampLocked(TInstant timestamp) { + SessionLastKnownGoodTimestamp = Max(SessionLastKnownGoodTimestamp, timestamp); + } + + void OnSelfPingTimer(IQueueClientContextPtr context) { + TInstant nextTimerTimestamp; + + { + std::lock_guard guard(Lock); + if (SessionSelfPingContext != context) { + // Context changed or dropped, ignore + return; + } + + if (!IsWriteAllowed()) { + // Not in a pingable state, ignore + SessionSelfPingContext.reset(); + return; + } + + if (auto* op = FindSentRequest(SessionSelfPingReqId)) { + // We have an active request which has not been answered yet + // Cancel everything, it's a timeout :( + SessionSelfPingContext.reset(); + + SessionTimeout = true; + if (!CurrentFailure) { + SetCurrentFailure( + TPlainStatus(EStatus::TIMEOUT, "Session ping request timed out")); + } + + Processor->Cancel(); + + return; + } + + auto now = TInstant::Now(); + auto half = SessionLastKnownGoodTimestamp + Settings_.Timeout_ / 2; + if (now < half) { + // Session has enough activity on its own + // We want to send the next ping 2/3 of the way towards timeout + nextTimerTimestamp = SessionLastKnownGoodTimestamp + Settings_.Timeout_ * (2.0 / 3.0); + } else { + // Send a new ping request right now + SessionSelfPingReqId = DoSendSimpleOp(std::make_unique()); + // We want to wait until either: + // 1. Expected session timeout from the client point of view + // 2. At least a quarter of session timeout from current time + auto expectedSessionDeadline = SessionLastKnownGoodTimestamp + Settings_.Timeout_; + auto minimalWaitDeadline = now + Settings_.Timeout_ / 4; + nextTimerTimestamp = Max(expectedSessionDeadline, minimalWaitDeadline); + } + } + + auto handler = [self = TPtr(this), context] (bool ok) mutable { + if (ok) { + self->OnSelfPingTimer(std::move(context)); + } + }; + Connections_->ScheduleCallback( + nextTimerTimestamp - TInstant::Now(), + std::move(handler), + std::move(context)); + } + + bool ProcessResponse(const IProcessor::TPtr& processor) { + switch (Response->response_case()) { + case TResponse::RESPONSE_NOT_SET: { + Y_ABORT("Unexpected empty response received"); + } + case TResponse::kPing: { + const auto& source = Response->ping(); + TRequest req; + req.mutable_pong()->set_opaque(source.opaque()); + processor->Write(std::move(req)); + return true; + } + case TResponse::kPong: { + const auto& source = Response->pong(); + const uint64_t reqId = source.opaque(); + TResultPromise replyPromise; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + replyPromise.Swap(op->Promise); + SentRequests.erase(reqId); + } + } + if (replyPromise.Initialized()) { + replyPromise.SetValue(TResult(MakeStatus())); + } + return true; + } + case TResponse::kFailure: { + const auto& source = Response->failure(); + auto plain = MakePlainStatus(source.status(), source.issues()); + bool expired = false; + bool detached = false; + { + std::lock_guard guard(Lock); + if (plain.Status == EStatus::SESSION_EXPIRED || + plain.Status == EStatus::UNAUTHORIZED || + plain.Status == EStatus::NOT_FOUND) + { + // Fatal errors move to the stopping state + IsStopping = true; + if (SessionId != 0 && SessionState != ESessionState::EXPIRED) { + SessionState = ESessionState::EXPIRED; + expired = true; + } + } + if (SessionState == ESessionState::ATTACHED) { + SessionState = ESessionState::DETACHED; + detached = true; + } + if (!CurrentFailure) { + SetCurrentFailure(std::move(plain)); + } + } + if (expired && Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + } else if (detached && Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::DETACHED); + } + return false; + } + case TResponse::kSessionStarted: { + const auto& source = Response->session_started(); + Y_ABORT_UNLESS(source.session_id() > 0); + TPromise replyPromise; + IQueueClientContextPtr sessionStartTimeoutContext; + IQueueClientContextPtr sessionSelfPingContext; + TInstant selfPingFirstTimestamp; + { + std::lock_guard guard(Lock); + // Remember session id for future reconnects + SessionId = source.session_id(); + if (SessionPromise.Initialized()) { + SessionPromise.Swap(replyPromise); + } + UpdateLastKnownGoodTimestampLocked(SessionStartTimestamp); + sessionStartTimeoutContext = std::exchange(SessionStartTimeoutContext, nullptr); + + SessionAttached(); + + if (Settings_.Timeout_ != TDuration::Max() && !SessionSelfPingContext) { + // We want to send the first ping 2/3 of the way towards session timeout + SessionSelfPingContext = sessionSelfPingContext = LocalContext->CreateContext(); + selfPingFirstTimestamp = SessionLastKnownGoodTimestamp + + Settings_.Timeout_ * (2.0 / 3.0); + } + } + if (Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::ATTACHED); + } + if (replyPromise.Initialized()) { + // If there are no listeners session destructor will be immediately called outside of the lock + auto session = TSession(this); + replyPromise.SetValue(TSessionResult(MakeStatus(), std::move(session))); + replyPromise = {}; + } + if (sessionStartTimeoutContext) { + sessionStartTimeoutContext->Cancel(); + } + if (sessionSelfPingContext) { + // N.B. to avoid deadlocks timers must be started outside of locks + auto handler = [self = TPtr(this), sessionSelfPingContext] (bool ok) mutable { + if (ok) { + self->OnSelfPingTimer(std::move(sessionSelfPingContext)); + } + }; + Connections_->ScheduleCallback( + selfPingFirstTimestamp - TInstant::Now(), + std::move(handler), + std::move(sessionSelfPingContext)); + } + return true; + } + case TResponse::kSessionStopped: { + bool expired = false; + { + std::lock_guard guard(Lock); + IsStopping = true; + if (!ClosedPromise.Initialized()) { + ClosedPromise = NewResultPromise(); + } + if (!CurrentFailure) { + SetCurrentFailure(TPlainStatus(EStatus::SESSION_EXPIRED, "Session expired")); + } + if (SessionId != 0 && SessionState != ESessionState::EXPIRED) { + SessionState = ESessionState::EXPIRED; + expired = true; + } + Y_ABORT_UNLESS(SessionState != ESessionState::ATTACHED); + } + if (expired && Settings_.OnStateChanged_) { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + } + return false; + } + case TResponse::kAcquireSemaphorePending: { + const auto& source = Response->acquire_semaphore_pending(); + uint64_t reqId = source.req_id(); + TResultPromise supersededPromise; + std::function acceptedCallback; + with_lock(Lock) { + auto statePtr = MapFindPtr(SemaphoreByReqId, reqId); + if (statePtr) { + auto* state = *statePtr; + auto op = state->LastSentOp; + Y_ABORT_UNLESS(op && op->ReqId == reqId && op->OpType == SEM_OP_ACQUIRE, + "Received AcquireSemaphorePending for an unexpected request"); + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + if (state->Restoring && state->LastAckedOp && state->LastAckedOp != op) { + // The last acknowledged op hasn't been retried + supersededPromise.Swap(state->LastAckedOp->Promise); + } + if (state->LastAckedOp != op) { + // This is the first time request is acknowledged + auto* acquire = static_cast(op.Get()); + acceptedCallback.swap(acquire->Settings.OnAccepted_); + } + state->LastAckedOp = op; + state->LastSentOp = {}; + state->Restoring = false; + DoSemaphoreProcessQueue(state); + } + } + if (supersededPromise.Initialized()) { + auto status = MakeStatus(EStatus::ABORTED, "Operation superseded by another request"); + supersededPromise.SetValue(TResult(std::move(status), false)); + } + if (acceptedCallback) { + acceptedCallback(); + } + return true; + } + case TResponse::kAcquireSemaphoreResult: { + const auto& source = Response->acquire_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TResultPromise supersededPromise; + TResultPromise resultPromise; + { + std::lock_guard guard(Lock); + DoSemaphoreProcessResult(reqId, SEM_OP_ACQUIRE, plain.Status, &supersededPromise, &resultPromise); + } + if (supersededPromise.Initialized()) { + auto status = MakeStatus(EStatus::ABORTED, "Operation superseded by another request"); + supersededPromise.SetValue(TResult(std::move(status), false)); + } + if (resultPromise.Initialized()) { + resultPromise.SetValue(TResult(MakeStatus(std::move(plain)), source.acquired())); + } + return true; + } + case TResponse::kReleaseSemaphoreResult: { + const auto& source = Response->release_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TResultPromise supersededPromise; + TResultPromise resultPromise; + { + std::lock_guard guard(Lock); + DoSemaphoreProcessResult(reqId, SEM_OP_RELEASE, plain.Status, &supersededPromise, &resultPromise); + } + if (supersededPromise.Initialized()) { + auto status = MakeStatus(EStatus::ABORTED, "Operation superseded by another request"); + supersededPromise.SetValue(TResult(std::move(status), false)); + } + if (resultPromise.Initialized()) { + resultPromise.SetValue(TResult(MakeStatus(std::move(plain)), source.released())); + } + return true; + } + case TResponse::kDescribeSemaphoreResult: { + const auto& source = Response->describe_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TPromise resultPromise; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + resultPromise.Swap(op->Promise); + if (!source.watch_added()) { + SentRequests.erase(reqId); + } + } + } + if (resultPromise.Initialized()) { + resultPromise.SetValue( + TDescribeSemaphoreResult( + MakeStatus(std::move(plain)), + source.semaphore_description())); + } + return true; + } + case TResponse::kDescribeSemaphoreChanged: { + const auto& source = Response->describe_semaphore_changed(); + const uint64_t reqId = source.req_id(); + std::function callback; + bool triggered = false; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + callback.swap(op->Settings.OnChanged_); + triggered = (op->Settings.WatchData_ && source.data_changed()) + || (op->Settings.WatchOwners_ && source.owners_changed()); + SentRequests.erase(reqId); + } + } + if (callback) { + callback(triggered); + } + return true; + } + case TResponse::kCreateSemaphoreResult: { + const auto& source = Response->create_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TResultPromise resultPromise; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + resultPromise.Swap(op->Promise); + SentRequests.erase(reqId); + } + } + if (resultPromise.Initialized()) { + resultPromise.SetValue(TResult(MakeStatus(std::move(plain)))); + } + return true; + } + case TResponse::kUpdateSemaphoreResult: { + const auto& source = Response->update_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TResultPromise resultPromise; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + resultPromise.Swap(op->Promise); + SentRequests.erase(reqId); + } + } + if (resultPromise.Initialized()) { + resultPromise.SetValue(TResult(MakeStatus(std::move(plain)))); + } + return true; + } + case TResponse::kDeleteSemaphoreResult: { + const auto& source = Response->delete_semaphore_result(); + const uint64_t reqId = source.req_id(); + auto plain = MakePlainStatus(source.status(), source.issues()); + TResultPromise resultPromise; + { + std::lock_guard guard(Lock); + if (auto* op = FindSentRequest(reqId)) { + UpdateLastKnownGoodTimestampLocked(op->SendTimestamp); + resultPromise.Swap(op->Promise); + SentRequests.erase(reqId); + } + } + if (resultPromise.Initialized()) { + resultPromise.SetValue(TResult(MakeStatus(std::move(plain)))); + } + return true; + } + case TResponse::kUnsupported6: + case TResponse::kUnsupported7: + case TResponse::kUnsupported16: + case TResponse::kUnsupported17: + case TResponse::kUnsupported18: + break; + } + + Y_ABORT("Unsupported message received"); + } + + void OnSessionStartTimeout(IQueueClientContextPtr context) { + Y_ABORT_UNLESS(context, "Unexpected empty timeout context"); + + std::lock_guard guard(Lock); + if (SessionStartTimeoutContext != context) { + // Context changed or dropped, ignore + return; + } + + SessionStartTimeoutContext.reset(); + + // Mark session as timed out + SessionTimeout = true; + if (!CurrentFailure) { + SetCurrentFailure( + TPlainStatus(EStatus::TIMEOUT, "Session start request timed out")); + } + + // We cannot report timeout until OnRead finishes with an error + if (Processor) { + Processor->Cancel(); + } + } + +private: + TGRpcConnectionsImpl* const Connections_; + TDbDriverStatePtr DbDriverState_; + const std::string Path_; + const TSessionSettings Settings_; + const std::string ProtectionKey_; + + TAdaptiveLock Lock; + + // Used for shutdown notification and keeping cq threads alive + IQueueClientContextPtr LocalContext; + + // Used for the initial attach only + TPromise SessionPromise; + TFuture SessionFuture; + + ESessionState SessionState = ESessionState::DETACHED; + EConnectionState ConnectionState = EConnectionState::DISCONNECTED; + + std::unique_ptr CurrentFailure; + TResultPromise ClosedPromise; + + // Used during a connection attempt for a custom timeout + IQueueClientContextPtr ConnectContext; + IQueueClientContextPtr ConnectTimeoutContext; + + std::unordered_map Semaphores; + std::unordered_map SemaphoreByReqId; + std::deque> PendingRequests; + std::unordered_map> SentRequests; + TResultPromise ReconnectPromise; + + // These are used to manage session timeout + IQueueClientContextPtr SessionStartTimeoutContext; + IQueueClientContextPtr SessionSelfPingContext; + uint64_t SessionSelfPingReqId = 0; + + // This will be set to true when session is timed out + bool SessionTimeout = false; + + // Timestamp of the last session start request + TInstant SessionStartTimestamp; + + // This is the last timestamp when session was confirmed to be good + TInstant SessionLastKnownGoodTimestamp; + + // Reconnection uses sleep between attempts + IQueueClientContextPtr SleepContext; + std::optional SessionExpireDeadline; + TDuration SessionReconnectDelay = TDuration::Zero(); + + IProcessor::TPtr Processor; + std::unique_ptr Response; + + uint64_t SessionSeqNo = 0; + uint64_t SessionId = 0; + uint64_t NextReqId = 1; + + bool IsStopping = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncSessionResult StartSession( + const std::string& path, + const TSessionSettings& settings) + { + auto session = MakeIntrusive( + Connections_.get(), + DbDriverState_, + path, + settings); + + auto result = session->TakeStartResult(); + + session->Start(Connections_.get()); + + return result; + } + + TAsyncStatus CreateNode( + Ydb::Coordination::CreateNodeRequest&& request, + const TCreateNodeSettings& settings) + { + return RunSimple( + std::move(request), + &Ydb::Coordination::V1::CoordinationService::Stub::AsyncCreateNode, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AlterNode( + Ydb::Coordination::AlterNodeRequest&& request, + const TAlterNodeSettings& settings) + { + return RunSimple( + std::move(request), + &Ydb::Coordination::V1::CoordinationService::Stub::AsyncAlterNode, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus DropNode( + Ydb::Coordination::DropNodeRequest&& request, + const TDropNodeSettings& settings) + { + return RunSimple( + std::move(request), + &Ydb::Coordination::V1::CoordinationService::Stub::AsyncDropNode, + TRpcRequestSettings::Make(settings)); + } + + TAsyncDescribeNodeResult DescribeNode( + Ydb::Coordination::DescribeNodeRequest&& request, + const TDescribeNodeSettings& settings) + { + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Coordination::DescribeNodeResult result; + if (any) { + any->UnpackTo(&result); + } + promise.SetValue( + TDescribeNodeResult( + TStatus(std::move(status)), + TNodeDescription(std::move(result)))); + }; + + Connections_->RunDeferred( + std::move(request), + std::move(extractor), + &Ydb::Coordination::V1::CoordinationService::Stub::AsyncDescribeNode, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } +}; + +TClient::TClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ } + +TClient::~TClient() { + // nothing special +} + +TAsyncSessionResult TClient::StartSession( + const std::string& path, + const TSessionSettings& settings) +{ + return Impl_->StartSession(path, settings); +} + +TAsyncStatus TClient::CreateNode( + const std::string& path, + const TCreateNodeSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + ConvertSettingsToProtoConfig(settings, request.mutable_config()); + return Impl_->CreateNode(std::move(request), settings); +} + +TAsyncStatus TClient::AlterNode( + const std::string& path, + const TAlterNodeSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + ConvertSettingsToProtoConfig(settings, request.mutable_config()); + return Impl_->AlterNode(std::move(request), settings); +} + +TAsyncStatus TClient::DropNode( + const std::string& path, + const TDropNodeSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + return Impl_->DropNode(std::move(request), settings); +} + +TAsyncDescribeNodeResult TClient::DescribeNode( + const std::string& path, + const TDescribeNodeSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + return Impl_->DescribeNode(std::move(request), settings); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TSession::TImpl { +public: + TImpl(TSessionContext* context) + : Context(context) + { } + + ~TImpl() { + Context->DoClose(true); + } + + uint64_t GetSessionId() { + return Context->SessionId; + } + + ESessionState GetSessionState() { + return Context->DoGetSessionState(); + } + + EConnectionState GetConnectionState() { + return Context->DoGetConnectionState(); + } + + TAsyncResult Close() { + return Context->DoClose(false); + } + + TAsyncResult Ping() { + return Context->DoPing(); + } + + TAsyncResult Reconnect() { + return Context->DoReconnect(); + } + + TAsyncResult AcquireSemaphore( + const std::string& name, + const TAcquireSemaphoreSettings& settings) + { + return Context->DoAcquireSemaphore(name, settings); + } + + TAsyncResult ReleaseSemaphore(const std::string& name) { + return Context->DoReleaseSemaphore(name); + } + + TAsyncDescribeSemaphoreResult DescribeSemaphore( + const std::string& name, + const TDescribeSemaphoreSettings& settings) + { + return Context->DoDescribeSemaphore(name, settings); + } + + TAsyncResult CreateSemaphore(const std::string& name, uint64_t limit, const std::string& data) { + return Context->DoCreateSemaphore(name, limit, data); + } + + TAsyncResult UpdateSemaphore(const std::string& name, const std::string& data) { + return Context->DoUpdateSemaphore(name, data); + } + + TAsyncResult DeleteSemaphore(const std::string& name, bool force) { + return Context->DoDeleteSemaphore(name, force); + } + +private: + const TIntrusivePtr Context; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TSession::TSession(TSessionContext* context) + : Impl_(std::make_shared(context)) +{ } + +uint64_t TSession::GetSessionId() { + return Impl_->GetSessionId(); +} + +ESessionState TSession::GetSessionState() { + return Impl_->GetSessionState(); +} + +EConnectionState TSession::GetConnectionState() { + return Impl_->GetConnectionState(); +} + +TAsyncResult TSession::Close() { + return Impl_->Close(); +} + +TAsyncResult TSession::Ping() { + return Impl_->Ping(); +} + +TAsyncResult TSession::Reconnect() { + return Impl_->Reconnect(); +} + +TAsyncResult TSession::AcquireSemaphore( + const std::string& name, + const TAcquireSemaphoreSettings& settings) +{ + return Impl_->AcquireSemaphore(name, settings); +} + +TAsyncResult TSession::ReleaseSemaphore(const std::string& name) { + return Impl_->ReleaseSemaphore(name); +} + +TAsyncDescribeSemaphoreResult TSession::DescribeSemaphore( + const std::string& name, + const TDescribeSemaphoreSettings& settings) +{ + return Impl_->DescribeSemaphore(name, settings); +} + +TAsyncResult TSession::CreateSemaphore( + const std::string& name, + uint64_t limit, + const std::string& data) +{ + return Impl_->CreateSemaphore(name, limit, data); +} + +TAsyncResult TSession::UpdateSemaphore( + const std::string& name, + const std::string& data) +{ + return Impl_->UpdateSemaphore(name, data); +} + +TAsyncResult TSession::DeleteSemaphore( + const std::string& name, + bool force) +{ + return Impl_->DeleteSemaphore(name, force); +} + +} +} diff --git a/ydb/public/sdk/cpp/src/client/coordination/proto_accessor.cpp b/ydb/public/sdk/cpp/src/client/coordination/proto_accessor.cpp new file mode 100644 index 000000000000..c6a1051d42d3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/coordination/proto_accessor.cpp @@ -0,0 +1,11 @@ +#include + +#include + +namespace NYdb::inline V3 { + +const Ydb::Coordination::DescribeNodeResult& TProtoAccessor::GetProto(const NCoordination::TNodeDescription& nodeDescription) { + return nodeDescription.GetProto(); +} + +} \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/coordination/ya.make b/ydb/public/sdk/cpp/src/client/coordination/ya.make new file mode 100644 index 000000000000..e31a14304948 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/coordination/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + coordination.cpp + proto_accessor.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/coordination/coordination.h) + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/types + ydb/public/sdk/cpp/src/client/types/status +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/datastreams/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/datastreams/CMakeLists.txt new file mode 100644 index 000000000000..ec4305e12874 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/datastreams/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-ydb_datastreams) + +target_link_libraries(client-ydb_datastreams PUBLIC + yutil + grpc-client + string_utils-url + api-grpc-draft + library-operation_id + impl-ydb_internal-make_request + client-ydb_driver +) + +target_sources(client-ydb_datastreams PRIVATE + datastreams.cpp +) + +_ydb_sdk_make_client_component(Datastreams client-ydb_datastreams) diff --git a/ydb/public/sdk/cpp/src/client/datastreams/datastreams.cpp b/ydb/public/sdk/cpp/src/client/datastreams/datastreams.cpp new file mode 100644 index 000000000000..aab7108f8b69 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/datastreams/datastreams.cpp @@ -0,0 +1,957 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NDataStreams::V1 { + + TPartitioningSettingsBuilder TCreateStreamSettings::BeginConfigurePartitioningSettings() { + return { *this }; + } + + TPartitioningSettingsBuilder TUpdateStreamSettings::BeginConfigurePartitioningSettings() { + return { *this }; + } + + void SetPartitionSettings(const TPartitioningSettings& ps, ::Ydb::DataStreams::V1::PartitioningSettings* pt) { + pt->set_max_active_partitions(ps.GetMaxActivePartitions()); + pt->set_min_active_partitions(ps.GetMinActivePartitions()); + + ::Ydb::DataStreams::V1::AutoPartitioningStrategy strategy; + switch (ps.GetAutoPartitioningSettings().GetStrategy()) { + case EAutoPartitioningStrategy::Unspecified: + case EAutoPartitioningStrategy::Disabled: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_DISABLED; + break; + case EAutoPartitioningStrategy::ScaleUp: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_SCALE_UP; + break; + case EAutoPartitioningStrategy::ScaleUpAndDown: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN; + break; + case EAutoPartitioningStrategy::Paused: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_PAUSED; + break; + } + + pt->mutable_auto_partitioning_settings()->set_strategy(strategy); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->mutable_stabilization_window()->set_seconds(ps.GetAutoPartitioningSettings().GetStabilizationWindow().Seconds()); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->set_up_utilization_percent(ps.GetAutoPartitioningSettings().GetUpUtilizationPercent()); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->set_down_utilization_percent(ps.GetAutoPartitioningSettings().GetDownUtilizationPercent()); + } + + class TDataStreamsClient::TImpl : public TClientImplCommon { + public: + TImpl(std::shared_ptr &&connections, const TCommonClientSettings &settings) + : TClientImplCommon(std::move(connections), settings) {} + + template + auto MakeResultExtractor(NThreading::TPromise promise) { + return [promise = std::move(promise)] + (google::protobuf::Any *any, TPlainStatus status) mutable { + std::unique_ptr result; + if (any) { + result.reset(new TProtoResult); + any->UnpackTo(result.get()); + } + + promise.SetValue( + TResultWrapper( + TStatus(std::move(status)), + std::move(result))); + }; + } + + template + NThreading::TFuture> CallImpl(const TSettings& settings, TAsyncCall grpcCall, TFillRequestFn fillRequest) { + using TResultWrapper = TProtoResultWrapper; + auto request = MakeOperationRequest(settings); + fillRequest(request); + + auto promise = NThreading::NewPromise(); + auto future = promise.GetFuture(); + + auto extractor = MakeResultExtractor(std::move(promise)); + + Connections_->RunDeferred( + std::move(request), + std::move(extractor), + grpcCall, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return future; + + } + + template + NThreading::TFuture> CallImpl(const TSettings& settings, TAsyncCall grpcCall) { + return CallImpl(settings, grpcCall, [](TProtoRequest&) {}); + } + + TAsyncCreateStreamResult CreateStream(const std::string &path, TCreateStreamSettings settings) { + if (settings.RetentionPeriodHours_.has_value() && settings.RetentionStorageMegabytes_.has_value()) { + return NThreading::MakeFuture(TProtoResultWrapper( + NYdb::TPlainStatus(NYdb::EStatus::BAD_REQUEST, "both retention types can not be set"), + std::make_unique())); + } + return CallImpl(settings, + &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream, + [&](Ydb::DataStreams::V1::CreateStreamRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_shard_count(settings.ShardCount_); + if (settings.RetentionStorageMegabytes_.has_value()) { + req.set_retention_storage_megabytes(*settings.RetentionStorageMegabytes_); + } else if (settings.RetentionPeriodHours_.has_value()) { + req.set_retention_period_hours(*settings.RetentionPeriodHours_); + } else { + req.set_retention_period_hours(24); + } + req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); + if (settings.StreamMode_.has_value()) { + req.mutable_stream_mode_details()->set_stream_mode( + *settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED + : Ydb::DataStreams::V1::StreamMode::ON_DEMAND); + } + + if (settings.PartitioningSettings_.has_value()) { + SetPartitionSettings(*settings.PartitioningSettings_, req.mutable_partitioning_settings()); + } + }); + } + + TAsyncListStreamsResult ListStreams(TListStreamsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams, + [&](Ydb::DataStreams::V1::ListStreamsRequest& req) { + req.set_exclusive_start_stream_name(TStringType{settings.ExclusiveStartStreamName_}); + req.set_limit(settings.Limit_); + req.set_recurse(settings.Recurse_); + }); + } + + TAsyncDescribeStreamResult DescribeStream(TDescribeStreamSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream); + } + + TAsyncListShardsResult ListShards(const std::string &path, + const Ydb::DataStreams::V1::ShardFilter& shardFilter, + TListShardsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards, + [&](Ydb::DataStreams::V1::ListShardsRequest& req) { + req.set_exclusive_start_shard_id(TStringType{settings.ExclusiveStartShardId_}); + req.set_max_results(settings.MaxResults_); + req.set_next_token(TStringType{settings.NextToken_}); + req.mutable_shard_filter()->CopyFrom(shardFilter); + req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_); + req.set_stream_name(TStringType{path}); + }); + } + + TAsyncPutRecordsResult PutRecords(const std::string& path, const std::vector& records, TPutRecordsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords, + [&](Ydb::DataStreams::V1::PutRecordsRequest& req) { + req.set_stream_name(TStringType{path}); + for (const auto& record : records) { + auto* protoRecord = req.add_records(); + protoRecord->set_partition_key(TStringType{record.PartitionKey}); + protoRecord->set_data(TStringType{record.Data}); + protoRecord->set_explicit_hash_key(TStringType{record.ExplicitHashDecimal}); + } + }); + } + + TAsyncGetRecordsResult GetRecords(const std::string& shardIterator, TGetRecordsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords, + [&](Ydb::DataStreams::V1::GetRecordsRequest& req) { + req.set_shard_iterator(TStringType{shardIterator}); + req.set_limit(settings.Limit_); + }); + } + + TAsyncGetShardIteratorResult GetShardIterator(const std::string& path, const std::string& shardId, + Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, + TGetShardIteratorSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator, + [&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_shard_id(TStringType{shardId}); + req.set_shard_iterator_type(shardIteratorType); + req.set_starting_sequence_number(TStringType{settings.StartingSequenceNumber_}); + req.set_timestamp(settings.Timestamp_); + }); + } + + /*TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSubscribeToShard); + }*/ + + TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits); + } + + TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const std::string& path, TDescribeStreamSummarySettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary, + [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) { + req.set_stream_name(TStringType{path}); + }); + } + + TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const std::string& path, TDecreaseStreamRetentionPeriodSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod, + [&](Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + }); + + } + + TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const std::string& path, TIncreaseStreamRetentionPeriodSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod, + [&](Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + }); + + } + + TAsyncUpdateShardCountResult UpdateShardCount(const std::string& path, TUpdateShardCountSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount, + [&](Ydb::DataStreams::V1::UpdateShardCountRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_target_shard_count(settings.TargetShardCount_); + }); + } + + TAsyncUpdateStreamModeResult UpdateStreamMode(const std::string& path, TUpdateStreamModeSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStreamMode, + [&](Ydb::DataStreams::V1::UpdateStreamModeRequest& req) { + req.set_stream_arn(TStringType{path}); + + req.mutable_stream_mode_details()->set_stream_mode( + settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED + : Ydb::DataStreams::V1::StreamMode::ON_DEMAND); + }); + } + + TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const std::string& path, const std::string& consumer_name, TRegisterStreamConsumerSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer, + [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) { + req.set_stream_arn(TStringType{path}); + req.set_consumer_name(TStringType{consumer_name}); + }); + } + + TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const std::string& path, const std::string& consumer_name, TDeregisterStreamConsumerSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer, + [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) { + req.set_stream_arn(TStringType{path}); + req.set_consumer_name(TStringType{consumer_name}); + }); + } + + TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer); + } + + TAsyncListStreamConsumersResult ListStreamConsumers(const std::string& path, TListStreamConsumersSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) { + req.set_stream_arn(TStringType{path}); + req.set_next_token(TStringType{settings.NextToken_}); + req.set_max_results(settings.MaxResults_); + }); + } + + TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream); + } + + TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring); + } + + TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring); + } + + TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream); + } + + TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards); + } + + TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream); + } + + TAsyncSplitShardResult SplitShard(TSplitShardSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard); + } + + TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption); + } + + TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption); + } + + TAsyncUpdateStreamResult UpdateStream(const std::string& streamName, TUpdateStreamSettings settings) { + if (settings.RetentionPeriodHours_.has_value() && settings.RetentionStorageMegabytes_.has_value()) { + return NThreading::MakeFuture(TProtoResultWrapper( + NYdb::TPlainStatus(NYdb::EStatus::BAD_REQUEST, "both retention types can not be set"), + std::make_unique())); + } + return CallImpl(settings, + &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream, + [&](Ydb::DataStreams::V1::UpdateStreamRequest& req) { + req.set_stream_name(TStringType{streamName}); + req.set_target_shard_count(settings.TargetShardCount_); + if (settings.RetentionPeriodHours_.has_value()) { + req.set_retention_period_hours(*settings.RetentionPeriodHours_); + } + if (settings.RetentionStorageMegabytes_.has_value()) { + req.set_retention_storage_megabytes(*settings.RetentionStorageMegabytes_); + } + req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); + if (settings.StreamMode_.has_value()) { + req.mutable_stream_mode_details()->set_stream_mode( + *settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED + : Ydb::DataStreams::V1::StreamMode::ON_DEMAND); + } + + if (settings.PartitioningSettings_.has_value()) { + SetPartitionSettings(*settings.PartitioningSettings_, req.mutable_partitioning_settings()); + } + }); + } + + TAsyncDeleteStreamResult DeleteStream(const std::string &path, TDeleteStreamSettings settings) { + return CallImpl(settings, + &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream, + [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_); + }); + } + + TAsyncDescribeStreamResult DescribeStream(const std::string &path, TDescribeStreamSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream, + [&](Ydb::DataStreams::V1::DescribeStreamRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_exclusive_start_shard_id(TStringType{settings.ExclusiveStartShardId_}); + req.set_limit(settings.Limit_); + }); + } + + TAsyncPutRecordResult PutRecord(const std::string &path, const TDataRecord& record, TPutRecordSettings settings) { + return CallImpl(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord, + [&](Ydb::DataStreams::V1::PutRecordRequest& req) { + req.set_stream_name(TStringType{path}); + req.set_explicit_hash_key(TStringType{record.ExplicitHashDecimal}); + req.set_partition_key(TStringType{record.PartitionKey}); + req.set_data(TStringType{record.Data}); + }); + } + + template + NThreading::TFuture> DoProtoRequest(const TProtoRequest& proto, TMethod method, const TProtoRequestSettings& settings) { + return CallImpl(settings, method, + [&](TProtoRequest& req) { + req.CopyFrom(proto); + }); + } + + }; + + TDataStreamsClient::TDataStreamsClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) + {} + + NThreading::TFuture TDataStreamsClient::DiscoveryCompleted() { + return Impl_->DiscoveryCompleted(); + } + + TAsyncCreateStreamResult TDataStreamsClient::CreateStream(const std::string& path, TCreateStreamSettings settings) { + return Impl_->CreateStream(path, settings); + } + + TAsyncDeleteStreamResult TDataStreamsClient::DeleteStream(const std::string& path, TDeleteStreamSettings settings) { + return Impl_->DeleteStream(path, settings); + } + + TAsyncDescribeStreamResult TDataStreamsClient::DescribeStream(const std::string& path, TDescribeStreamSettings settings) { + return Impl_->DescribeStream(path, settings); + } + + TAsyncPutRecordResult TDataStreamsClient::PutRecord(const std::string& path, const TDataRecord& record, TPutRecordSettings settings) { + return Impl_->PutRecord(path, record, settings); + } + + TAsyncListStreamsResult TDataStreamsClient::ListStreams(TListStreamsSettings settings) { + return Impl_->ListStreams(settings); + } + + TAsyncListShardsResult TDataStreamsClient::ListShards(const std::string& path, + const Ydb::DataStreams::V1::ShardFilter& shardFilter, + TListShardsSettings settings) { + return Impl_->ListShards(path, shardFilter, settings); + } + + TAsyncPutRecordsResult TDataStreamsClient::PutRecords(const std::string& path, const std::vector& records, TPutRecordsSettings settings) { + return Impl_->PutRecords(path, records, settings); + } + + TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const std::string& shardIterator, TGetRecordsSettings settings) { + return Impl_->GetRecords(shardIterator, settings); + } + + TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const std::string& path, const std::string& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) { + return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings); + } + + /* TAsyncSubscribeToShardResult TDataStreamsClient::SubscribeToShard(TSubscribeToShardSettings settings) { + return Impl_->SubscribeToShard(settings); + } */ + + TAsyncDescribeLimitsResult TDataStreamsClient::DescribeLimits(TDescribeLimitsSettings settings) { + return Impl_->DescribeLimits(settings); + } + + TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const std::string& path, TDescribeStreamSummarySettings settings) { + return Impl_->DescribeStreamSummary(path, settings); + } + + TAsyncDecreaseStreamRetentionPeriodResult TDataStreamsClient::DecreaseStreamRetentionPeriod(const std::string& path, TDecreaseStreamRetentionPeriodSettings settings) { + return Impl_->DecreaseStreamRetentionPeriod(path, settings); + } + + TAsyncIncreaseStreamRetentionPeriodResult TDataStreamsClient::IncreaseStreamRetentionPeriod(const std::string& path, TIncreaseStreamRetentionPeriodSettings settings) { + return Impl_->IncreaseStreamRetentionPeriod(path, settings); + } + + TAsyncUpdateShardCountResult TDataStreamsClient::UpdateShardCount(const std::string& path, TUpdateShardCountSettings settings) { + return Impl_->UpdateShardCount(path, settings); + } + + TAsyncUpdateStreamModeResult TDataStreamsClient::UpdateStreamMode(const std::string& path, TUpdateStreamModeSettings settings) { + return Impl_->UpdateStreamMode(path, settings); + } + + TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const std::string& path, const std::string& consumer_name, const TRegisterStreamConsumerSettings settings) { + return Impl_->RegisterStreamConsumer(path, consumer_name, settings); + } + + TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const std::string& path, const std::string& consumer_name, TDeregisterStreamConsumerSettings settings) { + return Impl_->DeregisterStreamConsumer(path, consumer_name, settings); + } + + TAsyncDescribeStreamConsumerResult TDataStreamsClient::DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { + return Impl_->DescribeStreamConsumer(settings); + } + + TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const std::string& path, TListStreamConsumersSettings settings) { + return Impl_->ListStreamConsumers(path, settings); + } + + TAsyncAddTagsToStreamResult TDataStreamsClient::AddTagsToStream(TAddTagsToStreamSettings settings) { + return Impl_->AddTagsToStream(settings); + } + + TAsyncDisableEnhancedMonitoringResult TDataStreamsClient::DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { + return Impl_->DisableEnhancedMonitoring(settings); + } + + TAsyncEnableEnhancedMonitoringResult TDataStreamsClient::EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { + return Impl_->EnableEnhancedMonitoring(settings); + } + + TAsyncListTagsForStreamResult TDataStreamsClient::ListTagsForStream(TListTagsForStreamSettings settings) { + return Impl_->ListTagsForStream(settings); + } + + TAsyncMergeShardsResult TDataStreamsClient::MergeShards(TMergeShardsSettings settings) { + return Impl_->MergeShards(settings); + } + + TAsyncRemoveTagsFromStreamResult TDataStreamsClient::RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { + return Impl_->RemoveTagsFromStream(settings); + } + + TAsyncSplitShardResult TDataStreamsClient::SplitShard(TSplitShardSettings settings) { + return Impl_->SplitShard(settings); + } + + TAsyncStartStreamEncryptionResult TDataStreamsClient::StartStreamEncryption(TStartStreamEncryptionSettings settings) { + return Impl_->StartStreamEncryption(settings); + } + + TAsyncStopStreamEncryptionResult TDataStreamsClient::StopStreamEncryption(TStopStreamEncryptionSettings settings) { + return Impl_->StopStreamEncryption(settings); + } + + TAsyncUpdateStreamResult TDataStreamsClient::UpdateStream(const std::string& streamName, TUpdateStreamSettings settings) { + return Impl_->UpdateStream(streamName, settings); + } + + template + NThreading::TFuture> TDataStreamsClient::DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings) { + return Impl_->DoProtoRequest(request, method, settings); + } + + + // Instantiate template protobuf methods + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::PutRecordsRequest, + Ydb::DataStreams::V1::PutRecordsResponse, + Ydb::DataStreams::V1::PutRecordsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) + >( + const Ydb::DataStreams::V1::PutRecordsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture>TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::PutRecordRequest, + Ydb::DataStreams::V1::PutRecordResponse, + Ydb::DataStreams::V1::PutRecordResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) + >( + const Ydb::DataStreams::V1::PutRecordRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListStreamsRequest, + Ydb::DataStreams::V1::ListStreamsResponse, + Ydb::DataStreams::V1::ListStreamsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) + >( + const Ydb::DataStreams::V1::ListStreamsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::CreateStreamRequest, + Ydb::DataStreams::V1::CreateStreamResponse, + Ydb::DataStreams::V1::CreateStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) + >( + const Ydb::DataStreams::V1::CreateStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::UpdateStreamRequest, + Ydb::DataStreams::V1::UpdateStreamResponse, + Ydb::DataStreams::V1::UpdateStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream) + >( + const Ydb::DataStreams::V1::UpdateStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DeleteStreamRequest, + Ydb::DataStreams::V1::DeleteStreamResponse, + Ydb::DataStreams::V1::DeleteStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) + >( + const Ydb::DataStreams::V1::DeleteStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamRequest, + Ydb::DataStreams::V1::DescribeStreamResponse, + Ydb::DataStreams::V1::DescribeStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) + >( + const Ydb::DataStreams::V1::DescribeStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListShardsRequest, + Ydb::DataStreams::V1::ListShardsResponse, + Ydb::DataStreams::V1::ListShardsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) + >( + const Ydb::DataStreams::V1::ListShardsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::GetRecordsRequest, + Ydb::DataStreams::V1::GetRecordsResponse, + Ydb::DataStreams::V1::GetRecordsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) + >( + const Ydb::DataStreams::V1::GetRecordsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::GetShardIteratorRequest, + Ydb::DataStreams::V1::GetShardIteratorResponse, + Ydb::DataStreams::V1::GetShardIteratorResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) + >( + const Ydb::DataStreams::V1::GetShardIteratorRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeLimitsRequest, + Ydb::DataStreams::V1::DescribeLimitsResponse, + Ydb::DataStreams::V1::DescribeLimitsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) + >( + const Ydb::DataStreams::V1::DescribeLimitsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) + >( + const Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) + >( + const Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::UpdateShardCountRequest, + Ydb::DataStreams::V1::UpdateShardCountResponse, + Ydb::DataStreams::V1::UpdateShardCountResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) + >( + const Ydb::DataStreams::V1::UpdateShardCountRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::UpdateStreamModeRequest, + Ydb::DataStreams::V1::UpdateStreamModeResponse, + Ydb::DataStreams::V1::UpdateStreamModeResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStreamMode) + >( + const Ydb::DataStreams::V1::UpdateStreamModeRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStreamMode) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::RegisterStreamConsumerRequest, + Ydb::DataStreams::V1::RegisterStreamConsumerResponse, + Ydb::DataStreams::V1::RegisterStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) + >( + const Ydb::DataStreams::V1::RegisterStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, + Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, + Ydb::DataStreams::V1::DeregisterStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) + >( + const Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamConsumerRequest, + Ydb::DataStreams::V1::DescribeStreamConsumerResponse, + Ydb::DataStreams::V1::DescribeStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) + >( + const Ydb::DataStreams::V1::DescribeStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListStreamConsumersRequest, + Ydb::DataStreams::V1::ListStreamConsumersResponse, + Ydb::DataStreams::V1::ListStreamConsumersResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) + >( + const Ydb::DataStreams::V1::ListStreamConsumersRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::AddTagsToStreamRequest, + Ydb::DataStreams::V1::AddTagsToStreamResponse, + Ydb::DataStreams::V1::AddTagsToStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) + >( + const Ydb::DataStreams::V1::AddTagsToStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) + >( + const Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) + >( + const Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::MergeShardsRequest, + Ydb::DataStreams::V1::MergeShardsResponse, + Ydb::DataStreams::V1::MergeShardsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) + >( + const Ydb::DataStreams::V1::MergeShardsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListTagsForStreamRequest, + Ydb::DataStreams::V1::ListTagsForStreamResponse, + Ydb::DataStreams::V1::ListTagsForStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) + >( + const Ydb::DataStreams::V1::ListTagsForStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, + Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, + Ydb::DataStreams::V1::RemoveTagsFromStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) + >( + const Ydb::DataStreams::V1::RemoveTagsFromStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::SplitShardRequest, + Ydb::DataStreams::V1::SplitShardResponse, + Ydb::DataStreams::V1::SplitShardResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) + >( + const Ydb::DataStreams::V1::SplitShardRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::StartStreamEncryptionRequest, + Ydb::DataStreams::V1::StartStreamEncryptionResponse, + Ydb::DataStreams::V1::StartStreamEncryptionResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) + >( + const Ydb::DataStreams::V1::StartStreamEncryptionRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::StopStreamEncryptionRequest, + Ydb::DataStreams::V1::StopStreamEncryptionResponse, + Ydb::DataStreams::V1::StopStreamEncryptionResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) + >( + const Ydb::DataStreams::V1::StopStreamEncryptionRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamSummaryRequest, + Ydb::DataStreams::V1::DescribeStreamSummaryResponse, + Ydb::DataStreams::V1::DescribeStreamSummaryResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) + >( + const Ydb::DataStreams::V1::DescribeStreamSummaryRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) method, + TProtoRequestSettings settings + ); +} diff --git a/ydb/public/sdk/cpp/src/client/datastreams/ya.make b/ydb/public/sdk/cpp/src/client/datastreams/ya.make new file mode 100644 index 000000000000..eaa3bd3165d4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/datastreams/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + datastreams.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client + library/cpp/string_utils/url + ydb/public/api/grpc/draft + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/driver +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/debug/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/debug/CMakeLists.txt new file mode 100644 index 000000000000..3095290c0457 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/debug/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-debug) + +target_link_libraries(client-debug + PUBLIC + client-ydb_driver + PRIVATE + api-grpc + api-protos + client-ydb_common_client-impl +) + +target_sources(client-debug + PRIVATE + client.cpp +) + +_ydb_sdk_make_client_component(Debug client-debug) diff --git a/ydb/public/sdk/cpp/src/client/debug/client.cpp b/ydb/public/sdk/cpp/src/client/debug/client.cpp new file mode 100644 index 000000000000..2cca2e64465d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/debug/client.cpp @@ -0,0 +1,76 @@ +#include + +#include +#include +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +namespace NYdb::inline V3::NDebug { + +using namespace Ydb; + +using namespace NThreading; + +class TDebugClient::TImpl: public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + {} + + template + auto Ping(const TSettings& settings, auto serviceMethod) { + auto pingPromise = NewPromise(); + auto responseCb = [pingPromise] (TResponse*, TPlainStatus status) mutable { + TResult val(TStatus(std::move(status))); + pingPromise.SetValue(std::move(val)); + }; + + Connections_->Run( + TRequest(), + responseCb, + serviceMethod, + DbDriverState_, + TRpcRequestSettings::Make(settings)); + + return pingPromise; + } + + ~TImpl() = default; +}; + +TDebugClient::TDebugClient(const TDriver& driver, const TClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncPlainGrpcPingResult TDebugClient::PingPlainGrpc(const TPlainGrpcPingSettings& settings) { + return Impl_->Ping( + settings, &Debug::V1::DebugService::Stub::AsyncPingPlainGrpc); +} + +TAsyncGrpcProxyPingResult TDebugClient::PingGrpcProxy(const TGrpcProxyPingSettings& settings) { + return Impl_->Ping( + settings, &Debug::V1::DebugService::Stub::AsyncPingGrpcProxy); +} + +TAsyncKqpProxyPingResult TDebugClient::PingKqpProxy(const TKqpProxyPingSettings& settings) { + return Impl_->Ping( + settings, &Debug::V1::DebugService::Stub::AsyncPingKqpProxy); +} + +TAsyncSchemeCachePingResult TDebugClient::PingSchemeCache(const TSchemeCachePingSettings& settings) { + return Impl_->Ping( + settings, &Debug::V1::DebugService::Stub::AsyncPingSchemeCache); +} + +TAsyncTxProxyPingResult TDebugClient::PingTxProxy(const TTxProxyPingSettings& settings) { + return Impl_->Ping( + settings, &Debug::V1::DebugService::Stub::AsyncPingTxProxy); +} + +} // namespace NYdb::NDebug diff --git a/ydb/public/sdk/cpp/src/client/debug/ya.make b/ydb/public/sdk/cpp/src/client/debug/ya.make new file mode 100644 index 000000000000..1c88ec57901d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/debug/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + client.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/common_client/impl +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/discovery/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/discovery/CMakeLists.txt new file mode 100644 index 000000000000..30e1f6aee714 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/discovery/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-ydb_discovery) + +target_compile_options(client-ydb_discovery PRIVATE + -Wno-deprecated +) + +target_link_libraries(client-ydb_discovery PUBLIC + yutil + client-ydb_common_client-impl + client-ydb_driver +) + +target_sources(client-ydb_discovery PRIVATE + discovery.cpp +) + +_ydb_sdk_make_client_component(Discovery client-ydb_discovery) diff --git a/ydb/public/sdk/cpp/src/client/discovery/discovery.cpp b/ydb/public/sdk/cpp/src/client/discovery/discovery.cpp new file mode 100644 index 000000000000..c3c97d435fb2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/discovery/discovery.cpp @@ -0,0 +1,284 @@ +#include + +#include + +namespace NYdb::inline V3 { +namespace NDiscovery { + +TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& proto) + : TStatus(std::move(status)) +{ + const auto& endpoints = proto.endpoints(); + Info_.reserve(proto.endpoints().size()); + for (const auto& endpointInfo : endpoints) { + TEndpointInfo& info = Info_.emplace_back(); + info.Address = endpointInfo.address(); + info.Port = endpointInfo.port(); + info.Location = endpointInfo.location(); + info.NodeId = endpointInfo.node_id(); + info.LoadFactor = endpointInfo.load_factor(); + info.Ssl = endpointInfo.ssl(); + info.Services.reserve(endpointInfo.service().size()); + for (const auto& s : endpointInfo.service()) { + info.Services.emplace_back(s); + } + info.IPv4Addrs.reserve(endpointInfo.ip_v4().size()); + for (const auto& addr : endpointInfo.ip_v4()) { + info.IPv4Addrs.emplace_back(addr); + } + info.IPv6Addrs.reserve(endpointInfo.ip_v6().size()); + for (const auto& addr : endpointInfo.ip_v6()) { + info.IPv6Addrs.emplace_back(addr); + } + info.SslTargetNameOverride = endpointInfo.ssl_target_name_override(); + } +} + +const std::vector& TListEndpointsResult::GetEndpointsInfo() const { + return Info_; +} + +TWhoAmIResult::TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto) + : TStatus(std::move(status)) +{ + UserName_ = proto.user(); + const auto& groups = proto.groups(); + Groups_.reserve(groups.size()); + for (const auto& group : groups) { + Groups_.emplace_back(group); + } +} + +const std::string& TWhoAmIResult::GetUserName() const { + return UserName_; +} + +const std::vector& TWhoAmIResult::GetGroups() const { + return Groups_; +} + +TNodeLocation::TNodeLocation(const Ydb::Discovery::NodeLocation& location) + : DataCenterNum(location.has_data_center_num() ? std::make_optional(location.data_center_num()) : std::nullopt) + , RoomNum(location.has_room_num() ? std::make_optional(location.room_num()) : std::nullopt) + , RackNum(location.has_rack_num() ? std::make_optional(location.rack_num()) : std::nullopt) + , BodyNum(location.has_body_num() ? std::make_optional(location.body_num()) : std::nullopt) + , Body(location.has_body() ? std::make_optional(location.body()) : std::nullopt) + , DataCenter(location.has_data_center() ? std::make_optional(location.data_center()) : std::nullopt) + , Module(location.has_module() ? std::make_optional(location.module()) : std::nullopt) + , Rack(location.has_rack() ? std::make_optional(location.rack()) : std::nullopt) + , Unit(location.has_unit() ? std::make_optional(location.unit()) : std::nullopt) + {} + +TNodeInfo::TNodeInfo(const Ydb::Discovery::NodeInfo& info) + : NodeId(info.node_id()) + , Host(info.host()) + , Port(info.port()) + , ResolveHost(info.resolve_host()) + , Address(info.address()) + , Location(info.location()) + , Expire(info.expire()) + {} + +TNodeRegistrationResult::TNodeRegistrationResult(TStatus&& status, const Ydb::Discovery::NodeRegistrationResult& proto) + : TStatus(std::move(status)) + , NodeId_(proto.node_id()) + , DomainPath_(proto.domain_path()) + , Expire_(proto.expire()) + , ScopeTableId_(proto.has_scope_tablet_id() ? std::make_optional(proto.scope_tablet_id()) : std::nullopt) + , ScopePathId_(proto.has_scope_path_id() ? std::make_optional(proto.scope_path_id()) : std::nullopt) + , NodeName_(proto.has_node_name() ? std::make_optional(proto.node_name()) : std::nullopt) +{ + const auto& nodes = proto.nodes(); + Nodes_.reserve(nodes.size()); + for (const auto& node : nodes) { + Nodes_.emplace_back(node); + } +} + +uint32_t TNodeRegistrationResult::GetNodeId() const { + return NodeId_; +} + +const std::string& TNodeRegistrationResult::GetDomainPath() const { + return DomainPath_; +} + +uint64_t TNodeRegistrationResult::GetExpire() const { + return Expire_; +} + +uint64_t TNodeRegistrationResult::GetScopeTabletId() const { + return ScopeTableId_.value(); +} + +bool TNodeRegistrationResult::HasScopeTabletId() const { + return ScopeTableId_.has_value(); +} + +uint64_t TNodeRegistrationResult::GetScopePathId() const { + return ScopePathId_.value(); +} + +bool TNodeRegistrationResult::HasScopePathId() const { + return ScopePathId_.has_value(); +} + +bool TNodeRegistrationResult::HasNodeName() const { + return NodeName_.has_value(); +} + +const std::string& TNodeRegistrationResult::GetNodeName() const { + return NodeName_.value(); +} + +const std::vector& TNodeRegistrationResult::GetNodes() const { + return Nodes_; +} + +class TDiscoveryClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { } + + TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings) { + Ydb::Discovery::ListEndpointsRequest request; + request.set_database(TStringType{DbDriverState_->Database}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::ListEndpointsResult result; + if (any) { + any->UnpackTo(&result); + } + TListEndpointsResult val{TStatus(std::move(status)), result}; + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings) { + Ydb::Discovery::WhoAmIRequest request; + if (settings.WithGroups_) { + request.set_include_groups(true); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::WhoAmIResult result; + if (any) { + any->UnpackTo(&result); + } + TWhoAmIResult val{ TStatus(std::move(status)), result }; + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncWhoAmI, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncNodeRegistrationResult NodeRegistration(const TNodeRegistrationSettings& settings) { + Ydb::Discovery::NodeRegistrationRequest request; + request.set_host(TStringType{settings.Host_}); + request.set_port(settings.Port_); + request.set_resolve_host(TStringType{settings.ResolveHost_}); + request.set_address(TStringType{settings.Address_}); + request.set_domain_path(TStringType{settings.DomainPath_}); + request.set_fixed_node_id(settings.FixedNodeId_); + if (!settings.Path_.empty()) { + request.set_path(TStringType{settings.Path_}); + } + + auto requestLocation = request.mutable_location(); + const auto& location = settings.Location_; + + if (location.DataCenter) { + requestLocation->set_data_center(TStringType{location.DataCenter.value()}); + } + if (location.Module) { + requestLocation->set_module(TStringType{location.Module.value()}); + } + if (location.Rack) { + requestLocation->set_rack(TStringType{location.Rack.value()}); + } + if (location.Unit) { + requestLocation->set_unit(TStringType{location.Unit.value()}); + } + + if (location.DataCenterNum) { + requestLocation->set_data_center_num(location.DataCenterNum.value()); + } + if (location.RoomNum) { + requestLocation->set_room_num(location.RoomNum.value()); + } + if (location.RackNum) { + requestLocation->set_rack_num(location.RackNum.value()); + } + if (location.BodyNum) { + requestLocation->set_body_num(location.BodyNum.value()); + } + if (location.Body) { + requestLocation->set_body(location.Body.value()); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::NodeRegistrationResult result; + if (any) { + any->UnpackTo(&result); + } + TNodeRegistrationResult val{TStatus(std::move(status)), result}; + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncNodeRegistration, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } +}; + +TDiscoveryClient::TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ } + +TAsyncListEndpointsResult TDiscoveryClient::ListEndpoints(const TListEndpointsSettings& settings) { + return Impl_->ListEndpoints(settings); +} + +TAsyncWhoAmIResult TDiscoveryClient::WhoAmI(const TWhoAmISettings& settings) { + return Impl_->WhoAmI(settings); +} + +TAsyncNodeRegistrationResult TDiscoveryClient::NodeRegistration(const TNodeRegistrationSettings& settings) { + return Impl_->NodeRegistration(settings); +} + +} // namespace NDiscovery +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/discovery/ya.make b/ydb/public/sdk/cpp/src/client/discovery/ya.make new file mode 100644 index 000000000000..ae4ef332641b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/discovery/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + discovery.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/draft/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/draft/CMakeLists.txt new file mode 100644 index 000000000000..988b7706ec9d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/CMakeLists.txt @@ -0,0 +1,25 @@ +_ydb_sdk_add_library(client-draft) + +target_link_libraries(client-draft PUBLIC + yutil + yql-public-issue + api-grpc-draft + client-ydb_table + client-ydb_types-operation + client-ydb_value +) + +target_sources(client-draft PRIVATE + ydb_dynamic_config.cpp + ydb_replication.cpp + ydb_scripting.cpp + ydb_view.cpp +) + +generate_enum_serilization(client-draft + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/draft/ydb_replication.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/draft/ydb_replication.h +) + +_ydb_sdk_make_client_component(Draft client-draft) diff --git a/ydb/public/sdk/cpp/src/client/draft/ya.make b/ydb/public/sdk/cpp/src/client/draft/ya.make new file mode 100644 index 000000000000..fd341620c839 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + ydb_dynamic_config.cpp + ydb_replication.cpp + ydb_scripting.cpp + ydb_view.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h) + +PEERDIR( + ydb/public/sdk/cpp/src/library/issue + ydb/public/api/grpc/draft + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/types/operation + ydb/public/sdk/cpp/src/client/value +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp new file mode 100644 index 000000000000..ddcbcaed3850 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp @@ -0,0 +1,434 @@ +#include + +#include +#include +#include + +namespace NYdb::inline V3::NDynamicConfig { + +class TDynamicConfigClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections) + : TClientImplCommon(std::move(connections), TDynamicConfigClientSettings{}) + { + } + + TAsyncStatus SetConfig(const std::string& config, bool dryRun, bool allowUnknownFields, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + request.set_dry_run(dryRun); + request.set_allow_unknown_fields(allowUnknownFields); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncSetConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ReplaceConfig(const std::string& config, bool dryRun, bool allowUnknownFields, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + request.set_dry_run(dryRun); + request.set_allow_unknown_fields(allowUnknownFields); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncReplaceConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus DropConfig(const std::string& cluster, uint64_t version, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + request.mutable_identity()->set_cluster(TStringType{cluster}); + request.mutable_identity()->set_version(version); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncDropConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AddVolatileConfig(const std::string& config, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncAddVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveVolatileConfig(const std::string& cluster, uint64_t version, const std::vector& ids, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + request.mutable_identity()->set_cluster(TStringType{cluster}); + request.mutable_identity()->set_version(version); + + for (auto& id: ids) { + request.mutable_ids()->add_ids(id); + } + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveAllVolatileConfigs(const std::string& cluster, uint64_t version, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + request.mutable_identity()->set_cluster(TStringType{cluster}); + request.mutable_identity()->set_version(version); + request.set_all(true); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ForceRemoveVolatileConfig(const std::vector& ids, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + for (auto& id: ids) { + request.mutable_ids()->add_ids(id); + } + + request.set_force(true); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ForceRemoveAllVolatileConfigs(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + request.set_all(true); + request.set_force(true); + + return RunSimple( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + + TAsyncGetNodeLabelsResult GetNodeLabels(uint64_t nodeId, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_node_id(nodeId); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::map labels; + if (Ydb::DynamicConfig::GetNodeLabelsResult result; any && any->UnpackTo(&result)) { + for (auto& label : result.labels()) { + labels[label.label()] = label.value(); + } + } + + TGetNodeLabelsResult val(TStatus(std::move(status)), std::move(labels)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetNodeLabels, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string clusterName = ""; + uint64_t version = 0; + std::string config; + std::map volatileConfigs; + if (Ydb::DynamicConfig::GetConfigResult result; any && any->UnpackTo(&result)) { + clusterName = result.identity().cluster(); + version = result.identity().version(); + config = result.config(); + for (const auto& config : result.volatile_configs()) { + volatileConfigs.emplace(config.id(), config.config()); + } + } + + TGetConfigResult val(TStatus(std::move(status)), std::move(clusterName), version, std::move(config), std::move(volatileConfigs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncGetMetadataResult GetMetadata(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string metadata; + std::map volatileConfigs; + if (Ydb::DynamicConfig::GetMetadataResult result; any && any->UnpackTo(&result)) { + metadata = result.metadata(); + for (const auto& config : result.volatile_configs()) { + volatileConfigs.emplace(config.id(), config.metadata()); + } + } + + TGetMetadataResult val(TStatus(std::move(status)),std::move(metadata), std::move(volatileConfigs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetMetadata, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncResolveConfigResult ResolveConfig(const std::string& config, const std::map& volatileConfigs, const std::map& labels, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(TStringType{volatileConfig}); + } + for (auto& [name, value] : labels) { + auto* proto = request.add_labels(); + proto->set_label(TStringType{name}); + proto->set_value(TStringType{value}); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string config; + if (Ydb::DynamicConfig::ResolveConfigResult result; any && any->UnpackTo(&result)) { + config = result.config(); + } + + TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncResolveConfigResult ResolveConfig(const std::string& config, const std::map& volatileConfigs, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(TStringType{volatileConfig}); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string config; + if (Ydb::DynamicConfig::ResolveAllConfigResult result; any && any->UnpackTo(&result)) { + config = result.config(); + } + + TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveAllConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncVerboseResolveConfigResult VerboseResolveConfig(const std::string& config, const std::map& volatileConfigs, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + request.set_config(TStringType{config}); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(TStringType{volatileConfig}); + } + request.set_verbose_response(true); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + auto convert = [] (const Ydb::DynamicConfig::YamlLabelExt::LabelType& label) -> TVerboseResolveConfigResult::TLabel::EType { + switch(label) { + case Ydb::DynamicConfig::YamlLabelExt::NOT_SET: + return TVerboseResolveConfigResult::TLabel::EType::Negative; + case Ydb::DynamicConfig::YamlLabelExt::COMMON: + return TVerboseResolveConfigResult::TLabel::EType::Common; + case Ydb::DynamicConfig::YamlLabelExt::EMPTY: + return TVerboseResolveConfigResult::TLabel::EType::Empty; + default: + Y_ABORT("unexpected enum value"); + } + }; + + std::set labels; + TVerboseResolveConfigResult::ConfigByLabelSet configs; + + if (Ydb::DynamicConfig::ResolveAllConfigResult result; any && any->UnpackTo(&result)) { + for (auto& config : result.configs()) { + std::set> labelSets; + for (auto& labelSet : config.label_sets()) { + std::vector set; + for (auto& label : labelSet.labels()) { + labels.insert(label.label()); + set.push_back(TVerboseResolveConfigResult::TLabel{convert(label.type()), label.value()}); + } + labelSets.insert(set); + } + configs[labelSets] = config.config(); + } + } + + TVerboseResolveConfigResult val(TStatus(std::move(status)), std::move(labels), std::move(configs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveAllConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } +}; + +TDynamicConfigClient::TDynamicConfigClient(const TDriver& driver) + : Impl_(new TDynamicConfigClient::TImpl(CreateInternalInterface(driver))) +{} + +TAsyncStatus TDynamicConfigClient::SetConfig( + const std::string& config, + bool dryRun, + bool allowUnknownFields, + const TClusterConfigSettings& settings) { + return Impl_->SetConfig(config, dryRun, allowUnknownFields, settings); +} + +TAsyncStatus TDynamicConfigClient::ReplaceConfig( + const std::string& config, + bool dryRun, + bool allowUnknownFields, + const TClusterConfigSettings& settings) { + return Impl_->ReplaceConfig(config, dryRun, allowUnknownFields, settings); +} + +TAsyncStatus TDynamicConfigClient::DropConfig( + const std::string& cluster, + uint64_t version, + const TClusterConfigSettings& settings) { + return Impl_->DropConfig(cluster, version, settings); +} + +TAsyncStatus TDynamicConfigClient::AddVolatileConfig( + const std::string& config, + const TClusterConfigSettings& settings) { + return Impl_->AddVolatileConfig(config, settings); +} + +TAsyncStatus TDynamicConfigClient::RemoveVolatileConfig( + const std::string& cluster, + uint64_t version, + const std::vector& ids, + const TClusterConfigSettings& settings) { + return Impl_->RemoveVolatileConfig(cluster, version, ids, settings); +} + +TAsyncStatus TDynamicConfigClient::RemoveAllVolatileConfigs( + const std::string& cluster, + uint64_t version, + const TClusterConfigSettings& settings) { + return Impl_->RemoveAllVolatileConfigs(cluster, version, settings); +} + +TAsyncStatus TDynamicConfigClient::ForceRemoveVolatileConfig( + const std::vector& ids, + const TClusterConfigSettings& settings) { + return Impl_->ForceRemoveVolatileConfig(ids, settings); +} + +TAsyncStatus TDynamicConfigClient::ForceRemoveAllVolatileConfigs( + const TClusterConfigSettings& settings) { + return Impl_->ForceRemoveAllVolatileConfigs(settings); +} + +TAsyncGetMetadataResult TDynamicConfigClient::GetMetadata(const TClusterConfigSettings& settings) { + return Impl_->GetMetadata(settings); +} + +TAsyncGetConfigResult TDynamicConfigClient::GetConfig(const TClusterConfigSettings& settings) { + return Impl_->GetConfig(settings); +} + +TAsyncGetNodeLabelsResult TDynamicConfigClient::GetNodeLabels(uint64_t nodeId, const TClusterConfigSettings& settings) { + return Impl_->GetNodeLabels(nodeId, settings); +} + +TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const std::map& labels, + const TClusterConfigSettings& settings) { + return Impl_->ResolveConfig(config, volatileConfigs, labels, settings); +} + +TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const TClusterConfigSettings& settings) { + return Impl_->ResolveConfig(config, volatileConfigs, settings); +} + +TAsyncVerboseResolveConfigResult TDynamicConfigClient::VerboseResolveConfig( + const std::string& config, + const std::map& volatileConfigs, + const TClusterConfigSettings& settings) { + return Impl_->VerboseResolveConfig(config, volatileConfigs, settings); +} + +} // namespace NYdb::V3::NDynamicConfig diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_replication.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_replication.cpp new file mode 100644 index 000000000000..ba3efab37e26 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_replication.cpp @@ -0,0 +1,274 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3 { +namespace NReplication { + +TConnectionParams::TConnectionParams(const Ydb::Replication::ConnectionParams& params) { + DiscoveryEndpoint(params.endpoint()); + Database(params.database()); + SslCredentials(params.enable_ssl()); + + switch (params.credentials_case()) { + case Ydb::Replication::ConnectionParams::kStaticCredentials: + Credentials_ = TStaticCredentials{ + .User = params.static_credentials().user(), + .PasswordSecretName = params.static_credentials().password_secret_name(), + }; + break; + + case Ydb::Replication::ConnectionParams::kOauth: + Credentials_ = TOAuthCredentials{ + .TokenSecretName = params.oauth().token_secret_name(), + }; + break; + + default: + break; + } +} + +const std::string& TConnectionParams::GetDiscoveryEndpoint() const { + return *DiscoveryEndpoint_; +} + +const std::string& TConnectionParams::GetDatabase() const { + return *Database_; +} + +bool TConnectionParams::GetEnableSsl() const { + return SslCredentials_->IsEnabled; +} + +TConnectionParams::ECredentials TConnectionParams::GetCredentials() const { + return static_cast(Credentials_.index()); +} + +const TStaticCredentials& TConnectionParams::GetStaticCredentials() const { + return std::get(Credentials_); +} + +const TOAuthCredentials& TConnectionParams::GetOAuthCredentials() const { + return std::get(Credentials_); +} + +static TDuration DurationToDuration(const google::protobuf::Duration& value) { + return TDuration::MilliSeconds(google::protobuf::util::TimeUtil::DurationToMilliseconds(value)); +} + +TGlobalConsistency::TGlobalConsistency(const Ydb::Replication::ConsistencyLevelGlobal& proto) + : CommitInterval_(DurationToDuration(proto.commit_interval())) +{ +} + +const TDuration& TGlobalConsistency::GetCommitInterval() const { + return CommitInterval_; +} + +TStats::TStats(const Ydb::Replication::DescribeReplicationResult_Stats& stats) + : Lag_(stats.has_lag() ? std::make_optional(DurationToDuration(stats.lag())) : std::nullopt) + , InitialScanProgress_(stats.has_initial_scan_progress() ? std::make_optional(stats.initial_scan_progress()) : std::nullopt) +{ +} + +const std::optional& TStats::GetLag() const { + return Lag_; +} + +const std::optional& TStats::GetInitialScanProgress() const { + return InitialScanProgress_; +} + +TRunningState::TRunningState(const TStats& stats) + : Stats_(stats) +{ +} + +const TStats& TRunningState::GetStats() const { + return Stats_; +} + +class TErrorState::TImpl { +public: + NYdb::NIssue::TIssues Issues; + + explicit TImpl(NYdb::NIssue::TIssues&& issues) + : Issues(std::move(issues)) + { + } +}; + +TErrorState::TErrorState(NYdb::NIssue::TIssues&& issues) + : Impl_(std::make_shared(std::move(issues))) +{ +} + +const NYdb::NIssue::TIssues& TErrorState::GetIssues() const { + return Impl_->Issues; +} + +NYdb::NIssue::TIssues IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message) { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(message, issues); + return issues; +} + +TReplicationDescription::TReplicationDescription(const Ydb::Replication::DescribeReplicationResult& desc) + : ConnectionParams_(desc.connection_params()) +{ + Items_.reserve(desc.items_size()); + for (const auto& item : desc.items()) { + Items_.push_back(TItem{ + .Id = item.id(), + .SrcPath = item.source_path(), + .DstPath = item.destination_path(), + .Stats = TStats(item.stats()), + .SrcChangefeedName = item.has_source_changefeed_name() + ? std::make_optional(item.source_changefeed_name()) : std::nullopt, + }); + } + + switch (desc.consistency_level_case()) { + case Ydb::Replication::DescribeReplicationResult::kGlobalConsistency: + ConsistencyLevel_ = TGlobalConsistency(desc.global_consistency()); + break; + + default: + break; + } + + switch (desc.state_case()) { + case Ydb::Replication::DescribeReplicationResult::kRunning: + State_ = TRunningState(desc.running().stats()); + break; + + case Ydb::Replication::DescribeReplicationResult::kError: + State_ = TErrorState(IssuesFromMessage(desc.error().issues())); + break; + + case Ydb::Replication::DescribeReplicationResult::kDone: + State_ = TDoneState(); + break; + + default: + break; + } +} + +const TConnectionParams& TReplicationDescription::GetConnectionParams() const { + return ConnectionParams_; +} + +const std::vector TReplicationDescription::GetItems() const { + return Items_; +} + +TReplicationDescription::EConsistencyLevel TReplicationDescription::GetConsistencyLevel() const { + return static_cast(ConsistencyLevel_.index()); +} + +const TGlobalConsistency& TReplicationDescription::GetGlobalConsistency() const { + return std::get(ConsistencyLevel_); +} + + +TReplicationDescription::EState TReplicationDescription::GetState() const { + return static_cast(State_.index()); +} + +const TRunningState& TReplicationDescription::GetRunningState() const { + return std::get(State_); +} + +const TErrorState& TReplicationDescription::GetErrorState() const { + return std::get(State_); +} + +const TDoneState& TReplicationDescription::GetDoneState() const { + return std::get(State_); +} + +TDescribeReplicationResult::TDescribeReplicationResult(TStatus&& status, Ydb::Replication::DescribeReplicationResult&& desc) + : NScheme::TDescribePathResult(std::move(status), desc.self()) + , ReplicationDescription_(desc) + , Proto_(std::make_unique()) +{ + *Proto_ = std::move(desc); +} + +const TReplicationDescription& TDescribeReplicationResult::GetReplicationDescription() const { + return ReplicationDescription_; +} + +const Ydb::Replication::DescribeReplicationResult& TDescribeReplicationResult::GetProto() const { + return *Proto_; +} + +class TReplicationClient::TImpl: public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncDescribeReplicationResult DescribeReplication(const std::string& path, const TDescribeReplicationSettings& settings) { + using namespace Ydb::Replication; + + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + request.set_include_stats(settings.IncludeStats_); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + DescribeReplicationResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeReplicationResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &V1::ReplicationService::Stub::AsyncDescribeReplication, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + +}; + +TReplicationClient::TReplicationClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncDescribeReplicationResult TReplicationClient::DescribeReplication(const std::string& path, const TDescribeReplicationSettings& settings) { + return Impl_->DescribeReplication(path, settings); +} + +} // NReplication + +const Ydb::Replication::DescribeReplicationResult& TProtoAccessor::GetProto(const NReplication::TDescribeReplicationResult& result) { + return result.GetProto(); +} + +} // NYdb diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_scripting.cpp new file mode 100644 index 000000000000..b364a57a69ec --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_scripting.cpp @@ -0,0 +1,383 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NScripting { + +using namespace NThreading; + +TExecuteYqlResult::TExecuteYqlResult(TStatus&& status, std::vector&& resultSets, + const std::optional& queryStats) + : TStatus(std::move(status)) + , ResultSets_(std::move(resultSets)) + , QueryStats_(queryStats) {} + +const std::vector& TExecuteYqlResult::GetResultSets() const { + return ResultSets_; +} + +TResultSet TExecuteYqlResult::GetResultSet(size_t resultIndex) const { + if (resultIndex >= ResultSets_.size()) { + RaiseError(std::string("Requested index out of range\n")); + } + + return ResultSets_[resultIndex]; +} + +TResultSetParser TExecuteYqlResult::GetResultSetParser(size_t resultIndex) const { + return TResultSetParser(GetResultSet(resultIndex)); +} + +const std::optional& TExecuteYqlResult::GetStats() const { + return QueryStats_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TYqlResultPartIterator::TReaderImpl { +public: + using TSelf = TYqlResultPartIterator::TReaderImpl; + using TResponse = Ydb::Scripting::ExecuteYqlPartialResponse; + using TStreamProcessorPtr = NYdbGrpc::IStreamRequestReadProcessor::TPtr; + using TReadCallback = NYdbGrpc::IStreamRequestReadProcessor::TReadCallback; + using TGRpcStatus = NYdbGrpc::TGrpcStatus; + + TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint) + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) + {} + + ~TReaderImpl() { + StreamProcessor_->Cancel(); + } + + bool IsFinished() const { + return Finished_; + } + + TAsyncYqlResultPart ReadNext(std::shared_ptr self) { + auto promise = NThreading::NewPromise(); + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + if (!grpcStatus.Ok()) { + self->Finished_ = true; + promise.SetValue({ TStatus(TPlainStatus(grpcStatus, self->Endpoint_)) }); + } else { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast(self->Response_.status()); + TPlainStatus plainStatus{ clientStatus, std::move(issues), self->Endpoint_, {} }; + TStatus status{ std::move(plainStatus) }; + std::optional queryStats; + + if (self->Response_.result().has_query_stats()) { + queryStats = NTable::TQueryStats(self->Response_.result().query_stats()); + } + if (self->Response_.result().has_result_set()) { + promise.SetValue( + { + std::move(status), + TYqlPartialResult( + self->Response_.result().result_set_index(), + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())) + ), + queryStats + } + ); + } else { + promise.SetValue({ std::move(status), queryStats }); + } + } + }; + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); + } +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + std::string Endpoint_; +}; + +TYqlResultPartIterator::TYqlResultPartIterator( + std::shared_ptr impl, + TPlainStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) +{} + +TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() { + if (ReaderImpl_->IsFinished()) + RaiseError("Attempt to perform read on invalid or finished stream"); + return ReaderImpl_->ReadNext(ReaderImpl_); +} + +//////////////////////////////////////////////////////////////////////////////// + +TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan) + : TStatus(std::move(status)) + , ParameterTypes_(std::move(types)) + , Plan_(plan) {} + +std::map TExplainYqlResult::GetParameterTypes() const { + std::map typesMap; + for (const auto& param : ParameterTypes_) { + typesMap.emplace(param.first, TType(param.second)); + } + return typesMap; +} + +const std::string& TExplainYqlResult::GetPlan() const { + return Plan_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TScriptingClient::TImpl : public TClientImplCommon { +public: + using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr; + + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) {} + + template + TAsyncExecuteYqlResult ExecuteYqlScript(const std::string& script, TParamsType params, + const TExecuteYqlRequestSettings& settings) + { + auto request = MakeOperationRequest(settings); + request.set_script(TStringType{script}); + SetParams(params, &request); + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + request.set_syntax(settings.Syntax_); + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector res; + std::optional queryStats; + if (any) { + Ydb::Scripting::ExecuteYqlResult result; + any->UnpackTo(&result); + + for (size_t i = 0; i < static_cast(result.result_sets_size()); i++) { + res.push_back(TResultSet(*result.mutable_result_sets(i))); + } + + if (result.has_query_stats()) { + queryStats = NTable::TQueryStats(result.query_stats()); + } + } + + TExecuteYqlResult executeResult(TStatus(std::move(status)), std::move(res), + queryStats); + promise.SetValue(std::move(executeResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExecuteYql, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + template + TFuture> StreamExecuteYqlScriptInternal(const std::string& script, + TParamsType params, const TExecuteYqlRequestSettings& settings) + { + auto request = MakeOperationRequest(settings); + request.set_script(TStringType{script}); + SetParams(params, &request); + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + request.set_syntax(settings.Syntax_); + + auto promise = NewPromise>(); + + Connections_->StartReadStream< + Ydb::Scripting::V1::ScriptingService, + Ydb::Scripting::ExecuteYqlRequest, + Ydb::Scripting::ExecuteYqlPartialResponse> + ( + std::move(request), + [promise](TPlainStatus status, TYqlScriptProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Scripting::V1::ScriptingService::Stub::AsyncStreamExecuteYql, + DbDriverState_, + TRpcRequestSettings::Make(settings) + ); + + return promise.GetFuture(); + } + + template + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const std::string& query, TParamsType params, + const TExecuteYqlRequestSettings& settings) + { + auto promise = NewPromise(); + + auto iteratorCallback = [promise](TFuture> future) mutable + { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TYqlResultPartIterator( + pair.second + ? std::make_shared(pair.second, pair.first.Endpoint) + : nullptr, + std::move(pair.first))); + }; + + StreamExecuteYqlScriptInternal(query, params, settings).Subscribe(iteratorCallback); + return promise.GetFuture(); + } + + TAsyncExplainYqlResult ExplainYqlScript(const std::string& script, + const TExplainYqlRequestSettings& settings) + { + auto request = MakeOperationRequest(settings); + request.set_script(TStringType{script}); + + switch (settings.Mode_) { + // KIKIMR-10990 + //case ExplainYqlRequestMode::Parse: + // request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PARSE); + // break; + case ExplainYqlRequestMode::Validate: + request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE); + break; + case ExplainYqlRequestMode::Plan: + request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PLAN); + break; + } + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string plan; + ::google::protobuf::Map types; + if (any) { + Ydb::Scripting::ExplainYqlResult result; + any->UnpackTo(&result); + + plan = result.plan(); + types = result.parameters_types(); + } + + TExplainYqlResult explainResult(TStatus(std::move(status)), + std::move(types), std::move(plan)); + promise.SetValue(std::move(explainResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExplainYql, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + +private: + template + static void SetParams(::google::protobuf::Map* params, TRequest* request) { + if (params) { + request->mutable_parameters()->swap(*params); + } + } + + template + static void SetParams(const ::google::protobuf::Map& params, TRequest* request) { + *request->mutable_parameters() = params; + } + +}; + +TScriptingClient::TScriptingClient(const TDriver& driver, const TCommonClientSettings &settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{} + +TParamsBuilder TScriptingClient::GetParamsBuilder() { + return TParamsBuilder(); +} + +TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const std::string &query, NYdb::TParams&& params, + const TExecuteYqlRequestSettings &settings) +{ + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + return Impl_->ExecuteYqlScript(query, paramsPtr, settings); +} + +TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const std::string &query, const NYdb::TParams& params, + const TExecuteYqlRequestSettings &settings) +{ + if (params.Empty()) { + return Impl_->ExecuteYqlScript( + query, + nullptr, + settings); + } else { + using TProtoParamsType = const ::google::protobuf::Map; + return Impl_->ExecuteYqlScript( + query, + params.GetProtoMap(), + settings); + } +} + +TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const std::string &script, + const TExecuteYqlRequestSettings &settings) +{ + return Impl_->ExecuteYqlScript(script, nullptr, settings); +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const std::string& script, + const TExecuteYqlRequestSettings& settings) +{ + return Impl_->StreamExecuteYqlScript(script, nullptr, settings); +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const std::string& script, const TParams& params, + const TExecuteYqlRequestSettings& settings) +{ + if (params.Empty()) { + return Impl_->StreamExecuteYqlScript(script, nullptr, settings); + } else { + using TProtoParamsType = const ::google::protobuf::Map; + return Impl_->StreamExecuteYqlScript(script, params.GetProtoMap(), settings); + } +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const std::string& script, TParams&& params, + const TExecuteYqlRequestSettings& settings) +{ + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + return Impl_->StreamExecuteYqlScript(script, paramsPtr, settings); +} + +TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const std::string& script, + const TExplainYqlRequestSettings& settings) +{ + return Impl_->ExplainYqlScript(script, settings); +} + +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_view.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_view.cpp new file mode 100644 index 000000000000..df26deb58206 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_view.cpp @@ -0,0 +1,92 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NView { + +TViewDescription::TViewDescription(const Ydb::View::DescribeViewResult& desc) + : QueryText_(desc.query_text()) +{ +} + +const std::string& TViewDescription::GetQueryText() const { + return QueryText_; +} + +TDescribeViewResult::TDescribeViewResult(TStatus&& status, Ydb::View::DescribeViewResult&& desc) + : NScheme::TDescribePathResult(std::move(status), desc.self()) + , Proto_(std::make_unique(std::move(desc))) +{ +} + +TViewDescription TDescribeViewResult::GetViewDescription() const { + return TViewDescription(*Proto_); +} + +const Ydb::View::DescribeViewResult& TDescribeViewResult::GetProto() const { + return *Proto_; +} + +class TViewClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncDescribeViewResult DescribeView(const std::string& path, const TDescribeViewSettings& settings) { + using namespace Ydb::View; + + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + DescribeViewResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeViewResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &V1::ViewService::Stub::AsyncDescribeView, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + +}; + +TViewClient::TViewClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncDescribeViewResult TViewClient::DescribeView(const std::string& path, const TDescribeViewSettings& settings) { + return Impl_->DescribeView(path, settings); +} + +} // NView + +const Ydb::View::DescribeViewResult& TProtoAccessor::GetProto(const NView::TDescribeViewResult& result) { + return result.GetProto(); +} + +} // NYdb diff --git a/ydb/public/sdk/cpp/src/client/driver/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/driver/CMakeLists.txt new file mode 100644 index 000000000000..db487c67c6bf --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/driver/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-ydb_driver) + +target_link_libraries(client-ydb_driver + PUBLIC + client-ydb_common_client + client-ydb_types-status + PRIVATE + impl-ydb_internal-common + impl-ydb_internal-grpc_connections + client-resources +) + +target_sources(client-ydb_driver PRIVATE + driver.cpp +) + +_ydb_sdk_make_client_component(Driver client-ydb_driver) diff --git a/ydb/public/sdk/cpp/src/client/driver/driver.cpp b/ydb/public/sdk/cpp/src/client/driver/driver.cpp new file mode 100644 index 000000000000..af4e96a2779b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/driver/driver.cpp @@ -0,0 +1,226 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include +#include +#include + +namespace NYdb::inline V3 { + +using NYdbGrpc::TGRpcClientLow; +using NYdbGrpc::TServiceConnection; +using NYdbGrpc::TSimpleRequestProcessor; +using NYdbGrpc::TGRpcClientConfig; +using NYdbGrpc::TResponseCallback; +using NYdbGrpc::TGrpcStatus; +using NYdbGrpc::TTcpKeepAliveSettings; + +using Ydb::StatusIds; + +using namespace NThreading; + +class TDriverConfig::TImpl : public IConnectionsParams { +public: + std::string GetEndpoint() const override { return Endpoint; } + size_t GetNetworkThreadsNum() const override { return NetworkThreadsNum; } + size_t GetClientThreadsNum() const override { return ClientThreadsNum; } + size_t GetMaxQueuedResponses() const override { return MaxQueuedResponses; } + TSslCredentials GetSslCredentials() const override { return SslCredentials; } + std::string GetDatabase() const override { return Database; } + std::shared_ptr GetCredentialsProviderFactory() const override { return CredentialsProviderFactory; } + EDiscoveryMode GetDiscoveryMode() const override { return DiscoveryMode; } + size_t GetMaxQueuedRequests() const override { return MaxQueuedRequests; } + TTcpKeepAliveSettings GetTcpKeepAliveSettings() const override { return TcpKeepAliveSettings; } + bool GetDrinOnDtors() const override { return DrainOnDtors; } + TBalancingSettings GetBalancingSettings() const override { return BalancingSettings; } + TDuration GetGRpcKeepAliveTimeout() const override { return GRpcKeepAliveTimeout; } + bool GetGRpcKeepAlivePermitWithoutCalls() const override { return GRpcKeepAlivePermitWithoutCalls; } + TDuration GetSocketIdleTimeout() const override { return SocketIdleTimeout; } + uint64_t GetMemoryQuota() const override { return MemoryQuota; } + uint64_t GetMaxInboundMessageSize() const override { return MaxInboundMessageSize; } + uint64_t GetMaxOutboundMessageSize() const override { return MaxOutboundMessageSize; } + uint64_t GetMaxMessageSize() const override { return MaxMessageSize; } + const TLog& GetLog() const override { return Log; } + + std::string Endpoint; + size_t NetworkThreadsNum = 2; + size_t ClientThreadsNum = 0; + size_t MaxQueuedResponses = 0; + TSslCredentials SslCredentials; + std::string Database; + std::shared_ptr CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); + EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; + size_t MaxQueuedRequests = 100; + NYdbGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings = + { + true, + TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_COUNT, + TCP_KEEPALIVE_INTERVAL + }; + bool DrainOnDtors = true; + TBalancingSettings BalancingSettings = TBalancingSettings{EBalancingPolicy::UsePreferableLocation, std::string()}; + TDuration GRpcKeepAliveTimeout; + bool GRpcKeepAlivePermitWithoutCalls = false; + TDuration SocketIdleTimeout = TDuration::Minutes(6); + uint64_t MemoryQuota = 0; + uint64_t MaxInboundMessageSize = 0; + uint64_t MaxOutboundMessageSize = 0; + uint64_t MaxMessageSize = 0; + TLog Log; // Null by default. +}; + +TDriverConfig::TDriverConfig(const std::string& connectionString) + : Impl_(new TImpl) { + if (connectionString != ""){ + auto connectionInfo = ParseConnectionString(connectionString); + SetEndpoint(connectionInfo.Endpoint); + SetDatabase(connectionInfo.Database); + Impl_->SslCredentials.IsEnabled = connectionInfo.EnableSsl; + } +} + +TDriverConfig& TDriverConfig::SetEndpoint(const std::string& endpoint) { + Impl_->Endpoint = endpoint; + return *this; +} + +TDriverConfig& TDriverConfig::SetNetworkThreadsNum(size_t sz) { + Impl_->NetworkThreadsNum = sz; + return *this; +} + +TDriverConfig& TDriverConfig::SetClientThreadsNum(size_t sz) { + Impl_->ClientThreadsNum = sz; + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxClientQueueSize(size_t sz) { + Impl_->MaxQueuedResponses = sz; + return *this; +} + +TDriverConfig& TDriverConfig::UseSecureConnection(const std::string& cert) { + Impl_->SslCredentials.IsEnabled = true; + Impl_->SslCredentials.CaCert = cert; + return *this; +} + +TDriverConfig& TDriverConfig::UseClientCertificate(const std::string& clientCert, const std::string& clientPrivateKey) { + Impl_->SslCredentials.Cert = clientCert; + Impl_->SslCredentials.PrivateKey = clientPrivateKey; + return *this; +} + +TDriverConfig& TDriverConfig::SetAuthToken(const std::string& token) { + return SetCredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token)); +} + +TDriverConfig& TDriverConfig::SetDatabase(const std::string& database) { + Impl_->Database = database; + Impl_->Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Impl_->Database))); + return *this; +} + +TDriverConfig& TDriverConfig::SetCredentialsProviderFactory(std::shared_ptr credentialsProviderFactory) { + Impl_->CredentialsProviderFactory = credentialsProviderFactory; + return *this; +} + +TDriverConfig& TDriverConfig::SetDiscoveryMode(EDiscoveryMode discoveryMode) { + Impl_->DiscoveryMode = discoveryMode; + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxQueuedRequests(size_t sz) { + Impl_->MaxQueuedRequests = sz; + return *this; +} + +TDriverConfig& TDriverConfig::SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval) { + Impl_->TcpKeepAliveSettings.Enabled = enable; + Impl_->TcpKeepAliveSettings.Idle = idle; + Impl_->TcpKeepAliveSettings.Count = count; + Impl_->TcpKeepAliveSettings.Interval = interval; + return *this; +} + +TDriverConfig& TDriverConfig::SetGrpcMemoryQuota(uint64_t bytes) { + Impl_->MemoryQuota = bytes; + return *this; +} + +TDriverConfig& TDriverConfig::SetDrainOnDtors(bool allowed) { + Impl_->DrainOnDtors = allowed; + return *this; +} + +TDriverConfig& TDriverConfig::SetBalancingPolicy(EBalancingPolicy policy, const std::string& params) { + Impl_->BalancingSettings = TBalancingSettings{policy, params}; + return *this; +} + +TDriverConfig& TDriverConfig::SetGRpcKeepAliveTimeout(TDuration timeout) { + Impl_->GRpcKeepAliveTimeout = timeout; + return *this; +} + +TDriverConfig& TDriverConfig::SetGRpcKeepAlivePermitWithoutCalls(bool permitWithoutCalls) { + Impl_->GRpcKeepAlivePermitWithoutCalls = permitWithoutCalls; + return *this; +} + +TDriverConfig& TDriverConfig::SetSocketIdleTimeout(TDuration timeout) { + Impl_->SocketIdleTimeout = timeout; + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxInboundMessageSize(uint64_t maxInboundMessageSize) { + Impl_->MaxInboundMessageSize = maxInboundMessageSize; + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxOutboundMessageSize(uint64_t maxOutboundMessageSize) { + Impl_->MaxOutboundMessageSize = maxOutboundMessageSize; + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxMessageSize(uint64_t maxMessageSize) { + Impl_->MaxMessageSize = maxMessageSize; + return *this; +} + +TDriverConfig& TDriverConfig::SetLog(std::unique_ptr&& log) { + Impl_->Log.ResetBackend(THolder(log.release())); + return *this; +} + +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr CreateInternalInterface(const TDriver connection) { + return connection.Impl_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TDriver::TDriver(const TDriverConfig& config) { + if (!config.Impl_) { + ythrow yexception() << "Invalid config object"; + } + + Impl_.reset(new TGRpcConnectionsImpl(config.Impl_)); +} + +void TDriver::Stop(bool wait) { + Impl_->Stop(wait); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/driver/ya.make b/ydb/public/sdk/cpp/src/client/driver/ya.make new file mode 100644 index 000000000000..e5a28314bd55 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/driver/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + driver.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/common + ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections + ydb/public/sdk/cpp/src/client/resources + ydb/public/sdk/cpp/src/client/common_client + ydb/public/sdk/cpp/src/client/types/status +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/export/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/export/CMakeLists.txt new file mode 100644 index 000000000000..7ae0ee465ed3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/export/CMakeLists.txt @@ -0,0 +1,25 @@ +_ydb_sdk_add_library(client-ydb_export) + +target_link_libraries(client-ydb_export PUBLIC + yutil + enum_serialization_runtime + api-grpc + api-protos + client-ydb_common_client-impl + client-ydb_driver + client-ydb_proto + client-ydb_types-operation +) + +target_sources(client-ydb_export PRIVATE + export.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_export + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/export/export.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/export/export.h +) + +_ydb_sdk_make_client_component(Export client-ydb_export) diff --git a/ydb/public/sdk/cpp/src/client/export/export.cpp b/ydb/public/sdk/cpp/src/client/export/export.cpp new file mode 100644 index 000000000000..e3d21a977f44 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/export/export.cpp @@ -0,0 +1,203 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NExport { + +using namespace NThreading; +using namespace Ydb::Export; + +/// Common +namespace { + +std::vector ItemsProgressFromProto(const google::protobuf::RepeatedPtrField& proto) { + std::vector result(proto.size()); + + for (const auto& protoItem : proto) { + auto& item = result.emplace_back(); + item.PartsTotal = protoItem.parts_total(); + item.PartsCompleted = protoItem.parts_completed(); + item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); + item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); + } + + return result; +} + +} // anonymous + +/// YT +TExportToYtResponse::TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ExportToYtMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Host(metadata.settings().host()); + Metadata_.Settings.Port(metadata.settings().port()); + Metadata_.Settings.Token(metadata.settings().token()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + Metadata_.Settings.UseTypeV3(metadata.settings().use_type_v3()); + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TExportToYtResponse::TMetadata& TExportToYtResponse::Metadata() const { + return Metadata_; +} + +/// S3 +TExportToS3Response::TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ExportToS3Metadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Endpoint(metadata.settings().endpoint()); + Metadata_.Settings.Scheme(TProtoAccessor::FromProto(metadata.settings().scheme())); + Metadata_.Settings.StorageClass(TProtoAccessor::FromProto(metadata.settings().storage_class())); + Metadata_.Settings.Bucket(metadata.settings().bucket()); + Metadata_.Settings.AccessKey(metadata.settings().access_key()); + Metadata_.Settings.SecretKey(metadata.settings().secret_key()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_prefix()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + if (!metadata.settings().compression().empty()) { + Metadata_.Settings.Compression(metadata.settings().compression()); + } + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TExportToS3Response::TMetadata& TExportToS3Response::Metadata() const { + return Metadata_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TExportClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TFuture ExportToYt(ExportToYtRequest&& request, + const TExportToYtSettings& settings) + { + return RunOperation( + std::move(request), + &V1::ExportService::Stub::AsyncExportToYt, + TRpcRequestSettings::Make(settings)); + } + + TFuture ExportToS3(ExportToS3Request&& request, + const TExportToS3Settings& settings) + { + return RunOperation( + std::move(request), + &V1::ExportService::Stub::AsyncExportToS3, + TRpcRequestSettings::Make(settings)); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TExportClient::TExportClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +TFuture TExportClient::ExportToYt(const TExportToYtSettings& settings) { + auto request = MakeOperationRequest(settings); + + request.mutable_settings()->set_host(TStringType{settings.Host_}); + request.mutable_settings()->set_port(settings.Port_.value_or(80)); + request.mutable_settings()->set_token(TStringType{settings.Token_}); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_path(TStringType{item.Src}); + protoItem.set_destination_path(TStringType{item.Dst}); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.value()); + } + + request.mutable_settings()->set_use_type_v3(settings.UseTypeV3_); + + return Impl_->ExportToYt(std::move(request), settings); +} + +TFuture TExportClient::ExportToS3(const TExportToS3Settings& settings) { + auto request = MakeOperationRequest(settings); + + request.mutable_settings()->set_endpoint(TStringType{settings.Endpoint_}); + request.mutable_settings()->set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); + request.mutable_settings()->set_storage_class(TProtoAccessor::GetProto(settings.StorageClass_)); + request.mutable_settings()->set_bucket(TStringType{settings.Bucket_}); + request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); + request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_path(TStringType{item.Src}); + protoItem.set_destination_prefix(TStringType{item.Dst}); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.value()); + } + + if (settings.Compression_) { + request.mutable_settings()->set_compression(TStringType{settings.Compression_.value()}); + } + + request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); + + return Impl_->ExportToS3(std::move(request), settings); +} + +} // namespace NExport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/export/out.cpp b/ydb/public/sdk/cpp/src/client/export/out.cpp new file mode 100644 index 000000000000..d652be82f379 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/export/out.cpp @@ -0,0 +1,9 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::NExport::TExportToYtResponse, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NExport::TExportToS3Response, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/export/ya.make b/ydb/public/sdk/cpp/src/client/export/ya.make new file mode 100644 index 000000000000..9b6b816aafb4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/export/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + export.cpp + out.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/export/export.h) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/extension_common/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/extension_common/CMakeLists.txt new file mode 100644 index 000000000000..e099b35a031c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extension_common/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(client-extension_common) + +target_link_libraries(client-extension_common PUBLIC + yutil + monlib-metrics + client-ydb_driver +) + +target_sources(client-extension_common PRIVATE + extension.cpp +) + +_ydb_sdk_make_client_component(Extension client-extension_common) diff --git a/ydb/public/sdk/cpp/src/client/extension_common/extension.cpp b/ydb/public/sdk/cpp/src/client/extension_common/extension.cpp new file mode 100644 index 000000000000..5ad050e1d1f3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extension_common/extension.cpp @@ -0,0 +1,44 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +namespace NYdb::inline V3 { + +void IExtension::SelfRegister(TDriver driver) { + CreateInternalInterface(driver)->RegisterExtension(this); +} + +void IExtensionApi::SelfRegister(TDriver driver) { + CreateInternalInterface(driver)->RegisterExtensionApi(this); +} + +namespace NSdkStats { + +IStatApi* IStatApi::Create(TDriver driver) { + return new TStatsExtractor(CreateInternalInterface(driver)); +} + +} // namespace YSdkStats + +class TDiscoveryMutator : public IDiscoveryMutatorApi { +public: + TDiscoveryMutator(std::shared_ptr driverImpl) + : DriverImpl(driverImpl.get()) + { } + + void SetMutatorCb(TMutatorCb&& cb) override { + DriverImpl->SetDiscoveryMutator(std::move(cb)); + } +private: + TGRpcConnectionsImpl* DriverImpl; +}; + +IDiscoveryMutatorApi* IDiscoveryMutatorApi::Create(TDriver driver) { + return new TDiscoveryMutator(CreateInternalInterface(driver)); +} + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/src/client/extension_common/ya.make b/ydb/public/sdk/cpp/src/client/extension_common/ya.make new file mode 100644 index 000000000000..85e5847bf4f0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extension_common/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + extension.cpp +) + +PEERDIR( + library/cpp/monlib/metrics + ydb/public/sdk/cpp/src/client/driver +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/extensions/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/extensions/CMakeLists.txt new file mode 100644 index 000000000000..99e80bd68371 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(discovery_mutator) +add_subdirectory(solomon_stats) diff --git a/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/CMakeLists.txt new file mode 100644 index 000000000000..2a7553dd4557 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(client-extensions-discovery_mutator) + +target_link_libraries(client-extensions-discovery_mutator PUBLIC + yutil + client-extension_common +) + +target_sources(client-extensions-discovery_mutator PRIVATE + discovery_mutator.cpp +) + +_ydb_sdk_make_client_component(DiscoveryMutator client-extensions-discovery_mutator) diff --git a/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/discovery_mutator.cpp b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/discovery_mutator.cpp new file mode 100644 index 000000000000..8d97e695fae9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/discovery_mutator.cpp @@ -0,0 +1 @@ +#include diff --git a/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/ya.make b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/ya.make new file mode 100644 index 000000000000..ae92c1b37425 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/discovery_mutator/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + discovery_mutator.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/extension_common +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/CMakeLists.txt new file mode 100644 index 000000000000..af5e7d3c7a63 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-extensions-solomon_stats) + +target_link_libraries(client-extensions-solomon_stats PUBLIC + yutil + monlib-encode-json + monlib-metrics + monlib-service + monlib-service-pages + client-extension_common +) + +target_sources(client-extensions-solomon_stats PRIVATE + pull_client.cpp + pull_connector.cpp +) + +_ydb_sdk_make_client_component(SolomonStats client-extensions-solomon_stats) diff --git a/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/README.md b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/README.md new file mode 100644 index 000000000000..b4e2ebb17954 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/README.md @@ -0,0 +1,92 @@ +# YDB C++ SDK Metrics + +You can plug in YDB C++ SDK extension to monitor how your application interacts with YDB. In particular you can monitor: +- Transport errors; +- Per host messages in fligh; +- How database nodes discover works; +- Session pool size; +- Number of sessions per host; +- Number of server endpoints available; +- Query size, latency, etc. + +You can get these metrics via http server provided by YDB C++ SDK or implement your own MetricRegistry if it more convenient. + +## Setting up Solomon Monitoring + +> This is Yandex specific section for setting up internal monitoring called Solomon + +### Setup Solomon Environment +TSolomonStatPullExtension class allows you to quickly setup you application monitoring. You need to prepare a Solomon project that can accept your metrics beforehand. +[Create project, cluster and service, and connect them.](https://wiki.yandex-team.ru/solomon/howtostart/). +Add hostnames which run your application to **Cluster hosts** (use hostnames without "http://"). Solomon's fetcher will use **Port** field for all added hosts. +Fill in the following params: +- **Monitoring model** : **PULL**; +- **URL Path** : **/stats**; +- **Interval** : **10**; +- **Port** : a port of http server YDB SDK runs. + +### Setup TSolomonStatPullExtension in Your Application Code + +After creating NYdb::TDriver you need to add Solomon Monitoring extension. If you set up incorrect hostname or port **TSystemError** exception will be thrown. + +> **Important**: you must plug in monitoring before driver creation. + +```cl +#include +#include + +... + +{ + auto config = NYdb::TDriverConfig(); + NYdb::TDriver driver(config); + try { + + const std::string host = ... // use hostname without http:// + const ui64 port = ... + const std::string project = ... // Solomon's project id + const std::string service = ... // Solomon's service id + const std::string cluster = ... // Solomon's cluster id + NSolomonStatExtension::TSolomonStatPullExtension::TParams params(host, port, project, service, cluster); + driver.AddExtension(params); + + } catch (TSystemError& error) { + ... + } +} + +``` + +## Setup Monitoring of Your Choice + +Implementing NMonitoring::IMetricRegistry provides more flexibility. You can deliver application metrics to Prometheus or any other system of your choice, just register your specific NMonitoring::IMetricRegistry implementation via AddMetricRegistry function. + +> **Important**: you must plug in monitoring before driver creation. + +Select a method which is right for you: +```cl +#include + +... + +void AddMetricRegistry(NYdb::TDriver& driver, NMonitoring::IMetricRegistry* ptr); +void AddMetricRegistry(NYdb::TDriver& driver, std::shared_ptr ptr); +void AddMetricRegistry(NYdb::TDriver& driver, TAtomicSharedPtr ptr); +``` + +If you provide a raw pointer, it's your responsibility to delete the registry. You must shutdown the SDK driver before destroying the registry. + +## Metrics Description + +The most valuable metrics are the following: + +- Grpc/InFlightByYdbHost - queries in flight for a selected YDB hostname +- Request/Latency - Query execution latency histogram from the client side, ms (without RetryOperation) +- Request/ParamsSize - Query params size histogram in bytes +- Request/QuerySize - Query text size histogram in bytes +- Request/ResultSize - Query response size histogram in bytes +- SessionBalancer/Variation - Coefficient of variation in the distribution of sessions across database hosts +- Sessions/InPool - Number of sessions in a pool +- Sessions/SessionsLimitExceeded - Rate of events for exceeding the limit on the number of sessions on the client side +- SessionsByYdbHost - Number of sessions per YDB hosts + diff --git a/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_client.cpp b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_client.cpp new file mode 100644 index 000000000000..fbb6129061af --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_client.cpp @@ -0,0 +1,47 @@ +#include + +namespace NSolomonStatExtension::inline V3 { + +TSolomonStatPullExtension::TParams::TParams(const std::string& host + , ui16 port + , const std::string& project + , const std::string& service + , const std::string& cluster + , const std::vector>& labels) + : Host_(host), Port_(port), Labels_() +{ + Labels_.Add("project", project); + Labels_.Add("service", service); + Labels_.Add("cluster", cluster); + for (const auto& label: labels) { + Labels_.Add(label.first, label.second); + } +} + +NMonitoring::TLabels TSolomonStatPullExtension::TParams::GetLabels() const { + return Labels_; +} + + +TSolomonStatPullExtension::TSolomonStatPage::TSolomonStatPage(const std::string& title, const std::string& path, IApi* api) + : NMonitoring::IMonPage(TString(title), TString(path)), Api_(api) + { } + +void TSolomonStatPullExtension::TSolomonStatPage::Output(NMonitoring::IMonHttpRequest& request) { + request.Output() << NMonitoring::HTTPOKJSON; + auto json = NMonitoring::EncoderJson(&request.Output()); + Api_->Accept(json.Get()); +} + +TSolomonStatPullExtension::TSolomonStatPullExtension(const TSolomonStatPullExtension::TParams& params, IApi* api) + : MetricRegistry_(new NMonitoring::TMetricRegistry(params.GetLabels())) + , MonService_(params.Port_, TString(params.Host_), 0), Page_( new TSolomonStatPage("stats", "Statistics", api) ) { + api->SetMetricRegistry(MetricRegistry_.get()); + MonService_.Register(Page_); + MonService_.StartOrThrow(); + } + +TSolomonStatPullExtension::~TSolomonStatPullExtension() + { } + +} // NSolomonStatExtension diff --git a/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_connector.cpp b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_connector.cpp new file mode 100644 index 000000000000..8405a7cbedde --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/pull_connector.cpp @@ -0,0 +1 @@ +#include diff --git a/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/ya.make b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/ya.make new file mode 100644 index 000000000000..9f0ac06b8d2a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/solomon_stats/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + pull_client.cpp + pull_connector.cpp +) + +PEERDIR( + library/cpp/monlib/encode/json + library/cpp/monlib/metrics + library/cpp/monlib/service + library/cpp/monlib/service/pages + ydb/public/sdk/cpp/src/client/extension_common +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/extensions/ya.make b/ydb/public/sdk/cpp/src/client/extensions/ya.make new file mode 100644 index 000000000000..b2c2d1058ca4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/extensions/ya.make @@ -0,0 +1,4 @@ +RECURSE( + discovery_mutator + solomon_stats +) diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/federated_topic/CMakeLists.txt new file mode 100644 index 000000000000..65fced13e79c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/CMakeLists.txt @@ -0,0 +1,18 @@ +add_subdirectory(impl) + +_ydb_sdk_add_library(client-ydb_federated_topic) + +target_link_libraries(client-ydb_federated_topic PUBLIC + yutil + enum_serialization_runtime + client-ydb_topic + client-ydb_federated_topic-impl +) + +generate_enum_serilization(client-ydb_federated_topic + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/federated_topic/federated_topic.h +) + +_ydb_sdk_make_client_component(FederatedTopic client-ydb_federated_topic) diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/federated_topic/impl/CMakeLists.txt new file mode 100644 index 000000000000..6e38bd7f8969 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/CMakeLists.txt @@ -0,0 +1,31 @@ +_ydb_sdk_add_library(client-ydb_federated_topic-impl) + +target_link_libraries(client-ydb_federated_topic-impl PUBLIC + yutil + grpc-client + monlib-dynamic_counters + monlib-metrics + string_utils-url + persqueue-obfuscate + api-grpc-draft + api-grpc + impl-ydb_internal-make_request + client-ydb_common_client-impl + client-ydb_driver + client-ydb_proto +) + +target_compile_options(client-ydb_federated_topic-impl PRIVATE + -Wno-deprecated +) + +target_sources(client-ydb_federated_topic-impl PRIVATE + federated_read_session.cpp + federated_read_session_event.cpp + federated_write_session.cpp + federated_topic_impl.cpp + federated_topic.cpp + federation_observer.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_federated_topic-impl) diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.cpp new file mode 100644 index 000000000000..90343c3d46fc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.cpp @@ -0,0 +1,291 @@ +#include "federated_read_session.h" + +#include +#include + + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +namespace NYdb::inline V3::NFederatedTopic { + +NTopic::TTopicClientSettings FromFederated(const TFederatedTopicClientSettings& settings); + +template +typename std::function WrapFederatedHandler(std::function outerHandler, std::shared_ptr db, std::shared_ptr federator) { + if (outerHandler) { + return [outerHandler, db = std::move(db), federator = std::move(federator)](TEvent& ev) { + auto fev = federator->LocateFederate(ev, std::move(db)); + return outerHandler(fev); + }; + } + return {}; +} + +NTopic::TReadSessionSettings FromFederated(const TFederatedReadSessionSettings& settings, const std::shared_ptr& db, std::shared_ptr federator) { + NTopic::TReadSessionSettings SubsessionSettings = settings; + SubsessionSettings.EventHandlers_.MaxMessagesBytes(settings.EventHandlers_.MaxMessagesBytes_); + SubsessionSettings.EventHandlers_.HandlersExecutor(settings.EventHandlers_.HandlersExecutor_); + + if (settings.FederatedEventHandlers_.SimpleDataHandlers_.DataHandler) { + SubsessionSettings.EventHandlers_.SimpleDataHandlers( + WrapFederatedHandler( + settings.FederatedEventHandlers_.SimpleDataHandlers_.DataHandler, db, federator), + settings.FederatedEventHandlers_.SimpleDataHandlers_.CommitDataAfterProcessing, + settings.FederatedEventHandlers_.SimpleDataHandlers_.GracefulStopAfterCommit); + } + +#define MAYBE_CONVERT_HANDLER(type, name) \ + if (settings.FederatedEventHandlers_.name##_) { \ + SubsessionSettings.EventHandlers_.name( \ + WrapFederatedHandler(settings.FederatedEventHandlers_.name##_, db, federator) \ + ); \ + } + + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TDataReceivedEvent, DataReceivedHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TCommitOffsetAcknowledgementEvent, CommitOffsetAcknowledgementHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TStartPartitionSessionEvent, StartPartitionSessionHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TStopPartitionSessionEvent, StopPartitionSessionHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TEndPartitionSessionEvent, EndPartitionSessionHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TPartitionSessionStatusEvent, PartitionSessionStatusHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TPartitionSessionClosedEvent, PartitionSessionClosedHandler); + MAYBE_CONVERT_HANDLER(TReadSessionEvent::TEvent, CommonHandler); + +#undef MAYBE_CONVERT_HANDLER + + SubsessionSettings.EventHandlers_.SessionClosedHandler(settings.FederatedEventHandlers_.SessionClosedHandler_); + + return SubsessionSettings; +} + +TFederatedReadSessionImpl::TFederatedReadSessionImpl(const TFederatedReadSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSettings, + std::shared_ptr observer, + std::shared_ptr>> codecs) + : Settings(settings) + , Connections(std::move(connections)) + , SubClientSettings(FromFederated(clientSettings)) + , ProvidedCodecs(std::move(codecs)) + , Observer(std::move(observer)) + , AsyncInit(Observer->WaitForFirstState()) + , FederationState(nullptr) + , EventFederator(std::make_shared()) + , Log(Connections->GetLog()) + , SessionId(CreateGuidAsString()) +{ +} + +TStringBuilder TFederatedReadSessionImpl::GetLogPrefix() const { + return TStringBuilder() << GetDatabaseLogPrefix(SubClientSettings.Database_.value_or("")) << "[" << SessionId << "] "; +} + +void TFederatedReadSessionImpl::Start() { + AsyncInit.Subscribe([selfCtx = SelfContext](const auto& f){ + Y_UNUSED(f); + if (auto self = selfCtx->LockShared()) { + std::lock_guard guard(self->Lock); + if (self->Closing) { + return; + } + self->FederationState = self->Observer->GetState(); + self->OnFederatedStateUpdateImpl(); + } + }); +} + +void TFederatedReadSessionImpl::OpenSubSessionsImpl(const std::vector>& dbInfos) { + { + TStringBuilder log(GetLogPrefix()); + log << "Open read subsessions to databases: "; + bool first = true; + for (const auto& db : dbInfos) { + if (first) first = false; else log << ", "; + log << "{ name: " << db->name() + << ", endpoint: " << db->endpoint() + << ", path: " << db->path() << " }"; + } + LOG_LAZY(Log, TLOG_INFO, log); + } + for (const auto& db : dbInfos) { + NTopic::TTopicClientSettings settings = SubClientSettings; + settings + .Database(db->path()) + .DiscoveryEndpoint(db->endpoint()); + auto subclient = make_shared(Connections, settings); + auto subsession = subclient->CreateReadSession(FromFederated(Settings, db, EventFederator)); + SubSessions.emplace_back(subsession, db); + } + SubsessionIndex = 0; +} + +void TFederatedReadSessionImpl::OnFederatedStateUpdateImpl() { + if (!FederationState->Status.IsSuccess()) { + LOG_LAZY(Log, TLOG_ERR, GetLogPrefix() << "Federated state update failed. FederationState: " << *FederationState); + CloseImpl(); + return; + } + + EventFederator->SetFederationState(FederationState); + + if (Settings.IsReadMirroredEnabled()) { + Y_ABORT_UNLESS(Settings.GetDatabasesToReadFrom().size() == 1); + auto dbToReadFrom = *Settings.GetDatabasesToReadFrom().begin(); + + std::vector dbNames = GetAllFederationDatabaseNames(); + auto topics = Settings.Topics_; + for (const auto& topic : topics) { + for (const auto& dbName : dbNames) { + if (AsciiEqualsIgnoreCase(dbName, dbToReadFrom)) { + continue; + } + auto mirroredTopic = topic; + mirroredTopic.PartitionIds_.clear(); + mirroredTopic.Path(topic.Path_ + "-mirrored-from-" + dbName); + Settings.AppendTopics(mirroredTopic); + } + } + } + + std::vector> databases; + + for (const auto& db : FederationState->DbInfos) { + if (IsDatabaseEligibleForRead(db)) { + databases.push_back(db); + } + } + + if (databases.empty()) { + // TODO: investigate here, why empty list? + // Reason (and returned status) could be BAD_REQUEST or UNAVAILABLE. + LOG_LAZY(Log, TLOG_ERR, GetLogPrefix() << "No available databases to read."); + auto issues = FederationState->Status.GetIssues(); + TStringBuilder issue; + issue << "Requested databases {"; + bool first = true; + for (auto const& dbFromSettings : Settings.GetDatabasesToReadFrom()) { + if (first) first = false; else issue << ","; + issue << " " << dbFromSettings; + } + issue << " } not found. Available databases {"; + first = true; + for (auto const& db : FederationState->DbInfos) { + if (db->status() == Ydb::FederationDiscovery::DatabaseInfo_Status_AVAILABLE) { + if (first) first = false; else issue << ","; + issue << " { name: " << db->name() + << ", endpoint: " << db->endpoint() + << ", path: " << db->path() << " }"; + } + } + issue << " }"; + issues.AddIssue(issue); + FederationState->Status = TStatus(EStatus::BAD_REQUEST, std::move(issues)); + CloseImpl(); + return; + } + + OpenSubSessionsImpl(databases); +} + +std::vector TFederatedReadSessionImpl::GetAllFederationDatabaseNames() { + std::vector result; + for (const auto& db : FederationState->DbInfos) { + result.push_back(db->name()); + } + return result; +} + +bool TFederatedReadSessionImpl::IsDatabaseEligibleForRead(const std::shared_ptr& db) { + if (db->status() != TDbInfo::Status::DatabaseInfo_Status_AVAILABLE && + db->status() != TDbInfo::Status::DatabaseInfo_Status_READ_ONLY) { + return false; + } + + if (Settings.GetDatabasesToReadFrom().empty()) { + return true; + } + + for (const auto& dbFromSettings : Settings.GetDatabasesToReadFrom()) { + if (AsciiEqualsIgnoreCase(db->name(), dbFromSettings) || + AsciiEqualsIgnoreCase(db->id(), dbFromSettings)) { + return true; + } + if (dbFromSettings == "_local" && + AsciiEqualsIgnoreCase(FederationState->SelfLocation, db->location())) { + return true; + } + } + return false; +} + +NThreading::TFuture TFederatedReadSessionImpl::WaitEvent() { + // TODO override with read session settings timeout + return AsyncInit.Apply([selfCtx = SelfContext](const NThreading::TFuture) { + if (auto self = selfCtx->LockShared()) { + std::vector> waiters; + std::lock_guard guard(self->Lock); + if (self->Closing) { + return NThreading::MakeFuture(); + } + Y_ABORT_UNLESS(!self->SubSessions.empty(), "SubSessions empty in discovered state"); + for (const auto& sub : self->SubSessions) { + waiters.emplace_back(sub.Session->WaitEvent()); + } + return NThreading::WaitAny(std::move(waiters)); + } + return NThreading::MakeFuture(); + }); +} + +std::vector TFederatedReadSessionImpl::GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) { + if (block) { + WaitEvent().Wait(); + } + { + std::lock_guard guard(Lock); + if (Closing) { + return {NTopic::TSessionClosedEvent(FederationState->Status.GetStatus(), NYdb::NIssue::TIssues(FederationState->Status.GetIssues()))}; + } + // TODO!!! handle aborting or closing state + // via handler on SessionClosedEvent { + // cancel all subsessions, empty SubSessions, set aborting + // } + if (SubSessions.empty()) { + return {}; + } + } + std::vector result; + with_lock (Lock) { + do { + auto sub = SubSessions[SubsessionIndex]; + for (auto&& ev : sub.Session->GetEvents(false, maxEventsCount, maxByteSize)) { + result.push_back(EventFederator->LocateFederate(std::move(ev), sub.DbInfo)); + } + SubsessionIndex = (SubsessionIndex + 1) % SubSessions.size(); + } + while (block && result.empty()); + } + return result; +} + +void TFederatedReadSessionImpl::CloseImpl() { + Closing = true; +} + +bool TFederatedReadSessionImpl::Close(TDuration timeout) { + std::lock_guard guard(Lock); + Closing = true; + + bool result = true; + for (const auto& sub : SubSessions) { + // TODO substract from user timeout + result = sub.Session->Close(timeout); + } + return result; +} + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.h b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.h new file mode 100644 index 000000000000..9b7d4cf9a61c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session.h @@ -0,0 +1,249 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace NYdb::inline V3::NFederatedTopic { + +class TEventFederator { +public: + auto LocateTopicOrigin(const NTopic::TReadSessionEvent::TEvent& event) { + std::shared_ptr topicOriginDbInfo; + std::string topicOriginPath = ""; + + auto topicPath = std::visit([](auto&& arg) -> std::string_view { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return ""; + } else { + return arg.GetPartitionSession()->GetTopicPath(); + } + }, event); + + if (topicPath.find("-mirrored-from-") != std::string_view::npos) { + std::string_view leftPart, rightPart; + auto res = NUtils::TryRSplit(topicPath, leftPart, rightPart, "-mirrored-from-"); + Y_ABORT_UNLESS(res); + + // no additional validation required: TryGetDbInfo just returns nullptr for any bad input + topicOriginDbInfo = FederationState->TryGetDbInfo(std::string(rightPart)); + if (topicOriginDbInfo) { + topicOriginPath = leftPart; + } + } + + return std::make_tuple(topicOriginDbInfo, topicOriginPath); + } + + template + auto LocateFederate(TEvent&& event, std::shared_ptr db) { + NTopic::TPartitionSession::TPtr psPtr; + TFederatedPartitionSession::TPtr fps; + + using T = std::decay_t; + if constexpr (std::is_same_v) { + return Federate(std::move(event), std::move(fps)); + } else if constexpr (std::is_same_v) { + psPtr = std::visit([](auto&& arg) -> NTopic::TPartitionSession::TPtr { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return nullptr; + } else { + return arg.GetPartitionSession(); + } + }, event); + + if (!psPtr) { // TSessionClosedEvent + return Federate(std::move(event), std::move(fps)); + } + } else { + psPtr = event.GetPartitionSession(); + } + + with_lock(Lock) { + if (!FederatedPartitionSessions.contains(psPtr.Get())) { + auto [topicOriginDbInfo, topicOriginPath] = LocateTopicOrigin(event); + FederatedPartitionSessions[psPtr.Get()] = MakeIntrusive(psPtr, std::move(db), std::move(topicOriginDbInfo), std::move(topicOriginPath)); + } + fps = FederatedPartitionSessions[psPtr.Get()]; + + if constexpr (std::is_same_v) { + FederatedPartitionSessions.erase(psPtr.Get()); + } + } + + return Federate(std::move(event), std::move(fps)); + } + + template + TReadSessionEvent::TFederated Federate(TEvent event, TFederatedPartitionSession::TPtr federatedPartitionSession) { + return {std::move(event), std::move(federatedPartitionSession)}; + } + + TReadSessionEvent::TDataReceivedEvent Federate(NTopic::TReadSessionEvent::TDataReceivedEvent event, + TFederatedPartitionSession::TPtr federatedPartitionSession) { + return {std::move(event), std::move(federatedPartitionSession)}; + } + + TReadSessionEvent::TEvent Federate(NTopic::TReadSessionEvent::TEvent event, + TFederatedPartitionSession::TPtr federatedPartitionSession) { + return std::visit([fps = std::move(federatedPartitionSession)](auto&& arg) { + using T = std::decay_t; + std::optional ev; + if constexpr (std::is_same_v) { + ev = TReadSessionEvent::TDataReceivedEvent(std::move(arg), std::move(fps)); + } else if constexpr (std::is_same_v) { + ev = std::move(arg); + } else { + ev = TReadSessionEvent::TFederated(std::move(arg), std::move(fps)); + } + return *ev; + }, + event); + } + + void SetFederationState(std::shared_ptr state) { + with_lock(Lock) { + FederationState = std::move(state); + } + } + +private: + TAdaptiveLock Lock; + std::unordered_map FederatedPartitionSessions; + std::shared_ptr FederationState; +}; + +class TFederatedReadSessionImpl : public NTopic::TEnableSelfContext { + friend class TFederatedTopicClient::TImpl; + friend class TFederatedReadSession; + +private: + struct TSubSession { + TSubSession(std::shared_ptr session = {}, std::shared_ptr dbInfo = {}) + : Session(std::move(session)) + , DbInfo(std::move(dbInfo)) + {} + + std::shared_ptr Session; + std::shared_ptr DbInfo; + }; + +public: + TFederatedReadSessionImpl(const TFederatedReadSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSetttings, + std::shared_ptr observer, + std::shared_ptr>> codecs); + + ~TFederatedReadSessionImpl() = default; + + NThreading::TFuture WaitEvent(); + std::vector GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize); + + bool Close(TDuration timeout); + + inline std::string GetSessionId() const { + return SessionId; + } + + inline NTopic::TReaderCounters::TPtr GetCounters() const { + return Settings.Counters_; // Always not nullptr. + } + +private: + TStringBuilder GetLogPrefix() const; + + void Start(); + bool ValidateSettings(); + void OpenSubSessionsImpl(const std::vector>& dbInfos); + + std::vector GetAllFederationDatabaseNames(); + + bool IsDatabaseEligibleForRead(const std::shared_ptr& db); + + void OnFederatedStateUpdateImpl(); + + void CloseImpl(); + +private: + TFederatedReadSessionSettings Settings; + + // For subsessions creation + std::shared_ptr Connections; + const NTopic::TTopicClientSettings SubClientSettings; + std::shared_ptr>> ProvidedCodecs; + + std::shared_ptr Observer; + NThreading::TFuture AsyncInit; + std::shared_ptr FederationState; + std::shared_ptr EventFederator; + + TLog Log; + + const std::string SessionId; + const TInstant StartSessionTime = TInstant::Now(); + + TAdaptiveLock Lock; + + std::vector SubSessions; + size_t SubsessionIndex = 0; + + // Exiting. + bool Closing = false; +}; + + +class TFederatedReadSession : public IFederatedReadSession, + public NTopic::TContextOwner { + friend class TFederatedTopicClient::TImpl; + +public: + TFederatedReadSession(const TFederatedReadSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSettings, + std::shared_ptr observer, + std::shared_ptr>> codecs) + : TContextOwner(settings, std::move(connections), clientSettings, std::move(observer), std::move(codecs)) { + } + + ~TFederatedReadSession() { + TryGetImpl()->Close(TDuration::Zero()); + } + + NThreading::TFuture WaitEvent() override { + return TryGetImpl()->WaitEvent(); + } + + std::vector GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) override { + return TryGetImpl()->GetEvents(block, maxEventsCount, maxByteSize); + } + + std::optional GetEvent(bool block, size_t maxByteSize) override { + auto events = GetEvents(block, 1, maxByteSize); + return events.empty() ? std::nullopt : std::optional{std::move(events.front())}; + } + + bool Close(TDuration timeout) override { + return TryGetImpl()->Close(timeout); + } + + inline std::string GetSessionId() const override { + return TryGetImpl()->GetSessionId(); + } + + inline NTopic::TReaderCounters::TPtr GetCounters() const override { + return TryGetImpl()->GetCounters(); + } + +private: + void Start() { + return TryGetImpl()->Start(); + } +}; + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session_event.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session_event.cpp new file mode 100644 index 000000000000..5c54f0257016 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_read_session_event.cpp @@ -0,0 +1,168 @@ +#include +#include + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Printable specializations + +namespace NYdb::inline V3::NTopic { + +using namespace NFederatedTopic; + +using TCommitOffsetAcknowledgementEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TStartPartitionSessionEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TStopPartitionSessionEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TEndPartitionSessionEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TPartitionSessionStatusEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TPartitionSessionClosedEvent = NFederatedTopic::TReadSessionEvent::TFederated; +using TMessage = NFederatedTopic::TReadSessionEvent::TFederated; +using TCompressedMessage = NFederatedTopic::TReadSessionEvent::TFederated; +using TDataReceivedEvent = NFederatedTopic::TReadSessionEvent::TDataReceivedEvent; + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const { + const auto* self = static_cast(this); + res << " Partition session id: " << self->GetPartitionSessionId() + << " Topic: \"" << self->GetTopicPath() << "\"" + << " Partition: " << self->GetPartitionId() + << " Database name: " << self->GetDatabaseName() + << " Database path: " << self->GetDatabasePath() + << " Database id: " << self->GetDatabaseId(); +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "Message {"; + auto ptr = dynamic_cast(self); + Y_ABORT_UNLESS(ptr); + ptr->DebugString(ret, printData); + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "CompressedMessage {"; + static_cast(self)->DebugString(ret, printData); + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " Codec: " << self->GetCodec() + << " Uncompressed size: " << self->GetUncompressedSize() + << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "CommitAcknowledgement {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "StartPartitionSession {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " EndOffset: " << self->GetEndOffset() + << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "StopPartitionSession {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " }"; +} + +void JoinIds(TStringBuilder& ret, const std::vector ids); + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "EndPartitionSession {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " AdjacentPartitionIds: "; + JoinIds(ret, self->GetAdjacentPartitionIds()); + ret << " ChildPartitionIds: "; + JoinIds(ret, self->GetChildPartitionIds()); + ret << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "PartitionSessionStatus {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " ReadOffset: " << self->GetReadOffset() + << " EndOffset: " << self->GetEndOffset() + << " WriteWatermark: " << self->GetWriteTimeHighWatermark() + << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "PartitionSessionClosed {"; + self->GetFederatedPartitionSession()->DebugString(ret); + ret << " Reason: " << self->GetReason() + << " }"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "DataReceived {"; + self->GetFederatedPartitionSession()->DebugString(ret); + if (self->HasCompressedMessages()) { + for (const auto& message : self->GetCompressedMessages()) { + ret << " "; + message.DebugString(ret, printData); + } + } else { + for (const auto& message : self->GetMessages()) { + ret << " "; + message.DebugString(ret, printData); + } + } + ret << " }"; +} + +} + +namespace NYdb::inline V3::NFederatedTopic { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NFederatedTopic::TReadSessionEvent::TDataReceivedEvent + +TReadSessionEvent::TDataReceivedEvent::TDataReceivedEvent(NTopic::TReadSessionEvent::TDataReceivedEvent event, TFederatedPartitionSession::TPtr federatedPartitionSession) + : NTopic::TReadSessionEvent::TPartitionSessionAccessor(event.GetPartitionSession()) + , TFederatedPartitionSessionAccessor(federatedPartitionSession) +{ + if (event.HasCompressedMessages()) { + for (auto& msg : event.GetCompressedMessages()) { + CompressedMessages.emplace_back(std::move(msg), federatedPartitionSession); + } + } else { + for (auto& msg : event.GetMessages()) { + Messages.emplace_back(std::move(msg), federatedPartitionSession); + } + } +} + +void TReadSessionEvent::TDataReceivedEvent::Commit() { + for (auto [from, to] : OffsetRanges) { + static_cast*>(PartitionSession.Get())->Commit(from, to); + } +} + +std::string DebugString(const TReadSessionEvent::TEvent& event) { + return std::visit([](const auto& ev) { return ev.DebugString(); }, event); +} + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic.cpp new file mode 100644 index 000000000000..f74e05e3842c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic.cpp @@ -0,0 +1,89 @@ +#include +#include + +namespace NYdb::inline V3::NFederatedTopic { + +// TFederatedReadSessionSettings +// Read policy settings + +using TReadOriginalSettings = TFederatedReadSessionSettings::TReadOriginalSettings; +TReadOriginalSettings& TReadOriginalSettings::AddDatabase(const std::string& database) { + Databases.insert(std::move(database)); + return *this; +} + +TReadOriginalSettings& TReadOriginalSettings::AddDatabases(const std::vector& databases) { + std::move(std::begin(databases), std::end(databases), std::inserter(Databases, Databases.end())); + return *this; +} + +TReadOriginalSettings& TReadOriginalSettings::AddLocal() { + Databases.insert("_local"); + return *this; +} + +TFederatedReadSessionSettings& TFederatedReadSessionSettings::ReadOriginal(TReadOriginalSettings settings) { + std::swap(DatabasesToReadFrom, settings.Databases); + ReadMirroredEnabled = false; + return *this; +} + +TFederatedReadSessionSettings& TFederatedReadSessionSettings::ReadMirrored(const std::string& database) { + if (database == "_local") { + ythrow TContractViolation("Reading from local database not supported, use specific database"); + } + DatabasesToReadFrom.clear(); + DatabasesToReadFrom.insert(std::move(database)); + ReadMirroredEnabled = true; + return *this; +} + +// TFederatedTopicClient + +NTopic::TTopicClientSettings FromFederated(const TFederatedTopicClientSettings& fedSettings) { + auto settings = NTopic::TTopicClientSettings() + .DefaultCompressionExecutor(fedSettings.DefaultCompressionExecutor_) + .DefaultHandlersExecutor(fedSettings.DefaultHandlersExecutor_); + + if (fedSettings.CredentialsProviderFactory_) { + settings.CredentialsProviderFactory(*fedSettings.CredentialsProviderFactory_); + } + if (fedSettings.SslCredentials_) { + settings.SslCredentials(*fedSettings.SslCredentials_); + } + if (fedSettings.DiscoveryMode_) { + settings.DiscoveryMode(*fedSettings.DiscoveryMode_); + } + return settings; +} + +TFederatedTopicClient::TFederatedTopicClient(const TDriver& driver, const TFederatedTopicClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ + ProvideCodec(NTopic::ECodec::GZIP, std::make_unique()); + ProvideCodec(NTopic::ECodec::LZOP, std::make_unique()); + ProvideCodec(NTopic::ECodec::ZSTD, std::make_unique()); +} + +void TFederatedTopicClient::ProvideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl) { + return Impl_->ProvideCodec(codecId, std::move(codecImpl)); +} + +std::shared_ptr TFederatedTopicClient::CreateReadSession(const TFederatedReadSessionSettings& settings) { + return Impl_->CreateReadSession(settings); +} + +// std::shared_ptr TFederatedTopicClient::CreateSimpleBlockingWriteSession( +// const TFederatedWriteSessionSettings& settings) { +// return Impl_->CreateSimpleBlockingWriteSession(settings); +// } + +std::shared_ptr TFederatedTopicClient::CreateWriteSession(const TFederatedWriteSessionSettings& settings) { + return Impl_->CreateWriteSession(settings); +} + +void TFederatedTopicClient::OverrideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl) { + return Impl_->OverrideCodec(codecId, std::move(codecImpl)); +} + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.cpp new file mode 100644 index 000000000000..4a1384800e51 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.cpp @@ -0,0 +1,61 @@ +#include "federated_topic_impl.h" + +#include "federated_read_session.h" +#include "federated_write_session.h" + +namespace NYdb::inline V3::NFederatedTopic { + +std::shared_ptr +TFederatedTopicClient::TImpl::CreateReadSession(const TFederatedReadSessionSettings& settings) { + InitObserver(); + auto session = std::make_shared(settings, Connections, ClientSettings, GetObserver(), ProvidedCodecs); + session->Start(); + return std::move(session); +} + +// std::shared_ptr +// TFederatedTopicClient::TImpl::CreateSimpleBlockingWriteSession(const TFederatedWriteSessionSettings& settings) { +// InitObserver(); +// auto session = std::make_shared(settings, Connections, ClientSettings, GetObserver()); +// session->Start(); +// return std::move(session); + +// } + +std::shared_ptr +TFederatedTopicClient::TImpl::CreateWriteSession(const TFederatedWriteSessionSettings& settings) { + // Split settings.MaxMemoryUsage_ by two. + // One half goes to subsession. Other half goes to federated session internal buffer. + const ui64 splitSize = (settings.MaxMemoryUsage_ + 1) / 2; + TFederatedWriteSessionSettings splitSettings = settings; + splitSettings.MaxMemoryUsage(splitSize); + InitObserver(); + with_lock(Lock) { + if (!splitSettings.EventHandlers_.HandlersExecutor_) { + splitSettings.EventHandlers_.HandlersExecutor(ClientSettings.DefaultHandlersExecutor_); + } + } + auto session = std::make_shared( + splitSettings, Connections, ClientSettings, GetObserver(), ProvidedCodecs, GetSubsessionHandlersExecutor()); + session->Start(); + return std::move(session); +} + +void TFederatedTopicClient::TImpl::InitObserver() { + std::lock_guard guard(Lock); + if (!Observer || Observer->IsStale()) { + Observer = std::make_shared(Connections, ClientSettings); + Observer->Start(); + } +} + +auto TFederatedTopicClient::TImpl::GetSubsessionHandlersExecutor() -> NTopic::IExecutor::TPtr { + with_lock (Lock) { + if (!SubsessionHandlersExecutor) { + SubsessionHandlersExecutor = NTopic::CreateThreadPoolExecutor(1); + } + return SubsessionHandlersExecutor; + } +} + +} diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.h b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.h new file mode 100644 index 000000000000..e283c5a8f3b5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_topic_impl.h @@ -0,0 +1,92 @@ +#pragma once + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include + +#include +#include +#include + +namespace NYdb::inline V3::NFederatedTopic { + +class TFederatedTopicClient::TImpl { +public: + // Constructor for main client. + TImpl(std::shared_ptr&& connections, const TFederatedTopicClientSettings& settings) + : Connections(std::move(connections)) + , ClientSettings(settings) + { + InitObserver(); + } + + ~TImpl() { + std::lock_guard guard(Lock); + if (Observer) { + Observer->Stop(); + } + } + + void ProvideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl) { + with_lock(Lock) { + if (ProvidedCodecs->contains(codecId)) { + throw yexception() << "codec with id " << ui32(codecId) << " already provided"; + } + (*ProvidedCodecs)[codecId] = std::move(codecImpl); + } + } + + void OverrideCodec(NTopic::ECodec codecId, std::unique_ptr&& codecImpl) { + with_lock(Lock) { + (*ProvidedCodecs)[codecId] = std::move(codecImpl); + } + } + + const NTopic::ICodec* GetCodecImplOrThrow(NTopic::ECodec codecId) const { + with_lock(Lock) { + if (!ProvidedCodecs->contains(codecId)) { + throw yexception() << "codec with id " << ui32(codecId) << " not provided"; + } + return ProvidedCodecs->at(codecId).get(); + } + } + + std::shared_ptr>> GetProvidedCodecs() const { + return ProvidedCodecs; + } + + // Runtime API. + std::shared_ptr CreateReadSession(const TFederatedReadSessionSettings& settings); + + std::shared_ptr CreateSimpleBlockingWriteSession(const TFederatedWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TFederatedWriteSessionSettings& settings); + + std::shared_ptr GetObserver() { + std::lock_guard guard(Lock); + return Observer; + } + + void InitObserver(); + +private: + + // Use single-threaded executor to prevent deadlocks inside subsession event handlers. + NTopic::IExecutor::TPtr GetSubsessionHandlersExecutor(); + +private: + std::shared_ptr Connections; + const TFederatedTopicClientSettings ClientSettings; + std::shared_ptr Observer; + std::shared_ptr>> ProvidedCodecs = + std::make_shared>>(); + + NTopic::IExecutor::TPtr SubsessionHandlersExecutor; + + TAdaptiveLock Lock; +}; + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.cpp new file mode 100644 index 000000000000..5d6b312fc6bf --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.cpp @@ -0,0 +1,470 @@ +#include "federated_write_session.h" + +#include +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include + +namespace NYdb::inline V3::NFederatedTopic { + +constexpr TDuration UPDATE_FEDERATION_STATE_DELAY = TDuration::Seconds(10); + +bool DatabasesAreSame(std::shared_ptr lhs, std::shared_ptr rhs) { + if (!lhs || !rhs) { + return false; + } + return lhs->name() == rhs->name() && lhs->path() == rhs->path() && lhs->endpoint() == rhs->endpoint(); +} + +NTopic::TTopicClientSettings FromFederated(const TFederatedTopicClientSettings& settings); + +TFederatedWriteSessionImpl::TFederatedWriteSessionImpl( + const TFederatedWriteSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSettings, + std::shared_ptr observer, + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor +) + : Settings(settings) + , Connections(std::move(connections)) + , SubclientSettings(FromFederated(clientSettings)) + , ProvidedCodecs(std::move(codecs)) + , SubsessionHandlersExecutor(subsessionHandlersExecutor) + , Observer(std::move(observer)) + , AsyncInit(Observer->WaitForFirstState()) + , FederationState(nullptr) + , Log(Connections->GetLog()) + , ClientEventsQueue(std::make_shared(Settings)) + , BufferFreeSpace(Settings.MaxMemoryUsage_) + , HasBeenClosed(NThreading::NewPromise()) +{ +} + +TStringBuilder TFederatedWriteSessionImpl::GetLogPrefixImpl() const { + return TStringBuilder() << GetDatabaseLogPrefix(SubclientSettings.Database_.value_or("")) << "[" << SessionId << "] "; +} + + +bool TFederatedWriteSessionImpl::MessageQueuesAreEmptyImpl() const { + Y_ABORT_UNLESS(Lock.IsLocked()); + return OriginalMessagesToGetAck.empty() && OriginalMessagesToPassDown.empty(); +} + +void TFederatedWriteSessionImpl::IssueTokenIfAllowed() { + // The session should not issue tokens after it has transitioned to CLOSING or CLOSE state. + // A user may have one spare token, so at most one additional message + // could be written to the internal queue after the transition. + bool issue = false; + with_lock(Lock) { + if (BufferFreeSpace > 0 && !ClientHasToken && SessionState < State::CLOSING) { + ClientHasToken = true; + issue = true; + } + } + if (issue) { + ClientEventsQueue->PushEvent(NTopic::TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } +} + +std::shared_ptr TFederatedWriteSessionImpl::UpdateFederationStateImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + // Even after the user has called the Close method, transitioning the session to the CLOSING state, + // we keep updating the federation state, as the session may still have some messages to send in its queues, + // and for that we need to know the current state of the federation. + if (SessionState < State::CLOSED) { + FederationState = Observer->GetState(); + return OnFederationStateUpdateImpl(); + } + return {}; +} + +void TFederatedWriteSessionImpl::Start() { + // TODO validate settings? + with_lock(Lock){ + if (SessionState != State::CREATED) { + return; + } + SessionState = State::WORKING; + Settings.EventHandlers_.HandlersExecutor_->Start(); + } + + IssueTokenIfAllowed(); + + AsyncInit.Subscribe([selfCtx = SelfContext](const auto& f) { + Y_UNUSED(f); + if (auto self = selfCtx->LockShared()) { + with_lock(self->Lock) { + self->UpdateFederationStateImpl(); + } + } + }); +} + +std::shared_ptr TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + ++SubsessionGeneration; + + std::shared_ptr oldSubsession; + + if (Subsession) { + PendingToken.reset(); + std::swap(oldSubsession, Subsession); + } + + auto clientSettings = SubclientSettings; + clientSettings + .Database(db->path()) + .DiscoveryEndpoint(db->endpoint()); + auto subclient = std::make_shared(Connections, clientSettings); + + auto handlers = NTopic::TWriteSessionSettings::TEventHandlers() + .HandlersExecutor(SubsessionHandlersExecutor) + .ReadyToAcceptHandler([selfCtx = SelfContext, generation = SubsessionGeneration](NTopic::TWriteSessionEvent::TReadyToAcceptEvent& ev) { + if (auto self = selfCtx->LockShared()) { + with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } + + Y_ABORT_UNLESS(!self->PendingToken.has_value()); + self->PendingToken = std::move(ev.ContinuationToken); + self->MaybeWriteImpl(); + } + } + }) + .AcksHandler([selfCtx = SelfContext, generation = SubsessionGeneration](NTopic::TWriteSessionEvent::TAcksEvent& ev) { + if (auto self = selfCtx->LockShared()) { + with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } + + Y_ABORT_UNLESS(ev.Acks.size() <= self->OriginalMessagesToGetAck.size()); + + for (size_t i = 0; i < ev.Acks.size(); ++i) { + self->BufferFreeSpace += self->OriginalMessagesToGetAck.front().Data.size(); + self->OriginalMessagesToGetAck.pop_front(); + } + + if (self->MessageQueuesAreEmptyImpl() && self->MessageQueuesHaveBeenEmptied.Initialized() && !self->MessageQueuesHaveBeenEmptied.HasValue()) { + self->MessageQueuesHaveBeenEmptied.SetValue(); + } + } + + self->ClientEventsQueue->PushEvent(std::move(ev)); + self->IssueTokenIfAllowed(); + } + }) + .SessionClosedHandler([selfCtx = SelfContext, generation = SubsessionGeneration](const NTopic::TSessionClosedEvent & ev) { + if (ev.IsSuccess()) { + // The subsession was closed by the federated write session itself while creating a new subsession. + // In this case we get SUCCESS status and don't need to propagate it further. + return; + } + if (auto self = selfCtx->LockShared()) { + with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } + self->CloseImpl(ev); + } + } + }); + + { + // Unacknowledged messages should be resent. + for (auto& msg : OriginalMessagesToPassDown) { + OriginalMessagesToGetAck.emplace_back(std::move(msg)); + } + OriginalMessagesToPassDown = std::move(OriginalMessagesToGetAck); + } + + NTopic::TWriteSessionSettings wsSettings = Settings; + wsSettings + // .MaxMemoryUsage(Settings.MaxMemoryUsage_) // to fix if split not by half on creation + .EventHandlers(handlers); + + Subsession = subclient->CreateWriteSession(wsSettings); + CurrentDatabase = db; + + return oldSubsession; +} + +std::pair, EStatus> SelectDatabaseByHashImpl( + NTopic::TFederatedWriteSessionSettings const& settings, + std::vector> const& dbInfos +) { + uint64_t totalWeight = 0; + std::vector> available; + + for (const auto& db : dbInfos) { + if (db->status() == TDbInfo::Status::DatabaseInfo_Status_AVAILABLE) { + available.push_back(db); + totalWeight += db->weight(); + } + } + + if (available.empty() || totalWeight == 0) { + return {nullptr, EStatus::NOT_FOUND}; + } + + std::sort(available.begin(), available.end(), [](auto const& lhs, auto const& rhs) { return lhs->name() < rhs->name(); }); + + size_t hashValue = THash()(settings.Path_); + hashValue = CombineHashes(hashValue, THash()(settings.ProducerId_)); + hashValue %= totalWeight; + + uint64_t borderWeight = 0; + for (auto const& db : available) { + borderWeight += db->weight(); + if (hashValue < borderWeight) { + return {db, EStatus::SUCCESS}; + } + } + Y_UNREACHABLE(); +} + +std::pair, EStatus> SelectDatabaseImpl( + NTopic::TFederatedWriteSessionSettings const& settings, + std::vector> const& dbInfos, std::string const& selfLocation +) { + /* Logic of the function should follow this table: + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | set | not found | - | any | NOT_FOUND | + | set | available | - | any | preferred | + | set | unavailable | - | false | UNAVAILABLE | + | set | unavailable | - | true | by hash | + | unset | - | not found | false | NOT_FOUND | + | unset | - | not found | true | by hash | + | unset | - | available | any | local | + | unset | - | unavailable | false | UNAVAILABLE | + | unset | - | unavailable | true | by hash | + */ + + decltype(begin(dbInfos)) it; + if (settings.PreferredDatabase_) { + it = std::find_if(begin(dbInfos), end(dbInfos), [&preferred = settings.PreferredDatabase_](auto const& db) { + return AsciiEqualsIgnoreCase(*preferred, db->name()) || AsciiEqualsIgnoreCase(*preferred, db->id()); + }); + if (it == end(dbInfos)) { + return {nullptr, EStatus::NOT_FOUND}; + } + } else { + it = std::find_if(begin(dbInfos), end(dbInfos), [&selfLocation](auto const& db) { + return AsciiEqualsIgnoreCase(selfLocation, db->location()); + }); + if (it == end(dbInfos)) { + if (!settings.AllowFallback_) { + return {nullptr, EStatus::NOT_FOUND}; + } + return SelectDatabaseByHashImpl(settings, dbInfos); + } + } + + auto db = *it; + if (db->status() == TDbInfo::Status::DatabaseInfo_Status_AVAILABLE) { + return {db, EStatus::SUCCESS}; + } + if (!settings.AllowFallback_) { + return {nullptr, EStatus::UNAVAILABLE}; + } + return SelectDatabaseByHashImpl(settings, dbInfos); +} + +std::shared_ptr TFederatedWriteSessionImpl::OnFederationStateUpdateImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (!FederationState->Status.IsSuccess()) { + // The observer became stale, it won't try to get federation state anymore due to retry policy, + // so there's no reason to keep the write session alive. + CloseImpl(FederationState->Status.GetStatus(), NYdb::NIssue::TIssues(FederationState->Status.GetIssues())); + return {}; + } + + Y_ABORT_UNLESS(!FederationState->DbInfos.empty()); + + auto [preferrableDb, status] = SelectDatabaseImpl(Settings, FederationState->DbInfos, FederationState->SelfLocation); + + if (!preferrableDb) { + if (!RetryState) { + RetryState = Settings.RetryPolicy_->CreateRetryState(); + } + if (auto delay = RetryState->GetNextRetryDelay(status)) { + LOG_LAZY(Log, TLOG_NOTICE, GetLogPrefixImpl() << "Retry to update federation state in " << delay); + ScheduleFederationStateUpdateImpl(*delay); + } else { + std::string message = "Failed to select database: no available database"; + LOG_LAZY(Log, TLOG_ERR, GetLogPrefixImpl() << message << ". Status: " << status); + CloseImpl(status, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue(message)}); + } + return {}; + } + RetryState.reset(); + + std::shared_ptr oldSubsession; + if (!DatabasesAreSame(preferrableDb, CurrentDatabase)) { + LOG_LAZY(Log, TLOG_INFO, GetLogPrefixImpl() + << "Start federated write session to database '" << preferrableDb->name() + << "' (previous was " << (CurrentDatabase ? CurrentDatabase->name() : "") << ")" + << " FederationState: " << *FederationState); + oldSubsession = OpenSubsessionImpl(preferrableDb); + } + + ScheduleFederationStateUpdateImpl(UPDATE_FEDERATION_STATE_DELAY); + + return oldSubsession; +} + +void TFederatedWriteSessionImpl::ScheduleFederationStateUpdateImpl(TDuration delay) { + Y_ABORT_UNLESS(Lock.IsLocked()); + auto cb = [selfCtx = SelfContext](bool ok) { + if (ok) { + if (auto self = selfCtx->LockShared()) { + std::shared_ptr old; + with_lock(self->Lock) { + old = self->UpdateFederationStateImpl(); + } + if (old) { + old->Close(TDuration::Zero()); + } + } + } + }; + + UpdateStateDelayContext = Connections->CreateContext(); + if (!UpdateStateDelayContext) { + CloseImpl(EStatus::TRANSPORT_UNAVAILABLE, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue("Could not update federation state")}); + // TODO log DRIVER_IS_STOPPING_DESCRIPTION + return; + } + Connections->ScheduleCallback(delay, + std::move(cb), + UpdateStateDelayContext); +} + +NThreading::TFuture TFederatedWriteSessionImpl::WaitEvent() { + return ClientEventsQueue->WaitEvent(); +} + +std::vector TFederatedWriteSessionImpl::GetEvents(bool block, std::optional maxEventsCount) { + return ClientEventsQueue->GetEvents(block, maxEventsCount); +} + +std::optional TFederatedWriteSessionImpl::GetEvent(bool block) { + auto events = GetEvents(block, 1); + return events.empty() ? std::nullopt : std::optional{std::move(events.front())}; +} + +NThreading::TFuture TFederatedWriteSessionImpl::GetInitSeqNo() { + return NThreading::MakeFuture(0u); +} + +void TFederatedWriteSessionImpl::Write(NTopic::TContinuationToken&& token, std::string_view data, std::optional seqNo, + std::optional createTimestamp) { + NTopic::TWriteMessage message{std::move(data)}; + if (seqNo.has_value()) + message.SeqNo(*seqNo); + if (createTimestamp.has_value()) + message.CreateTimestamp(*createTimestamp); + return WriteInternal(std::move(token), std::move(message)); +} + +void TFederatedWriteSessionImpl::Write(NTopic::TContinuationToken&& token, NTopic::TWriteMessage&& message) { + return WriteInternal(std::move(token), TWrappedWriteMessage(std::move(message))); +} + +void TFederatedWriteSessionImpl::WriteEncoded(NTopic::TContinuationToken&& token, std::string_view data, NTopic::ECodec codec, + ui32 originalSize, std::optional seqNo, std::optional createTimestamp) { + auto message = NTopic::TWriteMessage::CompressedMessage(std::move(data), codec, originalSize); + if (seqNo.has_value()) + message.SeqNo(*seqNo); + if (createTimestamp.has_value()) + message.CreateTimestamp(*createTimestamp); + return WriteInternal(std::move(token), TWrappedWriteMessage(std::move(message))); +} + +void TFederatedWriteSessionImpl::WriteEncoded(NTopic::TContinuationToken&& token, NTopic::TWriteMessage&& message) { + return WriteInternal(std::move(token), TWrappedWriteMessage(std::move(message))); +} + +void TFederatedWriteSessionImpl::WriteInternal(NTopic::TContinuationToken&&, TWrappedWriteMessage&& wrapped) { + with_lock(Lock) { + ClientHasToken = false; + if (!wrapped.Message.CreateTimestamp_.has_value()) { + wrapped.Message.CreateTimestamp_ = TInstant::Now(); + } + BufferFreeSpace -= wrapped.Message.Data.size(); + OriginalMessagesToPassDown.emplace_back(std::move(wrapped)); + MaybeWriteImpl(); + } + + IssueTokenIfAllowed(); +} + +bool TFederatedWriteSessionImpl::MaybeWriteImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (!PendingToken.has_value()) { + return false; + } + if (OriginalMessagesToPassDown.empty()) { + return false; + } + OriginalMessagesToGetAck.push_back(std::move(OriginalMessagesToPassDown.front())); + OriginalMessagesToPassDown.pop_front(); + Subsession->Write(std::move(*PendingToken), std::move(OriginalMessagesToGetAck.back().Message)); + PendingToken.reset(); + return true; +} + +void TFederatedWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + CloseImpl(TPlainStatus(statusCode, std::move(issues))); +} + +void TFederatedWriteSessionImpl::CloseImpl(NTopic::TSessionClosedEvent const& ev) { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (SessionState == State::CLOSED) { + return; + } + SessionState = State::CLOSED; + NTopic::Cancel(UpdateStateDelayContext); + if (!HasBeenClosed.HasValue()) { + HasBeenClosed.SetValue(); + } + { + auto unguard = Unguard(Lock); + ClientEventsQueue->Close(ev); + } +} + +bool TFederatedWriteSessionImpl::Close(TDuration timeout) { + with_lock(Lock) { + if (SessionState == State::CLOSED) { + return MessageQueuesAreEmptyImpl(); + } + SessionState = State::CLOSING; + if (!MessageQueuesHaveBeenEmptied.Initialized()) { + MessageQueuesHaveBeenEmptied = NThreading::NewPromise(); + if (MessageQueuesAreEmptyImpl()) { + MessageQueuesHaveBeenEmptied.SetValue(); + } + } + } + + std::vector> futures{MessageQueuesHaveBeenEmptied.GetFuture(), HasBeenClosed.GetFuture()}; + NThreading::WaitAny(futures).Wait(timeout); + + with_lock(Lock) { + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); + return MessageQueuesAreEmptyImpl(); + } +} + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.h b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.h new file mode 100644 index 000000000000..670558d69fb0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federated_write_session.h @@ -0,0 +1,209 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3::NFederatedTopic { + +std::pair, EStatus> SelectDatabaseByHashImpl( + NTopic::TFederatedWriteSessionSettings const& settings, + std::vector> const& dbInfos); + +std::pair, EStatus> SelectDatabaseImpl( + NTopic::TFederatedWriteSessionSettings const& settings, + std::vector> const& dbInfos, std::string const& selfLocation); + +class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, + public NTopic::TEnableSelfContext { + friend class TFederatedWriteSession; + friend class TFederatedTopicClient::TImpl; + +public: + TFederatedWriteSessionImpl(const TFederatedWriteSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSetttings, + std::shared_ptr observer, + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor); + + ~TFederatedWriteSessionImpl() = default; + + NThreading::TFuture WaitEvent(); + std::optional GetEvent(bool block); + std::vector GetEvents(bool block, std::optional maxEventsCount); + + NThreading::TFuture GetInitSeqNo(); + + void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message); + + void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params); + + void Write(NTopic::TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, + std::optional createTimestamp = std::nullopt); + + void WriteEncoded(NTopic::TContinuationToken&&, std::string_view, NTopic::ECodec, ui32, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); + + bool Close(TDuration timeout); + +private: + + struct TWrappedWriteMessage { + const std::string Data; + NTopic::TWriteMessage Message; + TWrappedWriteMessage(NTopic::TWriteMessage&& message) + : Data(message.Data) + , Message(std::move(message)) + { + Message.Data = Data; + } + + explicit TWrappedWriteMessage(const TWrappedWriteMessage& other) + : Data(other.Data) + , Message(other.Message) + { + Message.Data = Data; + } + + explicit TWrappedWriteMessage(TWrappedWriteMessage&& other) + : Data(std::move(other.Data)) + , Message(std::move(other.Message)) + { + Message.Data = Data; + } + + TWrappedWriteMessage& operator=(const TWrappedWriteMessage& other) = delete; + TWrappedWriteMessage& operator=(TWrappedWriteMessage&& other) = delete; + + ~TWrappedWriteMessage() = default; + }; + +private: + void Start(); + + std::shared_ptr OpenSubsessionImpl(std::shared_ptr db); + std::shared_ptr UpdateFederationStateImpl(); + std::shared_ptr OnFederationStateUpdateImpl(); + + void ScheduleFederationStateUpdateImpl(TDuration delay); + + void WriteInternal(NTopic::TContinuationToken&&, TWrappedWriteMessage&& message); + bool MaybeWriteImpl(); + + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); + void CloseImpl(NTopic::TSessionClosedEvent const& ev); + + bool MessageQueuesAreEmptyImpl() const; + + void IssueTokenIfAllowed(); + + TStringBuilder GetLogPrefixImpl() const; + +private: + // For subsession creation + const NTopic::TFederatedWriteSessionSettings Settings; + std::shared_ptr Connections; + const NTopic::TTopicClientSettings SubclientSettings; + std::shared_ptr>> ProvidedCodecs; + NTopic::IExecutor::TPtr SubsessionHandlersExecutor; + + NTopic::IRetryPolicy::IRetryState::TPtr RetryState; + std::shared_ptr Observer; + NThreading::TFuture AsyncInit; + std::shared_ptr FederationState; + NYdbGrpc::IQueueClientContextPtr UpdateStateDelayContext; + + TLog Log; + + std::shared_ptr CurrentDatabase; + + std::string SessionId; + const TInstant StartSessionTime = TInstant::Now(); + + TAdaptiveLock Lock; + + size_t SubsessionGeneration = 0; + std::shared_ptr Subsession; + + std::shared_ptr ClientEventsQueue; + + std::optional PendingToken; // from Subsession + bool ClientHasToken = false; + std::deque OriginalMessagesToPassDown; + std::deque OriginalMessagesToGetAck; + i64 BufferFreeSpace; + + enum class State { + CREATED, // The session has not been started. + WORKING, // Start method has been called. + CLOSING, // Close method has been called, but the session may still send some messages. + CLOSED // The session is closed, either due to the user request or some server error. + }; + State SessionState{State::CREATED}; + NThreading::TPromise MessageQueuesHaveBeenEmptied; + NThreading::TPromise HasBeenClosed; +}; + +class TFederatedWriteSession : public NTopic::IWriteSession, + public NTopic::TContextOwner { + friend class TFederatedTopicClient::TImpl; + +public: + + TFederatedWriteSession(const TFederatedWriteSessionSettings& settings, + std::shared_ptr connections, + const TFederatedTopicClientSettings& clientSettings, + std::shared_ptr observer, + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor) + : TContextOwner(settings, std::move(connections), clientSettings, std::move(observer), codecs, subsessionHandlersExecutor) {} + + NThreading::TFuture WaitEvent() override { + return TryGetImpl()->WaitEvent(); + } + std::optional GetEvent(bool block) override { + return TryGetImpl()->GetEvent(block); + } + std::vector GetEvents(bool block, std::optional maxEventsCount) override { + return TryGetImpl()->GetEvents(block, maxEventsCount); + } + NThreading::TFuture GetInitSeqNo() override { + return TryGetImpl()->GetInitSeqNo(); + } + void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message, NTable::TTransaction* tx = nullptr) override { + if (tx) { + ythrow yexception() << "transactions are not supported"; + } + TryGetImpl()->Write(std::move(continuationToken), std::move(message)); + } + void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params, NTable::TTransaction* tx = nullptr) override { + if (tx) { + ythrow yexception() << "transactions are not supported"; + } + TryGetImpl()->WriteEncoded(std::move(continuationToken), std::move(params)); + } + void Write(NTopic::TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, + std::optional createTimestamp = std::nullopt) override { + TryGetImpl()->Write(std::move(continuationToken), data, seqNo, createTimestamp); + } + void WriteEncoded(NTopic::TContinuationToken&& continuationToken, std::string_view data, NTopic::ECodec codec, ui32 originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override { + TryGetImpl()->WriteEncoded(std::move(continuationToken), data, codec, originalSize, seqNo, createTimestamp); + } + bool Close(TDuration timeout) override { + return TryGetImpl()->Close(timeout); + } + + inline NTopic::TWriterCounters::TPtr GetCounters() override {Y_ABORT("Unimplemented"); } //ToDo - unimplemented; + +private: + + void Start() { + TryGetImpl()->Start(); + } +}; + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.cpp new file mode 100644 index 000000000000..036574c32026 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.cpp @@ -0,0 +1,203 @@ +#include +#include + +#include + +namespace NYdb::inline V3::NFederatedTopic { + +constexpr TDuration REDISCOVERY_DELAY = TDuration::Seconds(30); + +TFederatedDbObserverImpl::TFederatedDbObserverImpl(std::shared_ptr connections, const TFederatedTopicClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + , FederatedDbState(std::make_shared()) + , PromiseToInitState(NThreading::NewPromise()) + , FederationDiscoveryRetryPolicy(settings.RetryPolicy_) +{ + RpcSettings.ClientTimeout = settings.ConnectionTimeout_; + RpcSettings.EndpointPolicy = TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint; + RpcSettings.UseAuth = true; +} + +TFederatedDbObserverImpl::~TFederatedDbObserverImpl() { + Stop(); +} + +std::shared_ptr TFederatedDbObserverImpl::GetState() { + std::lock_guard guard(Lock); + return FederatedDbState; +} + +NThreading::TFuture TFederatedDbObserverImpl::WaitForFirstState() { + return PromiseToInitState.GetFuture(); +} + +void TFederatedDbObserverImpl::Start() { + std::lock_guard guard(Lock); + if (Stopping) { + return; + } + ScheduleFederationDiscoveryImpl(TDuration::Zero()); +} + +void TFederatedDbObserverImpl::Stop() { + NYdbGrpc::IQueueClientContextPtr ctx; + { + std::lock_guard guard(Lock); + Stopping = true; + ctx = std::exchange(FederationDiscoveryDelayContext, nullptr); + } + if (ctx) { + ctx->Cancel(); + } +} + +// If observer is stale it will never update state again because of client retry policy +bool TFederatedDbObserverImpl::IsStale() const { + std::lock_guard guard(const_cast(Lock)); + return PromiseToInitState.HasValue() && !FederatedDbState->Status.IsSuccess(); +} + +Ydb::FederationDiscovery::ListFederationDatabasesRequest TFederatedDbObserverImpl::ComposeRequest() const { + return {}; +} + +void TFederatedDbObserverImpl::RunFederationDiscoveryImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + FederationDiscoveryDelayContext = Connections_->CreateContext(); + if (!FederationDiscoveryDelayContext) { + Stopping = true; + // TODO log DRIVER_IS_STOPPING_DESCRIPTION + return; + } + + auto extractor = [selfCtx = SelfContext] + (google::protobuf::Any* any, TPlainStatus status) mutable { + if (auto self = selfCtx->LockShared()) { + Ydb::FederationDiscovery::ListFederationDatabasesResult result; + if (any) { + any->UnpackTo(&result); + } + self->OnFederationDiscovery(std::move(status), std::move(result)); + } + }; + + Connections_->RunDeferred( + ComposeRequest(), + std::move(extractor), + &Ydb::FederationDiscovery::V1::FederationDiscoveryService::Stub::AsyncListFederationDatabases, + DbDriverState_, + {}, // no polling unready operations, so no need in delay parameter + RpcSettings, + FederationDiscoveryDelayContext); +} + +void TFederatedDbObserverImpl::ScheduleFederationDiscoveryImpl(TDuration delay) { + Y_ABORT_UNLESS(Lock.IsLocked()); + auto cb = [selfCtx = SelfContext](bool ok) { + if (ok) { + if (auto self = selfCtx->LockShared()) { + std::lock_guard guard(self->Lock); + if (self->Stopping) { + return; + } + self->RunFederationDiscoveryImpl(); + } + } + }; + + FederationDiscoveryDelayContext = Connections_->CreateContext(); + if (!FederationDiscoveryDelayContext) { + Stopping = true; + // TODO log DRIVER_IS_STOPPING_DESCRIPTION + return; + } + Connections_->ScheduleCallback(delay, + std::move(cb), + FederationDiscoveryDelayContext); + +} + +void TFederatedDbObserverImpl::OnFederationDiscovery(TStatus&& status, Ydb::FederationDiscovery::ListFederationDatabasesResult&& result) { + { + std::lock_guard guard(Lock); + if (Stopping) { + // TODO log something + return; + } + + // BAD_REQUEST may be returned from FederationDiscovery: + // 1) The request was meant for a non-federated topic: fall back to single db mode. + // 2) The database path in the request is simply wrong: the client should get the BAD_REQUEST status. + if (status.GetStatus() == EStatus::CLIENT_CALL_UNIMPLEMENTED || status.GetStatus() == EStatus::BAD_REQUEST) { + LOG_LAZY(DbDriverState_->Log, TLOG_INFO, TStringBuilder() + << "OnFederationDiscovery fall back to single mode, database=" << DbDriverState_->Database); + FederatedDbState->Status = TPlainStatus{}; // SUCCESS + FederatedDbState->ControlPlaneEndpoint = DbDriverState_->DiscoveryEndpoint; + auto dbState = Connections_->GetDriverState(std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt); + FederatedDbState->ControlPlaneEndpoint = dbState->DiscoveryEndpoint; + // FederatedDbState->SelfLocation = ???; + auto db = std::make_shared(); + db->set_path(TStringType{DbDriverState_->Database}); + db->set_endpoint(TStringType{DbDriverState_->DiscoveryEndpoint}); + db->set_status(Ydb::FederationDiscovery::DatabaseInfo_Status_AVAILABLE); + db->set_weight(100); + FederatedDbState->DbInfos.emplace_back(std::move(db)); + + } else { + if (status.IsSuccess()) { + ScheduleFederationDiscoveryImpl(REDISCOVERY_DELAY); + } else { + LOG_LAZY(DbDriverState_->Log, TLOG_ERR, TStringBuilder() + << "OnFederationDiscovery: Got error. Status: " << status.GetStatus() + << ". Description: " << status.GetIssues().ToOneLineString()); + if (!FederationDiscoveryRetryState) { + FederationDiscoveryRetryState = FederationDiscoveryRetryPolicy->CreateRetryState(); + } + + if (auto d = FederationDiscoveryRetryState->GetNextRetryDelay(status.GetStatus())) { + ScheduleFederationDiscoveryImpl(*d); + return; + } + // If there won't be another retry, we replace FederatedDbState with the unsuccessful one + // and set the PromiseToInitState to make the observer stale (see IsStale method). + } + + // TODO validate new state and check if differs from previous + auto newInfo = std::make_shared(std::move(result), std::move(status)); + + // TODO update only if new state differs + std::swap(FederatedDbState, newInfo); + } + } + + if (!PromiseToInitState.HasValue()) { + PromiseToInitState.SetValue(); + } +} + +IOutputStream& operator<<(IOutputStream& out, TFederatedDbState const& state) { + out << "{ Status: " << state.Status.GetStatus() + << " SelfLocation: \"" << state.SelfLocation << '"'; + if (auto const& issues = state.Status.GetIssues(); !issues.Empty()) { + out << " Issues: { " << issues.ToOneLineString() << " }"; + } + if (!state.DbInfos.empty()) { + out << " DbInfos: [ "; + bool first = true; + for (auto const& info : state.DbInfos) { + if (first) { + first = false; + } else { + out << " "; + } + out << "{ " << info->ShortDebugString() << " }"; + } + out << " ]"; + } + return out << " }"; +} + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.h b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.h new file mode 100644 index 000000000000..94f43b5ceb26 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/federation_observer.h @@ -0,0 +1,124 @@ +#pragma once + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include +#include +#include + +#include + +#include +#include + +namespace NYdb::inline V3::NFederatedTopic { + +struct TFederatedDbState { +public: + using TDbInfo = Ydb::FederationDiscovery::DatabaseInfo; + + TStatus Status; + std::string ControlPlaneEndpoint; + std::string SelfLocation; + std::vector> DbInfos; + +public: + TFederatedDbState() : Status(EStatus::STATUS_UNDEFINED, {}) {} + TFederatedDbState(Ydb::FederationDiscovery::ListFederationDatabasesResult result, TStatus status) + : Status(std::move(status)) + , ControlPlaneEndpoint(result.control_plane_endpoint()) + , SelfLocation(result.self_location()) + { + // TODO ensure that all databases have unique names? + for (const auto& db : result.federation_databases()) { + DbInfos.push_back(std::make_shared(db)); + } + } + + std::shared_ptr TryGetDbInfo(const std::string& name) const noexcept { + // There are few databases per federation usually, so the linear search is probably ok. + // TODO better profile this + for (const auto& dbInfo : DbInfos) { + if (AsciiEqualsIgnoreCase(dbInfo->name(), name)) { + return dbInfo; + } + } + return nullptr; + } + + friend IOutputStream& operator<<(IOutputStream& out, TFederatedDbState const& state); +}; + + +class TFederatedDbObserverImpl : public TClientImplCommon, + public NTopic::TEnableSelfContext { +public: + static constexpr TDuration REDISCOVER_DELAY = TDuration::Seconds(60); + +public: + TFederatedDbObserverImpl(std::shared_ptr connections, const TFederatedTopicClientSettings& settings); + + ~TFederatedDbObserverImpl(); + + std::shared_ptr GetState(); + + NThreading::TFuture WaitForFirstState(); + + void Start(); + void Stop(); + + bool IsStale() const; + +private: + Ydb::FederationDiscovery::ListFederationDatabasesRequest ComposeRequest() const; + void RunFederationDiscoveryImpl(); + void ScheduleFederationDiscoveryImpl(TDuration delay); + void OnFederationDiscovery(TStatus&& status, Ydb::FederationDiscovery::ListFederationDatabasesResult&& result); + +private: + std::shared_ptr FederatedDbState; + NThreading::TPromise PromiseToInitState; + TRpcRequestSettings RpcSettings; + TSpinLock Lock; + + NTopic::IRetryPolicy::TPtr FederationDiscoveryRetryPolicy; + NTopic::IRetryPolicy::IRetryState::TPtr FederationDiscoveryRetryState; + NYdbGrpc::IQueueClientContextPtr FederationDiscoveryDelayContext; + + bool Stopping = false; +}; + +class TFederatedDbObserver : public NTopic::TContextOwner { +public: + inline TFederatedDbObserver(std::shared_ptr connections, + const TFederatedTopicClientSettings& settings) + : TContextOwner(connections, settings) { + } + + inline std::shared_ptr GetState() { + return TryGetImpl()->GetState(); + } + + inline NThreading::TFuture WaitForFirstState() { + return TryGetImpl()->WaitForFirstState(); + } + + inline void Start() { + return TryGetImpl()->Start(); + } + + inline void Stop() { + return TryGetImpl()->Stop(); + } + + inline bool IsStale() const { + return TryGetImpl()->IsStale(); + } +}; + +} // namespace NYdb::V3::NFederatedTopic diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/impl/ya.make b/ydb/public/sdk/cpp/src/client/federated_topic/impl/ya.make new file mode 100644 index 000000000000..6a102764f98f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/impl/ya.make @@ -0,0 +1,33 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + federated_read_session.h + federated_read_session.cpp + federated_read_session_event.cpp + federated_write_session.h + federated_write_session.cpp + federated_topic_impl.h + federated_topic_impl.cpp + federated_topic.cpp + federation_observer.h + federation_observer.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client + library/cpp/monlib/dynamic_counters + library/cpp/monlib/metrics + library/cpp/string_utils/url + ydb/public/sdk/cpp/src/library/persqueue/obfuscate + ydb/public/api/grpc/draft + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic/impl + ydb/public/sdk/cpp/src/client/proto +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/src/client/federated_topic/ut/basic_usage_ut.cpp similarity index 94% rename from ydb/public/sdk/cpp/client/ydb_federated_topic/ut/basic_usage_ut.cpp rename to ydb/public/sdk/cpp/src/client/federated_topic/ut/basic_usage_ut.cpp index 909fd7bd9c49..d7dd6e5d4992 100644 --- a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/federated_topic/ut/basic_usage_ut.cpp @@ -1,16 +1,16 @@ -#include -#include +#include +#include -#include +#include -#include +#include -#include -#include -#include +#include +#include +#include -#include -#include +#include +#include #include #include @@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -58,7 +58,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { Cerr << "Session was created" << Endl; ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - TMaybe event = ReadSession->GetEvent(false); + std::optional event = ReadSession->GetEvent(false); Y_ASSERT(!event); auto fdsRequest = fdsMock.GetNextPendingRequest(); @@ -99,7 +99,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { for (size_t i = 0; i < partitionsCount; ++i) { ReadSession->WaitEvent().Wait(); // Get event - TMaybe event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); + std::optional event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); Cerr << "Got new read session event: " << DebugString(*event) << Endl; auto* startPartitionSessionEvent = std::get_if(&*event); @@ -126,7 +126,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -170,7 +170,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings() .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( @@ -215,7 +215,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings() .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( @@ -324,7 +324,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings() .RetryPolicy(NTopic::IRetryPolicy::GetNoRetryPolicy()); @@ -343,14 +343,14 @@ Y_UNIT_TEST_SUITE(BasicUsage) { ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); auto event = ReadSession->GetEvent(false); - UNIT_ASSERT(!event.Defined()); + UNIT_ASSERT(!event.has_value()); auto fdsRequest = fdsMock.WaitNextPendingRequest(); fdsRequest.Result.SetValue({{}, grpc::Status(grpc::StatusCode::UNAVAILABLE, "mock 'unavailable'")}); ReadSession->WaitEvent().Wait(); event = ReadSession->GetEvent(false); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); Cerr << ">>> Got event: " << DebugString(*event) << Endl; UNIT_ASSERT(std::holds_alternative(*event)); @@ -359,13 +359,13 @@ Y_UNIT_TEST_SUITE(BasicUsage) { ReadSession2->WaitEvent().Wait(TDuration::Seconds(1)); event = ReadSession2->GetEvent(false); - UNIT_ASSERT(!event.Defined()); + UNIT_ASSERT(!event.has_value()); fdsRequest = fdsMock.WaitNextPendingRequest(); fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); event = ReadSession2->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); Cerr << ">>> Got event: " << DebugString(*event) << Endl; UNIT_ASSERT(std::holds_alternative(*event)); @@ -393,7 +393,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { Cerr << "Session was created" << Endl; ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - TMaybe event = ReadSession->GetEvent(false); + std::optional event = ReadSession->GetEvent(false); Y_ASSERT(event); Cerr << "Got new read session event: " << DebugString(*event) << Endl; @@ -418,7 +418,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings(); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); @@ -434,7 +434,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { Cerr << "Session was created" << Endl; ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - TMaybe event = ReadSession->GetEvent(false); + std::optional event = ReadSession->GetEvent(false); Y_ASSERT(!event); { @@ -449,7 +449,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { } ReadSession->WaitEvent().Wait(); - TMaybe event2 = ReadSession->GetEvent(true); + std::optional event2 = ReadSession->GetEvent(true); Cerr << "Got new read session event: " << DebugString(*event2) << Endl; auto* sessionEvent = std::get_if(&*event2); @@ -475,7 +475,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings() .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( @@ -568,7 +568,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); auto clientSettings = TFederatedTopicClientSettings() .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( @@ -613,10 +613,10 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto& message = messages[i]; UNIT_ASSERT(message.GetFederatedPartitionSession()->GetReadSourceDatabaseName() == "dc1"); UNIT_ASSERT(message.GetFederatedPartitionSession()->GetTopicPath() == setup->GetTestTopic()); - UNIT_ASSERT(message.GetData().EndsWith(message.GetFederatedPartitionSession()->GetTopicOriginDatabaseName())); + UNIT_ASSERT(message.GetData().ends_with(message.GetFederatedPartitionSession()->GetTopicOriginDatabaseName())); UNIT_ASSERT(!sentSet.empty()); - UNIT_ASSERT_C(sentSet.erase(message.GetData()), "no such element is sentSet: " + message.GetData()); + UNIT_ASSERT_C(sentSet.erase(TString{message.GetData()}), "no such element is sentSet: " + message.GetData()); totalReceived++; } if (totalReceived == 3 * sentMessages.size()) { @@ -719,7 +719,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -787,7 +787,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -821,7 +821,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -878,7 +878,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); TFederatedTopicClientSettings clientSettings; clientSettings.RetryPolicy(NPersQueue::IRetryPolicy::GetNoRetryPolicy()); @@ -934,7 +934,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); @@ -1017,7 +1017,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); TFederatedTopicClientSettings clientSettings; NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); @@ -1036,13 +1036,13 @@ Y_UNIT_TEST_SUITE(BasicUsage) { { auto e = WriteSession->GetEvent(true); - UNIT_ASSERT(e.Defined()); + UNIT_ASSERT(e.has_value()); Cerr << ">>> Got event: " << DebugString(*e) << Endl; UNIT_ASSERT(std::holds_alternative(*e)); } { auto e = WriteSession->GetEvent(true); - UNIT_ASSERT(e.Defined()); + UNIT_ASSERT(e.has_value()); Cerr << ">>> Got event: " << DebugString(*e) << Endl; UNIT_ASSERT(std::holds_alternative(*e)); } @@ -1052,7 +1052,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NTopic::TContinuationToken GetToken(std::shared_ptr writer) { auto e = writer->GetEvent(true); - UNIT_ASSERT(e.Defined()); + UNIT_ASSERT(e.has_value()); Cerr << ">>> Got event: " << DebugString(*e) << Endl; auto* readyToAcceptEvent = std::get_if(&*e); UNIT_ASSERT(readyToAcceptEvent); @@ -1076,7 +1076,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto driverConfig = NYdb::TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << newServicePort) .SetDatabase("/Root") - .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); auto driver = NYdb::TDriver(driverConfig); auto topicClient = NYdb::NFederatedTopic::TFederatedTopicClient(driver); @@ -1169,7 +1169,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << setup->GetGrpcPort()); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); NYdb::TDriver driver(cfg); NYdb::NFederatedTopic::TFederatedTopicClient client(driver); diff --git a/ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/fds_mock.h b/ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/fds_mock.h similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/fds_mock.h rename to ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/fds_mock.h diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/ya.make b/ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/ya.make new file mode 100644 index 000000000000..c2d5ffb21406 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + fds_mock.h +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/ut/ya.make b/ydb/public/sdk/cpp/src/client/federated_topic/ut/ya.make new file mode 100644 index 000000000000..cad268660693 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/ut/ya.make @@ -0,0 +1,38 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + library/cpp/testing/gmock_in_unittest + ydb/core/testlib/default + ydb/public/lib/json_value + ydb/public/lib/yson_value + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public/include + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/topic/impl + + ydb/public/sdk/cpp/src/client/federated_topic + ydb/public/sdk/cpp/src/client/federated_topic/impl +) + +YQL_LAST_ABI_VERSION() + +SRCS( + basic_usage_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/federated_topic/ya.make b/ydb/public/sdk/cpp/src/client/federated_topic/ya.make new file mode 100644 index 000000000000..b58abff2e196 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/federated_topic/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h) + +SRCS( + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/federated_topic/impl + ydb/public/sdk/cpp/src/client/federated_topic/ut/fds_mock +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/sdk/cpp/src/client/helpers/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/helpers/CMakeLists.txt new file mode 100644 index 000000000000..dc47a456af98 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/helpers/CMakeLists.txt @@ -0,0 +1,16 @@ +_ydb_sdk_add_library(client-helpers) + +target_link_libraries(client-helpers + PUBLIC + yutil + client-ydb_types-credentials-oauth2 + client-iam + client-ydb_types-credentials +) + +target_sources(client-helpers + PRIVATE + helpers.cpp +) + +_ydb_sdk_make_client_component(Helpers client-helpers) diff --git a/ydb/public/sdk/cpp/src/client/helpers/helpers.cpp b/ydb/public/sdk/cpp/src/client/helpers/helpers.cpp new file mode 100644 index 000000000000..cd5e8b10c70d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/helpers/helpers.cpp @@ -0,0 +1,77 @@ +#include + +#include +#include +#include + +#include +#include + +#include + +namespace NYdb::inline V3 { + +TDriverConfig CreateFromEnvironment(const std::string& connectionString) { + TDriverConfig driverConfig; + if (connectionString != "") { + auto connectionInfo = ParseConnectionString(connectionString); + driverConfig.SetEndpoint(connectionInfo.Endpoint); + driverConfig.SetDatabase(connectionInfo.Database); + if (connectionInfo.EnableSsl) { + std::string rootCertsFile = GetStrFromEnv("YDB_SSL_ROOT_CERTIFICATES_FILE", ""); + std::string rootCerts = GetRootCertificate(); + if (rootCertsFile != "") { + TFileInput in(rootCertsFile.c_str()); + rootCerts += in.ReadAll(); + } + driverConfig.UseSecureConnection(rootCerts); + } + } + + std::string saKeyFile = GetStrFromEnv("YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS", ""); + if (!saKeyFile.empty()) { + driverConfig.SetCredentialsProviderFactory( + CreateIamJwtFileCredentialsProviderFactory({.JwtFilename = saKeyFile})); + return driverConfig; + } + bool anonymousUsed = GetStrFromEnv("YDB_ANONYMOUS_CREDENTIALS", "0") == "1"; + if (anonymousUsed) { + return driverConfig; + } + + bool useMetadataCredentials = GetStrFromEnv("YDB_METADATA_CREDENTIALS", "0") == "1"; + if (useMetadataCredentials) { + auto factory = CreateIamCredentialsProviderFactory(); + try { + factory->CreateProvider(); + } catch (yexception& e) { + ythrow yexception() << "Unable to get token from metadata service: " << e.what(); + } + driverConfig.SetCredentialsProviderFactory(factory); + return driverConfig; + } + + std::string accessToken = GetStrFromEnv("YDB_ACCESS_TOKEN_CREDENTIALS", ""); + if (accessToken != "") { + driverConfig.SetAuthToken(accessToken); + return driverConfig; + } + + std::string oauth2KeyFile = GetStrFromEnv("YDB_OAUTH2_KEY_FILE", ""); + if (!oauth2KeyFile.empty()) { + driverConfig.SetCredentialsProviderFactory( + CreateOauth2TokenExchangeFileCredentialsProviderFactory(oauth2KeyFile)); + return driverConfig; + } + + ythrow yexception() << "Unable to create driver config from environment"; +} + +TDriverConfig CreateFromSaKeyFile(const std::string& saKeyFile, const std::string& connectionString) { + TDriverConfig driverConfig(connectionString); + driverConfig.SetCredentialsProviderFactory( + CreateIamJwtFileCredentialsProviderFactory({.JwtFilename = saKeyFile})); + return driverConfig; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/helpers/ya.make b/ydb/public/sdk/cpp/src/client/helpers/ya.make new file mode 100644 index 000000000000..e1b7b97088ed --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/helpers/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + helpers.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/iam + ydb/public/sdk/cpp/src/client/types/credentials + ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/iam/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/iam/CMakeLists.txt new file mode 100644 index 000000000000..4dd6d707856b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/CMakeLists.txt @@ -0,0 +1,20 @@ +add_subdirectory(common) + +_ydb_sdk_add_library(client-iam) + +target_link_libraries(client-iam + PUBLIC + client-iam-types + yutil + PRIVATE + api-client-yc_public + client-iam-common + json + http-simple +) + +target_sources(client-iam PRIVATE + iam.cpp +) + +_ydb_sdk_make_client_component(Iam client-iam) diff --git a/ydb/public/sdk/cpp/src/client/iam/common/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/iam/common/CMakeLists.txt new file mode 100644 index 000000000000..2d50e166c43e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/common/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(client-iam-common INTERFACE) + +target_link_libraries(client-iam-common + INTERFACE + client-iam-types + grpc-client + threading-future + yutil +) + +_ydb_sdk_install_targets(client-iam-common) diff --git a/ydb/public/sdk/cpp/src/client/iam/common/iam.h b/ydb/public/sdk/cpp/src/client/iam/common/iam.h new file mode 100644 index 000000000000..c7f7742c5a5d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/common/iam.h @@ -0,0 +1,228 @@ +#pragma once + +#include + +#include + +#include + +#include +#include + +namespace NYdb::inline V3 { + +constexpr TDuration BACKOFF_START = TDuration::MilliSeconds(50); +constexpr TDuration BACKOFF_MAX = TDuration::Seconds(10); + +template +class TGrpcIamCredentialsProvider : public ICredentialsProvider { +protected: + using TRequestFiller = std::function; + +private: + class TImpl : public std::enable_shared_from_this::TImpl> { + public: + TImpl(const TIamEndpoint& iamEndpoint, const TRequestFiller& requestFiller) + : Client(std::make_unique()) + , Connection_(nullptr) + , Ticket_("") + , NextTicketUpdate_(TInstant::Zero()) + , IamEndpoint_(iamEndpoint) + , RequestFiller_(requestFiller) + , RequestInflight_(false) + , LastRequestError_("") + , NeedStop_(false) + , BackoffTimeout_(BACKOFF_START) + , Lock_() + { + NYdbGrpc::TGRpcClientConfig grpcConf; + grpcConf.Locator = IamEndpoint_.Endpoint; + grpcConf.EnableSsl = IamEndpoint_.EnableSsl; + Connection_ = std::unique_ptr>(Client->CreateGRpcServiceConnection(grpcConf).release()); + } + + void UpdateTicket(bool sync = false) { + { + std::lock_guard guard(Lock_); + if (NeedStop_ || RequestInflight_) { + return; + } + RequestInflight_ = true; + } + + auto resultPromise = NThreading::NewPromise(); + + std::shared_ptr self = TGrpcIamCredentialsProvider::TImpl::shared_from_this(); + + auto cb = [self, resultPromise, sync]( + NYdbGrpc::TGrpcStatus&& status, TResponse&& result) mutable { + self->ProcessIamResponse(std::move(status), std::move(result), sync); + resultPromise.SetValue(); + }; + + TRequest req; + + RequestFiller_(req); + + Connection_->template DoRequest( + std::move(req), + std::move(cb), + &TService::Stub::AsyncCreate, + { {}, {}, IamEndpoint_.RequestTimeout } + ); + + if (sync) { + resultPromise.GetFuture().Wait(2 * IamEndpoint_.RequestTimeout); + } + } + + std::string GetTicket() { + TInstant nextTicketUpdate; + std::string ticket; + { + std::lock_guard guard(Lock_); + ticket = Ticket_; + nextTicketUpdate = NextTicketUpdate_; + if (ticket.empty()) + ythrow yexception() << "IAM-token not ready yet. " << LastRequestError_; + } + if (TInstant::Now() >= nextTicketUpdate) { + UpdateTicket(); + } + return ticket; + } + + void Stop() { + { + std::lock_guard guard(Lock_); + if (NeedStop_) { + return; + } + NeedStop_ = true; + } + + Client.reset(); // Will trigger destroy + } + + private: + void ProcessIamResponse(NYdbGrpc::TGrpcStatus&& status, TResponse&& result, bool sync) { + if (!status.Ok()) { + TDuration sleepDuration; + { + std::lock_guard guard(Lock_); + LastRequestError_ = TStringBuilder() + << "Last request error was at " << TInstant::Now() + << ". GrpcStatusCode: " << status.GRpcStatusCode + << " Message: \"" << status.Msg + << "\" internal: " << status.InternalError + << " iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; + + RequestInflight_ = false; + sleepDuration = std::min(BackoffTimeout_, BACKOFF_MAX); + BackoffTimeout_ *= 2; + } + + Sleep(sleepDuration); + + UpdateTicket(sync); + } else { + std::lock_guard guard(Lock_); + LastRequestError_ = ""; + Ticket_ = result.iam_token(); + RequestInflight_ = false; + BackoffTimeout_ = BACKOFF_START; + + const auto now = Now(); + NextTicketUpdate_ = std::min( + now + IamEndpoint_.RefreshPeriod, + TInstant::Seconds(result.expires_at().seconds()) + ) - IamEndpoint_.RequestTimeout; + NextTicketUpdate_ = std::max(NextTicketUpdate_, now + TDuration::MilliSeconds(100)); + } + } + + private: + + std::unique_ptr Client; + std::unique_ptr> Connection_; + std::string Ticket_; + TInstant NextTicketUpdate_; + const TIamEndpoint IamEndpoint_; + const TRequestFiller RequestFiller_; + bool RequestInflight_; + std::string LastRequestError_; + bool NeedStop_; + TDuration BackoffTimeout_; + TAdaptiveLock Lock_; + }; + +public: + TGrpcIamCredentialsProvider(const TIamEndpoint& endpoint, const TRequestFiller& requestFiller) + : Impl_(std::make_shared(endpoint, requestFiller)) + { + Impl_->UpdateTicket(true); + } + + ~TGrpcIamCredentialsProvider() { + Impl_->Stop(); + } + + std::string GetAuthInfo() const override { + return Impl_->GetTicket(); + } + + bool IsValid() const override { + return true; + } + +private: + std::shared_ptr Impl_; +}; + +template +class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider { +public: + TIamJwtCredentialsProvider(const TIamJwtParams& params) + : TGrpcIamCredentialsProvider(params, + [jwtParams = params.JwtParams](TRequest& req) { + req.set_jwt(MakeSignedJwt(jwtParams)); + }) {} +}; + +template +class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider { +public: + TIamOAuthCredentialsProvider(const TIamOAuth& params) + : TGrpcIamCredentialsProvider(params, + [token = params.OAuthToken](TRequest& req) { + req.set_yandex_passport_oauth_token(TStringType{token}); + }) {} +}; + +template +class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TIamJwtCredentialsProviderFactory(const TIamJwtParams& params): Params_(params) {} + + TCredentialsProviderPtr CreateProvider() const final { + return std::make_shared>(Params_); + } + +private: + TIamJwtParams Params_; +}; + +template +class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TIamOAuthCredentialsProviderFactory(const TIamOAuth& params): Params_(params) {} + + TCredentialsProviderPtr CreateProvider() const final { + return std::make_shared>(Params_); + } + +private: + TIamOAuth Params_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/iam/common/ya.make b/ydb/public/sdk/cpp/src/client/iam/common/ya.make new file mode 100644 index 000000000000..f27a760e584f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/common/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + iam.h +) + +PEERDIR( + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common + ydb/public/sdk/cpp/src/library/grpc/client + library/cpp/threading/future +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/iam/iam.cpp b/ydb/public/sdk/cpp/src/client/iam/iam.cpp new file mode 100644 index 000000000000..fff7b5f63bd7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/iam.cpp @@ -0,0 +1,113 @@ +#include + +#include "common/iam.h" + +#include +#include + +#include +#include + +using namespace yandex::cloud::iam::v1; + +namespace NYdb::inline V3 { + +class TIAMCredentialsProvider : public ICredentialsProvider { +public: + TIAMCredentialsProvider(const TIamHost& params) + : HttpClient_(TSimpleHttpClient(TString(params.Host), params.Port)) + , Request_("/computeMetadata/v1/instance/service-accounts/default/token") + , NextTicketUpdate_(TInstant::Zero()) + , RefreshPeriod_(params.RefreshPeriod) + { + GetTicket(); + } + + std::string GetAuthInfo() const override { + if (TInstant::Now() >= NextTicketUpdate_) { + GetTicket(); + } + return Ticket_; + } + + bool IsValid() const override { + return true; + } + +private: + TSimpleHttpClient HttpClient_; + std::string Request_; + mutable std::string Ticket_; + mutable TInstant NextTicketUpdate_; + TDuration RefreshPeriod_; + + void GetTicket() const { + try { + TStringStream out; + TSimpleHttpClient::THeaders headers; + headers["Metadata-Flavor"] = "Google"; + HttpClient_.DoGet(Request_, &out, headers); + NJson::TJsonValue resp; + NJson::ReadJsonTree(&out, &resp, true); + + auto respMap = resp.GetMap(); + + if (auto it = respMap.find("access_token"); it == respMap.end()) + ythrow yexception() << "Result doesn't contain access_token"; + else if (std::string ticket = it->second.GetStringSafe(); ticket.empty()) + ythrow yexception() << "Got empty ticket"; + else + Ticket_ = std::move(ticket); + + if (auto it = respMap.find("expires_in"); it == respMap.end()) + ythrow yexception() << "Result doesn't contain expires_in"; + else { + const TDuration expiresIn = TDuration::Seconds(it->second.GetUInteger()) / 2; + + const auto interval = std::max(std::min(expiresIn, RefreshPeriod_), TDuration::MilliSeconds(100)); + + NextTicketUpdate_ = TInstant::Now() + interval; + } + } catch (...) { + } + } +}; + +class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TIamCredentialsProviderFactory(const TIamHost& params): Params_(params) {} + + TCredentialsProviderPtr CreateProvider() const final { + return std::make_shared(Params_); + } + +private: + TIamHost Params_; +}; + +/// Acquire an IAM token using a local metadata service on a virtual machine. +TCredentialsProviderFactoryPtr CreateIamCredentialsProviderFactory(const TIamHost& params ) { + return std::make_shared(params); +} + +TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactory(const TIamJwtFilename& params) { + TIamJwtParams jwtParams = { params, ReadJwtKeyFile(params.JwtFilename) }; + return std::make_shared>(std::move(jwtParams)); +} + +TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactory(const TIamJwtContent& params) { + TIamJwtParams jwtParams = { params, ParseJwtParams(params.JwtContent) }; + return std::make_shared>(std::move(jwtParams)); +} + +TCredentialsProviderFactoryPtr CreateIamOAuthCredentialsProviderFactory(const TIamOAuth& params) { + return std::make_shared>(params); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/iam/ya.make b/ydb/public/sdk/cpp/src/client/iam/ya.make new file mode 100644 index 000000000000..449886b88d86 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + iam.cpp +) + +PEERDIR( + library/cpp/http/simple + library/cpp/json + ydb/public/api/client/yc_public/iam + ydb/public/sdk/cpp/src/client/iam/common +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/iam_private/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/iam_private/CMakeLists.txt new file mode 100644 index 000000000000..981a26a58c18 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam_private/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-iam_private) + +target_link_libraries(client-iam_private + PUBLIC + client-iam-types + yutil + PRIVATE + api-client-yc_private + client-iam-common +) + +target_sources(client-iam_private + PRIVATE + iam.cpp +) + +_ydb_sdk_make_client_component(IamPrivate client-iam_private) diff --git a/ydb/public/sdk/cpp/src/client/iam_private/iam.cpp b/ydb/public/sdk/cpp/src/client/iam_private/iam.cpp new file mode 100644 index 000000000000..09d7e56683f5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam_private/iam.cpp @@ -0,0 +1,28 @@ +#include + +#include + +#include +#include + +namespace NYdb::inline V3 { + +TCredentialsProviderFactoryPtr CreateIamJwtCredentialsProviderFactoryImplPrivate(TIamJwtParams&& jwtParams) { + return std::make_shared>(std::move(jwtParams)); +} + +TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactoryPrivate(const TIamJwtFilename& params) { + TIamJwtParams jwtParams = { params, ReadJwtKeyFile(params.JwtFilename) }; + return CreateIamJwtCredentialsProviderFactoryImplPrivate(std::move(jwtParams)); +} + +TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactoryPrivate(const TIamJwtContent& params) { + TIamJwtParams jwtParams = { params, ParseJwtParams(params.JwtContent) }; + return CreateIamJwtCredentialsProviderFactoryImplPrivate(std::move(jwtParams)); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/iam_private/ya.make b/ydb/public/sdk/cpp/src/client/iam_private/ya.make new file mode 100644 index 000000000000..251d3483071f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/iam_private/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + iam.cpp +) + +PEERDIR( + ydb/public/api/client/yc_private/iam + ydb/public/sdk/cpp/src/client/iam/common +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/CMakeLists.txt new file mode 100644 index 000000000000..05e626b4d059 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(ydb_endpoints) +add_subdirectory(ydb_internal) +add_subdirectory(ydb_stats) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/CMakeLists.txt new file mode 100644 index 000000000000..5889276c7cb2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(client-impl-ydb_endpoints) + +target_link_libraries(client-impl-ydb_endpoints PUBLIC + yutil + monlib-metrics + api-grpc +) + +target_sources(client-impl-ydb_endpoints PRIVATE + endpoints.cpp +) + +_ydb_sdk_install_targets(TARGETS client-impl-ydb_endpoints) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.cpp new file mode 100644 index 000000000000..40f9b8402fc6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.cpp @@ -0,0 +1,273 @@ +#include "endpoints.h" + +#include +#include + +#include + +#include +#include + +namespace NYdb::inline V3 { + +using std::string; + +class TEndpointElectorSafe::TObjRegistry : public IObjRegistryHandle { +public: + TObjRegistry(const ui64& nodeId) + : NodeId_(nodeId) + {} + + bool Add(TEndpointObj* obj) { + std::unique_lock lock(Mutex_); + return Objs_.insert(obj).second; + } + + void Remove(TEndpointObj* obj) { + std::unique_lock lock(Mutex_); + Y_ABORT_UNLESS(Objs_.find(obj) != Objs_.end()); + Objs_.erase(obj); + } + + void NotifyEndpointRemoved() { + std::shared_lock lock(Mutex_); + for (auto obj : Objs_) { + obj->OnEndpointRemoved(); + } + } + + size_t Size() const override { + std::shared_lock lock(Mutex_); + return Objs_.size(); + } + + ui64 GetNodeId() const { + return NodeId_; + } + +private: + std::set Objs_; + ui64 NodeId_; + + mutable std::shared_mutex Mutex_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Returns index of last resord with same priority or -1 in case of empty input +static i32 GetBestK(const std::vector& records) { + if (records.empty()) { + return -1; + } + + const i32 bestPriority = records[0].Priority; + + size_t pos = 1; + while (pos < records.size()) { + if (records[pos].Priority != bestPriority) { + break; + } + ++pos; + } + return pos - 1; +} + +std::vector TEndpointElectorSafe::SetNewState(std::vector&& records) { + std::unordered_set index; + std::vector uniqRec; + + for (auto&& record : records) { + if (index.insert(record.Endpoint).second) { + uniqRec.emplace_back(std::move(record)); + } + } + + Sort(uniqRec.begin(), uniqRec.end()); + + auto bestK = GetBestK(uniqRec); + + std::vector removed; + std::vector> notifyRemoved; + + { + std::unique_lock guard(Mutex_); + // Find endpoins which were removed + for (const auto& record : Records_) { + if (index.find(record.Endpoint) == index.end()) { + removed.emplace_back(record.Endpoint); + + auto it = KnownEndpoints_.find(record.Endpoint); + Y_ABORT_UNLESS(it != KnownEndpoints_.end()); + KnownEndpoints_.erase(it); + + auto nodeIdIt = KnownEndpointsByNodeId_.find(record.NodeId); + if (nodeIdIt != KnownEndpointsByNodeId_.end()) { + for (const auto& registry : nodeIdIt->second.TaggedObjs) { + notifyRemoved.emplace_back(registry.second); + } + KnownEndpointsByNodeId_.erase(nodeIdIt); + } + } + } + // Find endpoints which were added + Records_ = std::move(uniqRec); + for (const auto& record : Records_) { + KnownEndpoints_[record.Endpoint] = record; + KnownEndpointsByNodeId_[record.NodeId].Record = record; + } + Y_ABORT_UNLESS(Records_.size() == KnownEndpoints_.size()); + EndpointCountGauge_.SetValue(Records_.size()); + EndpointActiveGauge_.SetValue(Records_.size()); + BestK_ = bestK; + PessimizationRatio_.store(0); + PessimizationRatioGauge_.SetValue(0); + } + + for (auto& obj : notifyRemoved) { + obj->NotifyEndpointRemoved(); + } + + return removed; +} + +TEndpointRecord TEndpointElectorSafe::GetEndpoint(const TEndpointKey& preferredEndpoint, bool onlyPreferred) const { + std::shared_lock guard(Mutex_); + + if (preferredEndpoint.GetNodeId()) { + auto it = KnownEndpointsByNodeId_.find(preferredEndpoint.GetNodeId()); + if (it != KnownEndpointsByNodeId_.end()) { + return it->second.Record; + } + } + + if (!preferredEndpoint.GetEndpoint().empty()) { + auto it = KnownEndpoints_.find(preferredEndpoint.GetEndpoint()); + if (it != KnownEndpoints_.end()) { + return it->second; + } + } + + if(onlyPreferred) + return {}; + + if (BestK_ == -1) { + Y_ASSERT(Records_.empty()); + return {}; + } else { + // returns value in range [0, n) + auto idx = RandomNumber(BestK_ + 1); + return Records_[idx]; + } +} + +// TODO: Suboptimal, but should not be used often +void TEndpointElectorSafe::PessimizeEndpoint(const string& endpoint) { + std::unique_lock guard(Mutex_); + for (auto& r : Records_) { + if (r.Endpoint == endpoint && r.Priority != Max()) { + int pessimizationRatio = PessimizationRatio_.load(); + auto newRatio = (pessimizationRatio * Records_.size() + 100) / Records_.size(); + PessimizationRatio_.store(newRatio); + PessimizationRatioGauge_.SetValue(newRatio); + EndpointActiveGauge_.Dec(); + r.Priority = Max(); + + auto it = KnownEndpoints_.find(endpoint); + if (it != KnownEndpoints_.end()) { + it->second.Priority = Max(); + } + } + } + Sort(Records_.begin(), Records_.end()); + BestK_ = GetBestK(Records_); +} + +// % of endpoints which was pessimized +int TEndpointElectorSafe::GetPessimizationRatio() const { + return PessimizationRatio_.load(); +} + +void TEndpointElectorSafe::SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector) { + EndpointCountGauge_.Set(endpointStatCollector.EndpointCount); + PessimizationRatioGauge_.Set(endpointStatCollector.PessimizationRatio); + EndpointActiveGauge_.Set(endpointStatCollector.EndpointActive); +} + +bool TEndpointElectorSafe::LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag) { + { + std::unique_lock guard(Mutex_); + // Find obj registry for given endpoint + // No endpoint - no registry, return false + auto objIt = KnownEndpointsByNodeId_.find(endpoint.GetNodeId()); + if (objIt == KnownEndpointsByNodeId_.end()) { + return false; + } + + TTaggedObjRegistry& taggedObjs = objIt->second.TaggedObjs; + TTaggedObjRegistry::iterator registryIt = taggedObjs.find(tag); + + if (registryIt == taggedObjs.end()) { + registryIt = taggedObjs.emplace(tag, new TObjRegistry(endpoint.GetNodeId())).first; + } + + // Call Link under endpoint elector mutex. + // Probably a bit more optimal way is: + // - get TObjRegistry (as shared ptr) and release this mutex + // - call obj->Link whithout mutex + // - check KnownEndpoints_ stil has same TObjRegistry for given endpoint + // - in case of false send notification + return obj->Link(registryIt->second); + } +} + +void TEndpointElectorSafe::ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const { + std::shared_lock guard(Mutex_); + + auto it = std::lower_bound(Records_.begin(), Records_.end(), minPriority, [](const TEndpointRecord& l, i32 r) { + return l.Priority < r; + }); + + while (it != Records_.end()) { + if (it->Priority > maxPriority) + break; + + const TTaggedObjRegistry& taggedObjs = KnownEndpointsByNodeId_.at(it->NodeId).TaggedObjs; + + auto registry = taggedObjs.find(tag); + if (registry != taggedObjs.end()) { + cb(it->NodeId, *registry->second); + } else { + cb(it->NodeId, TObjRegistry(it->NodeId)); + } + + it++; + } +} + +void TEndpointObj::Unlink() { + if (ObjRegistry_) { + ObjRegistry_->Remove(this); + } + ObjRegistry_ = nullptr; +} + +bool TEndpointObj::Link(std::shared_ptr registry) { + if (registry->Add(this)) { + if (ObjRegistry_) { + ObjRegistry_->Remove(this); + } + ObjRegistry_ = registry; + return true; + } + return false; +} + +size_t TEndpointObj::ObjectCount() const { + return ObjRegistry_->Size(); +} + +bool TEndpointObj::ObjectRegistred() const { + return bool(ObjRegistry_); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.h b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.h new file mode 100644 index 000000000000..ec6225311e65 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/endpoints.h @@ -0,0 +1,150 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace NYdb::inline V3 { + +struct TEndpointRecord { + std::string Endpoint; + i32 Priority; + std::string SslTargetNameOverride; + ui64 NodeId = 0; + + TEndpointRecord() + : Endpoint() + , Priority(0) + , SslTargetNameOverride() + , NodeId(0) + { + } + + TEndpointRecord(std::string endpoint, i32 priority, std::string sslTargetNameOverride = std::string(), ui64 nodeId = 0) + : Endpoint(std::move(endpoint)) + , Priority(priority) + , SslTargetNameOverride(std::move(sslTargetNameOverride)) + , NodeId(nodeId) + { + } + + explicit operator bool() const { + return !Endpoint.empty(); + } + + bool operator<(const TEndpointRecord& rhs) const { + return Priority < rhs.Priority; + } +}; + +struct TEndpointKey { + std::string Endpoint; + ui64 NodeId = 0; + + TEndpointKey() + : Endpoint() + , NodeId(0) + {} + + TEndpointKey(std::string endpoint, ui64 nodeId) + : Endpoint(std::move(endpoint)) + , NodeId(nodeId) + {} + + TEndpointKey(ui64 nodeId) + : Endpoint() + , NodeId(nodeId) + {} + + const std::string& GetEndpoint() const { + return Endpoint; + } + + const ui64& GetNodeId() const { + return NodeId; + } + + friend IOutputStream& operator<<(IOutputStream& out, const TEndpointKey& value) + { + out << "{ Endpoint: " << value.Endpoint << ", NodeId: " << value.NodeId << "}"; + return out; + } +}; + +class IObjRegistryHandle { +public: + virtual ~IObjRegistryHandle() = default; + virtual size_t Size() const = 0; +}; + +class TEndpointObj; +class TEndpointElectorSafe { +public: + TEndpointElectorSafe() = default; + + // Sets new endpoints, returns removed + std::vector SetNewState(std::vector&& records); + + // Allows to get stats + void SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector); + + // Returns preferred (if presents) or best endpoint + TEndpointRecord GetEndpoint(const TEndpointKey& preferredEndpoint, bool onlyPreferred = false) const; + + // Move endpoint to the end + void PessimizeEndpoint(const std::string& endpoint); + + // Returns % of pessimized endpoints + int GetPessimizationRatio() const; + + // Associate object with the endpoint + // Returns false if no required endpoint, or object already registered + bool LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag); + + // Perform some action for each object group associated with endpoint + using THandleCb = std::function; + void ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const; + + class TObjRegistry; +private: + using TTaggedObjRegistry = std::unordered_map>; + + struct TKnownEndpoint { + TEndpointRecord Record; + TTaggedObjRegistry TaggedObjs; + }; + +private: + mutable std::shared_mutex Mutex_; + std::vector Records_; + std::unordered_map KnownEndpoints_; + std::unordered_map KnownEndpointsByNodeId_; + i32 BestK_ = -1; + std::atomic_int PessimizationRatio_ = 0; + NSdkStats::TAtomicCounter<::NMonitoring::TIntGauge> EndpointCountGauge_; + NSdkStats::TAtomicCounter<::NMonitoring::TIntGauge> PessimizationRatioGauge_; + NSdkStats::TAtomicCounter<::NMonitoring::TIntGauge> EndpointActiveGauge_; +}; + +// Used to track object +// The derived class must call Unlink() before destroying +class TEndpointObj { + friend class TEndpointElectorSafe; +public: + virtual ~TEndpointObj() = default; + + virtual void OnEndpointRemoved() {} + + size_t ObjectCount() const; + bool ObjectRegistred() const; + void Unlink(); + +private: + bool Link(std::shared_ptr registry); + std::shared_ptr ObjRegistry_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/ya.make new file mode 100644 index 000000000000..e580efaafdcc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_endpoints/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + endpoints.cpp +) + +PEERDIR( + library/cpp/monlib/metrics + ydb/public/api/grpc +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/CMakeLists.txt new file mode 100644 index 000000000000..5c9ae68e698b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/CMakeLists.txt @@ -0,0 +1,11 @@ +add_subdirectory(common) +add_subdirectory(db_driver_state) +add_subdirectory(grpc_connections) +add_subdirectory(kqp_session_common) +add_subdirectory(logger) +add_subdirectory(make_request) +add_subdirectory(plain_status) +add_subdirectory(retry) +add_subdirectory(session_pool) +add_subdirectory(thread_pool) +add_subdirectory(value_helpers) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/CMakeLists.txt new file mode 100644 index 000000000000..0e21f8a082c3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/CMakeLists.txt @@ -0,0 +1,15 @@ +_ydb_sdk_add_library(impl-ydb_internal-common) + +target_link_libraries(impl-ydb_internal-common PUBLIC + yutil + grpc-client + yql-public-issue +) + +target_sources(impl-ydb_internal-common PRIVATE + parser.cpp + getenv.cpp + client_pid.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-common) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.cpp new file mode 100644 index 000000000000..ec82d6b613f0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.cpp @@ -0,0 +1,33 @@ +#include "client_pid.h" + +#include + +#ifdef _win_ + // copied from util/system/getpid.cpp + // to avoid extra util dep. + #include + #if defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8) + #include + #endif +#else + #include +#endif + +namespace NYdb::inline V3 { + +namespace { +ui32 GetProcessId() { +#ifdef _win_ + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +} + +std::string GetClientPIDHeaderValue() { + return std::to_string(GetProcessId()); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.h new file mode 100644 index 000000000000..b8136e16c041 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/client_pid.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +std::string GetClientPIDHeaderValue(); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.cpp new file mode 100644 index 000000000000..d6d136dbd73f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.cpp @@ -0,0 +1,12 @@ +#include "getenv.h" + +#include + +namespace NYdb::inline V3 { + +std::string GetStrFromEnv(const char* envVarName, const std::string& defaultValue) { + auto envVarPointer = getenv(envVarName); + return envVarPointer ? std::string(envVarPointer) : defaultValue; +} + +} // namespace NYdb \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.h new file mode 100644 index 000000000000..b105bf5e9482 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/getenv.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +std::string GetStrFromEnv(const char* envVarName, const std::string& defaultValue = ""); + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.cpp new file mode 100644 index 000000000000..879048743cbe --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.cpp @@ -0,0 +1,51 @@ +#include "parser.h" + +#include + +namespace NYdb::inline V3 { + +TConnectionInfo ParseConnectionString(const std::string& connectionString) { + if (connectionString.length() == 0) { + ythrow TContractViolation("Empty connection string"); + } + + const std::string databaseFlag = "/?database="; + const std::string grpcProtocol = "grpc://"; + const std::string grpcsProtocol = "grpcs://"; + const std::string localhostDomain = "localhost:"; + + TConnectionInfo connectionInfo; + std::string endpoint; + + size_t pathIndex = connectionString.find(databaseFlag); + if (pathIndex == std::string::npos){ + pathIndex = connectionString.length(); + } + if (pathIndex != connectionString.length()) { + connectionInfo.Database = connectionString.substr(pathIndex + databaseFlag.length()); + endpoint = connectionString.substr(0, pathIndex); + } else { + endpoint = connectionString; + } + + if (!std::string_view{endpoint}.starts_with(grpcProtocol) && !std::string_view{endpoint}.starts_with(grpcsProtocol) && + !std::string_view{endpoint}.starts_with(localhostDomain)) + { + connectionInfo.Endpoint = endpoint; + connectionInfo.EnableSsl = true; + } else if (std::string_view{endpoint}.starts_with(grpcProtocol)) { + connectionInfo.Endpoint = endpoint.substr(grpcProtocol.length()); + connectionInfo.EnableSsl = false; + } else if (std::string_view{endpoint}.starts_with(grpcsProtocol)) { + connectionInfo.Endpoint = endpoint.substr(grpcsProtocol.length()); + connectionInfo.EnableSsl = true; + } else { + connectionInfo.Endpoint = endpoint; + connectionInfo.EnableSsl = false; + } + + return connectionInfo; +} + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.h new file mode 100644 index 000000000000..13df795e6ae5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/parser.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +struct TConnectionInfo { + std::string Endpoint = ""; + std::string Database = ""; + bool EnableSsl = false; +}; + +TConnectionInfo ParseConnectionString(const std::string& connectionString); + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/types.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/types.h new file mode 100644 index 000000000000..7deb54f51182 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/types.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include +#include + +#include + +#include + + +namespace NYdb::inline V3 { + +// Other callbacks +using TSimpleCb = std::function; +using TErrorCb = std::function; + +struct TBalancingSettings { + EBalancingPolicy Policy; + std::string PolicyParams; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/ya.make new file mode 100644 index 000000000000..cf73914736a9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/common/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + parser.cpp + getenv.cpp + client_pid.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client + ydb/public/sdk/cpp/src/library/issue +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/CMakeLists.txt new file mode 100644 index 000000000000..910d497c4625 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/CMakeLists.txt @@ -0,0 +1,19 @@ +_ydb_sdk_add_library(impl-ydb_internal-db_driver_state) + +target_link_libraries(impl-ydb_internal-db_driver_state PUBLIC + yutil + string_utils-quote + threading-future + client-impl-ydb_endpoints + impl-ydb_internal-logger + impl-ydb_internal-plain_status + client-ydb_types-credentials +) + +target_sources(impl-ydb_internal-db_driver_state PRIVATE + authenticator.cpp + endpoint_pool.cpp + state.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-db_driver_state) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.cpp new file mode 100644 index 000000000000..c0bfce96e0eb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.cpp @@ -0,0 +1,34 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "authenticator.h" + +#include + +namespace NYdb::inline V3 { + +TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr credentialsProvider) + : CredentialsProvider_(std::move(credentialsProvider)) +{} + +grpc::Status TYdbAuthenticator::GetMetadata( + grpc::string_ref, + grpc::string_ref, + const grpc::AuthContext&, + std::multimap* metadata +) { + try { + metadata->insert(std::make_pair(YDB_AUTH_TICKET_HEADER, CredentialsProvider_->GetAuthInfo())); + } catch (const std::exception& e) { + return grpc::Status( + grpc::StatusCode::UNAUTHENTICATED, + "Can't get Authentication info from CredentialsProvider", + e.what() + ); + } + return grpc::Status::OK; +} + +bool TYdbAuthenticator::IsBlocking() const { + return false; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.h new file mode 100644 index 000000000000..ff0ecb0fb265 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/authenticator.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin { +public: + TYdbAuthenticator(std::shared_ptr credentialsProvider); + + grpc::Status GetMetadata( + grpc::string_ref, + grpc::string_ref, + const grpc::AuthContext&, + std::multimap* metadata + ) override; + + bool IsBlocking() const override; + +private: + std::shared_ptr CredentialsProvider_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp new file mode 100644 index 000000000000..50432a49bc34 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp @@ -0,0 +1,191 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "endpoint_pool.h" + +namespace NYdb::inline V3 { + +using std::string; +using std::vector; + +TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client) + : Provider_(provider) + , LastUpdateTime_(TInstant::Zero().MicroSeconds()) + , BalancingSettings_(client->GetBalancingSettings()) +{} + +TEndpointPool::~TEndpointPool() { + try { + NThreading::TFuture future; + { + std::lock_guard guard(Mutex_); + if (DiscoveryPromise_.Initialized()) { + future = DiscoveryPromise_.GetFuture(); + } + } + if (future.Initialized()) { + future.Wait(); + } + } catch (...) { + Y_ABORT("Unexpected exception from endpoint pool dtor"); + } +} + +std::pair, bool> TEndpointPool::UpdateAsync() { + NThreading::TFuture future; + { + std::lock_guard guard(Mutex_); + if (DiscoveryPromise_.Initialized()) { + return {DiscoveryPromise_.GetFuture(), false}; + } else { + DiscoveryPromise_ = NThreading::NewPromise(); + future = DiscoveryPromise_.GetFuture(); + } + } + auto handler = [this](const TAsyncListEndpointsResult& future) { + TListEndpointsResult result = future.GetValue(); + vector removed; + if (result.DiscoveryStatus.Status == EStatus::SUCCESS) { + vector records; + // Is used to convert float to integer load factor + // same integer values will be selected randomly. + const float multiplicator = 10.0; + const auto& preferredLocation = GetPreferredLocation(result.Result.self_location()); + for (const auto& endpoint : result.Result.endpoints()) { + i32 loadFactor = (i32)(multiplicator * Min(LoadMax, Max(LoadMin, endpoint.load_factor()))); + ui64 nodeId = endpoint.node_id(); + if (BalancingSettings_.Policy != EBalancingPolicy::UseAllNodes) { + if (endpoint.location() != preferredLocation) { + // Location missmatch, shift this endpoint + loadFactor += GetLocalityShift(); + } + } + + std::string sslTargetNameOverride = endpoint.ssl_target_name_override(); + auto getIpSslTargetNameOverride = [&]() -> std::string { + if (!sslTargetNameOverride.empty()) { + return sslTargetNameOverride; + } + if (endpoint.ssl()) { + return endpoint.address(); + } + return std::string(); + }; + + bool addDefault = true; + for (const auto& addr : endpoint.ip_v6()) { + if (addr.empty()) { + continue; + } + TStringBuilder endpointBuilder; + endpointBuilder << "ipv6:"; + if (addr[0] != '[') { + endpointBuilder << "["; + } + endpointBuilder << addr; + if (addr[addr.size()-1] != ']') { + endpointBuilder << "]"; + } + endpointBuilder << ":" << endpoint.port(); + std::string endpointString = std::move(endpointBuilder); + records.emplace_back(std::move(endpointString), loadFactor, getIpSslTargetNameOverride(), nodeId); + addDefault = false; + } + for (const auto& addr : endpoint.ip_v4()) { + if (addr.empty()) { + continue; + } + std::string endpointString = + TStringBuilder() + << "ipv4:" + << addr + << ":" + << endpoint.port(); + records.emplace_back(std::move(endpointString), loadFactor, getIpSslTargetNameOverride(), nodeId); + addDefault = false; + } + if (addDefault) { + std::string endpointString = + TStringBuilder() + << endpoint.address() + << ":" + << endpoint.port(); + records.emplace_back(std::move(endpointString), loadFactor, std::move(sslTargetNameOverride), nodeId); + } + } + LastUpdateTime_ = TInstant::Now().MicroSeconds(); + removed = Elector_.SetNewState(std::move(records)); + } + NThreading::TPromise promise; + { + std::lock_guard guard(Mutex_); + DiscoveryPromise_.Swap(promise); + } + promise.SetValue({std::move(removed), result.DiscoveryStatus}); + }; + + Provider_().Subscribe(handler); + return {future, true}; +} + +TEndpointRecord TEndpointPool::GetEndpoint(const TEndpointKey& preferredEndpoint, bool onlyPreferred) const { + return Elector_.GetEndpoint(preferredEndpoint, onlyPreferred); +} + +TDuration TEndpointPool::TimeSinceLastUpdate() const { + auto now = TInstant::Now().MicroSeconds(); + return TDuration::MicroSeconds(now - LastUpdateTime_.load()); +} + +void TEndpointPool::BanEndpoint(const string& endpoint) { + Elector_.PessimizeEndpoint(endpoint); +} + +int TEndpointPool::GetPessimizationRatio() { + return Elector_.GetPessimizationRatio(); +} + +bool TEndpointPool::LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag) { + return Elector_.LinkObjToEndpoint(endpoint, obj, tag); +} + +void TEndpointPool::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, 0, Max(), tag); +} + +void TEndpointPool::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, 0, GetLocalityShift() - 1, tag); +} + +void TEndpointPool::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, GetLocalityShift(), Max() - 1, tag); +} + +EBalancingPolicy TEndpointPool::GetBalancingPolicy() const { + return BalancingSettings_.Policy; +} + +void TEndpointPool::SetStatCollector(NSdkStats::TStatCollector& statCollector) { + if (!statCollector.IsCollecting()) + return; + Elector_.SetStatCollector(statCollector.GetEndpointElectorStatCollector()); + StatCollector_ = &statCollector; +} + +constexpr i32 TEndpointPool::GetLocalityShift() { + return LoadMax * Multiplicator; +} + +string TEndpointPool::GetPreferredLocation(const string& selfLocation) { + switch (BalancingSettings_.Policy) { + case EBalancingPolicy::UseAllNodes: + return {}; + case EBalancingPolicy::UsePreferableLocation: + if (BalancingSettings_.PolicyParams.empty()) { + return selfLocation; + } else { + return BalancingSettings_.PolicyParams; + } + } + return {}; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h new file mode 100644 index 000000000000..7a83b5316781 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h @@ -0,0 +1,68 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +struct TListEndpointsResult { + Ydb::Discovery::ListEndpointsResult Result; + TPlainStatus DiscoveryStatus; +}; + +using TAsyncListEndpointsResult = NThreading::TFuture; +using TListEndpointsResultProvider = std::function; + +struct TEndpointUpdateResult { + std::vector Removed; + TPlainStatus DiscoveryStatus; +}; + +class TEndpointPool { +public: + TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client); + ~TEndpointPool(); + std::pair, bool> UpdateAsync(); + TEndpointRecord GetEndpoint(const TEndpointKey& preferredEndpoint, bool onlyPreferred = false) const; + TDuration TimeSinceLastUpdate() const; + void BanEndpoint(const std::string& endpoint); + int GetPessimizationRatio(); + bool LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag); + void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + EBalancingPolicy GetBalancingPolicy() const; + // TODO: Remove this mess + void SetStatCollector(NSdkStats::TStatCollector& statCollector); + static constexpr i32 GetLocalityShift(); + +private: + std::string GetPreferredLocation(const std::string& selfLocation); + +private: + TListEndpointsResultProvider Provider_; + std::mutex Mutex_; + TEndpointElectorSafe Elector_; + NThreading::TPromise DiscoveryPromise_; + std::atomic_uint64_t LastUpdateTime_; + const TBalancingSettings BalancingSettings_; + + NSdkStats::TStatCollector* StatCollector_ = nullptr; + + // Max, min load factor returned by discovery service + static constexpr float LoadMax = 100.0; + static constexpr float LoadMin = -100.0; + // Is used to convert float to integer load factor + // same integer values will be selected randomly. + static constexpr float Multiplicator = 10.0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.cpp new file mode 100644 index 000000000000..76f94af60c1b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.cpp @@ -0,0 +1,287 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "state.h" + +#include +#include + +#include + +#include +#include + +namespace { + void Quote(std::string& url, const char* safe = "/") { + TTempBuf tempBuf(CgiEscapeBufLen(url.size())); + char* to = tempBuf.Data(); + + url.assign(to, Quote(to, TStringBuf(url), safe)); + } +} + +namespace NYdb::inline V3 { + +constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck +constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case +constexpr TDuration DISCOVERY_RECHECK_PERIOD = TDuration::Seconds(5); // period to run periodic discovery task + +TDbDriverState::TDbDriverState( + const std::string& database, + const std::string& discoveryEndpoint, + EDiscoveryMode discoveryMode, + const TSslCredentials& sslCredentials, + IInternalClient* client +) + : Database(database) + , DiscoveryEndpoint(discoveryEndpoint) + , DiscoveryMode(discoveryMode) + , SslCredentials(sslCredentials) + , Client(client) + , EndpointPool([this, client]() mutable { + // this callback will be called just after shared_ptr initialization + // so this call is safe + auto self = shared_from_this(); + return client->GetEndpoints(self); + }, client) + , StatCollector(database, client->GetMetricRegistry()) + , Log(Client->GetLog()) + , DiscoveryCompletedPromise(NThreading::NewPromise()) +{ + EndpointPool.SetStatCollector(StatCollector); + Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database))); +} + +void TDbDriverState::SetCredentialsProvider(std::shared_ptr credentialsProvider) { + CredentialsProvider = std::move(credentialsProvider); +#ifndef YDB_GRPC_UNSECURE_AUTH + CallCredentials = grpc::MetadataCredentialsFromPlugin( + std::unique_ptr(new TYdbAuthenticator(CredentialsProvider))); +#endif +} + +void TDbDriverState::AddCb(TCb&& cb, ENotifyType type) { + std::lock_guard lock(NotifyCbsLock); + NotifyCbs[static_cast(type)].emplace_back(std::move(cb)); +} + +void TDbDriverState::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachEndpoint(cb, tag); +} + +void TDbDriverState::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachLocalEndpoint(cb, tag); +} + +void TDbDriverState::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachForeignEndpoint(cb, tag); +} + +EBalancingPolicy TDbDriverState::GetBalancingPolicy() const { + return EndpointPool.GetBalancingPolicy(); +} + +std::string TDbDriverState::GetEndpoint() const { + return EndpointPool.GetEndpoint(TEndpointKey()).Endpoint; +} + +NThreading::TFuture TDbDriverState::DiscoveryCompleted() const { + return DiscoveryCompletedPromise.GetFuture(); +} +void TDbDriverState::SignalDiscoveryCompleted() { + DiscoveryCompletedPromise.TrySetValue(); +} + +TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) { + auto weak = std::weak_ptr(driverState); + return [weak](NYdb::NIssue::TIssues&&, EStatus status) { + if (status != EStatus::SUCCESS) { + return false; + } + + TDbDriverState::TPtr strong = weak.lock(); + if (!strong) { + return false; + } else { + + bool pessThreshold = strong->EndpointPool.GetPessimizationRatio() > PESSIMIZATION_DISCOVERY_THRESHOLD; + bool expiration = strong->EndpointPool.TimeSinceLastUpdate() > ENDPOINT_UPDATE_PERIOD; + + if (pessThreshold) { + strong->StatCollector.IncDiscoveryDuePessimization(); + } + if (expiration) { + strong->StatCollector.IncDiscoveryDueExpiration(); + } + + if (pessThreshold || expiration) { + auto asyncResult = strong->EndpointPool.UpdateAsync(); + // true - we were first who run UpdateAsync + if (asyncResult.second) { + auto cb = [strong](const NThreading::TFuture& future) { + const auto& updateResult = future.GetValue(); +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + strong->Client->DeleteChannels(updateResult.Removed); +#endif + if (strong->DiscoveryMode == EDiscoveryMode::Sync) { + std::unique_lock guard(strong->LastDiscoveryStatusRWLock); + strong->LastDiscoveryStatus = updateResult.DiscoveryStatus; + } + }; + asyncResult.first.Subscribe(std::move(cb)); + } + } + } + return true; + }; +} + +TDbDriverStateTracker::TDbDriverStateTracker(IInternalClient* client) + : DiscoveryClient_(client) +{} + +TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( + std::string database, + std::string discoveryEndpoint, + EDiscoveryMode discoveryMode, + const TSslCredentials& sslCredentials, + std::shared_ptr credentialsProviderFactory +) { + std::string clientIdentity; + if (credentialsProviderFactory) { + clientIdentity = credentialsProviderFactory->GetClientIdentity(); + } + Quote(database); + const TStateKey key{database, discoveryEndpoint, clientIdentity, discoveryMode, sslCredentials}; + { + std::shared_lock lock(Lock_); + auto state = States_.find(key); + if (state != States_.end()) { + auto strong = state->second.lock(); + if (strong) { + return strong; + } + // If we can't promote to shared see bellow + } + } + TDbDriverStatePtr strongState; + for (;;) { + std::unique_lock lock(Lock_); + { + auto state = States_.find(key); + if (state != States_.end()) { + auto strong = state->second.lock(); + if (strong) { + return strong; + } else { + // We could find state record, but couldn't promote weak to shared + // this means weak ptr already expired but dtor hasn't been + // called yet. Likely other thread now is waiting on mutex to + // remove expired record from hashmap. So give him chance + // to do it after that we will be able to create new state + lock.unlock(); + std::this_thread::yield(); + continue; + } + } + } + { + auto deleter = [this, key](TDbDriverState* p) { + { + std::unique_lock lock(Lock_); + States_.erase(key); + } + delete p; + }; + strongState = std::shared_ptr( + new TDbDriverState( + database, + discoveryEndpoint, + discoveryMode, + sslCredentials, + DiscoveryClient_), + deleter); + + strongState->SetCredentialsProvider( + credentialsProviderFactory + ? credentialsProviderFactory->CreateProvider(strongState) + : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); + + DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); + Y_ABORT_UNLESS(States_.emplace(key, strongState).second); + break; + } + } + auto updateResult = strongState->EndpointPool.UpdateAsync(); + if (updateResult.second) { + auto cb = [strongState](const NThreading::TFuture&) { + strongState->SignalDiscoveryCompleted(); + }; + updateResult.first.Subscribe(cb); + } + + if (strongState->DiscoveryMode == EDiscoveryMode::Sync) { + const auto& discoveryStatus = updateResult.first.GetValueSync().DiscoveryStatus; + // Almost always true, except the situation when the current thread was + // preempted just before UpdateAsync call and other one get + // state from cache and call UpdateAsync before us. + if (Y_LIKELY(updateResult.second)) { + std::unique_lock guard(strongState->LastDiscoveryStatusRWLock); + strongState->LastDiscoveryStatus = discoveryStatus; + } + } + + return strongState; +} + +void TDbDriverState::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { + Client->AddPeriodicTask(std::move(cb), period); +} + +NThreading::TFuture TDbDriverStateTracker::SendNotification( + TDbDriverState::ENotifyType type +) { + std::vector> states; + { + std::shared_lock lock(Lock_); + states.reserve(States_.size()); + for (auto& weak : States_) { + states.push_back(weak.second); + } + } + std::vector> results; + for (auto& state : states) { + auto strong = state.lock(); + if (strong) { + std::lock_guard lock(strong->NotifyCbsLock); + for (auto& cb : strong->NotifyCbs[static_cast(type)]) { + if (cb) { + auto future = cb(); + if (!future.HasException()) { + results.push_back(future); + } + //TODO: Add loger + } + } + } + } + return NThreading::WaitExceptionOrAll(results); +} + +void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) { + std::vector> states; + { + std::shared_lock lock(Lock_); + states.reserve(States_.size()); + for (auto& weak : States_) { + states.push_back(weak.second); + } + } + + for (auto& weak : states) { + if (auto strong = weak.lock()) { + strong->StatCollector.SetMetricRegistry(sensorsRegistry); + strong->EndpointPool.SetStatCollector(strong->StatCollector); + } + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.h new file mode 100644 index 000000000000..ad2ff25b8f13 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/state.h @@ -0,0 +1,109 @@ +#pragma once + +#include "endpoint_pool.h" + +#include + +#include +#include +#include + +namespace NYdb::inline V3 { + +class ICredentialsProvider; +class ICredentialsProviderFactory; + +// Represents state of driver for one particular database +class TDbDriverState + : public std::enable_shared_from_this + , public ICoreFacility +{ +public: + enum class ENotifyType : size_t { + STOP = 0, + COUNT = 1 // types count + }; + + using TCb = std::function()>; + using TPtr = std::shared_ptr; + + TDbDriverState( + const std::string& database, + const std::string& discoveryEndpoint, + EDiscoveryMode discoveryMode, + const TSslCredentials& sslCredentials, + IInternalClient* client + ); + + NThreading::TFuture DiscoveryCompleted() const; + + void SignalDiscoveryCompleted(); + + void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; + + void AddCb(TCb&& cb, ENotifyType type); + void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + EBalancingPolicy GetBalancingPolicy() const; + std::string GetEndpoint() const; + void SetCredentialsProvider(std::shared_ptr credentialsProvider); + + const std::string Database; + const std::string DiscoveryEndpoint; + const EDiscoveryMode DiscoveryMode; + const TSslCredentials SslCredentials; + std::shared_ptr CredentialsProvider; + IInternalClient* Client; + TEndpointPool EndpointPool; + // StopCb allow client to subscribe for notifications from lower layer + std::mutex NotifyCbsLock; + std::array, static_cast(ENotifyType::COUNT)> NotifyCbs; +#ifndef YDB_GRPC_UNSECURE_AUTH + std::shared_ptr CallCredentials; +#endif + // Status of last discovery call, used in sync mode, coresponding mutex + std::shared_mutex LastDiscoveryStatusRWLock; + TPlainStatus LastDiscoveryStatus; + NSdkStats::TStatCollector StatCollector; + TLog Log; + NThreading::TPromise DiscoveryCompletedPromise; +}; + +// Tracker allows to get driver state by database and credentials +class TDbDriverStateTracker { + using TStateKey = std::tuple; + struct TStateKeyHash { + size_t operator()(const TStateKey& k) const noexcept { + THash strHash; + const size_t h0 = strHash(std::get<0>(k)); + const size_t h1 = strHash(std::get<1>(k)); + const size_t h2 = strHash(std::get<2>(k)); + const auto& sslCredentials = std::get<4>(k); + const size_t h3 = (static_cast(std::get<3>(k)) << 1) + static_cast(sslCredentials.IsEnabled); + const size_t h5 = strHash(sslCredentials.CaCert); + const size_t h6 = strHash(sslCredentials.Cert); + return (h0 ^ h1 ^ h2 ^ h3 ^ h5 ^ h6); + } + }; +public: + TDbDriverStateTracker(IInternalClient* client); + TDbDriverState::TPtr GetDriverState( + std::string database, + std::string DiscoveryEndpoint, + EDiscoveryMode discoveryMode, + const TSslCredentials& sslCredentials, + std::shared_ptr credentialsProviderFactory + ); + NThreading::TFuture SendNotification( + TDbDriverState::ENotifyType type); + void SetMetricRegistry(::NMonitoring::TMetricRegistry *sensorsRegistry); +private: + IInternalClient* DiscoveryClient_; + std::unordered_map, TStateKeyHash> States_; + std::shared_mutex Lock_; +}; + +using TDbDriverStatePtr = TDbDriverState::TPtr; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/ya.make new file mode 100644 index 000000000000..ca5bf2b7a6e1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + authenticator.cpp + endpoint_pool.cpp + state.cpp +) + +PEERDIR( + library/cpp/string_utils/quote + library/cpp/threading/future + ydb/public/sdk/cpp/src/client/impl/ydb_endpoints + ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger + ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status + ydb/public/sdk/cpp/src/client/types/credentials +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/driver/constants.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/driver/constants.h new file mode 100644 index 000000000000..3b51aa92c0bb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/driver/constants.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +constexpr uint64_t TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle + // before TCP starts sending keepalive probes, seconds +constexpr uint64_t TCP_KEEPALIVE_COUNT = 5; // The maximum number of keepalive probes TCP should send before + // dropping the connection +constexpr uint64_t TCP_KEEPALIVE_INTERVAL = 10; // The time between individual keepalive probes, seconds + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/CMakeLists.txt new file mode 100644 index 000000000000..5b3d03f673b1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/CMakeLists.txt @@ -0,0 +1,20 @@ +_ydb_sdk_add_library(impl-ydb_internal-grpc_connections) + +target_link_libraries(impl-ydb_internal-grpc_connections PUBLIC + yutil + api-grpc + api-protos + impl-ydb_internal-db_driver_state + impl-ydb_internal-plain_status + impl-ydb_internal-thread_pool + client-impl-ydb_stats + client-resources + client-ydb_types-exceptions +) + +target_sources(impl-ydb_internal-grpc_connections PRIVATE + actions.cpp + grpc_connections.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-grpc_connections) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.cpp new file mode 100644 index 000000000000..df109a4e2401 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.cpp @@ -0,0 +1,113 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "actions.h" +#include "grpc_connections.h" + +#include + +namespace NYdb::inline V3 { + +constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation + +TSimpleCbResult::TSimpleCbResult( + TSimpleCb&& cb, + TGRpcConnectionsImpl* connections, + std::shared_ptr context) + : TGenericCbHolder(std::move(cb), connections, std::move(context)) +{ } + +void TSimpleCbResult::Process(void*) { + UserResponseCb_(); + delete this; +} + +TDeferredAction::TDeferredAction(const std::string& operationId, + TDeferredOperationCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDuration delay, + TDbDriverStatePtr dbState, + const std::string& endpoint) + : TAlarmActionBase(std::move(userCb), connection, std::move(context)) + , NextDelay_(Min(delay * 2, MAX_DEFERRED_CALL_DELAY)) + , DbDriverState_(dbState) + , OperationId_(operationId) + , Endpoint_(endpoint) +{ + Deadline_ = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(delay.MicroSeconds(), GPR_TIMESPAN)); +} + +void TDeferredAction::OnAlarm() { + Y_ABORT_UNLESS(Connection_); + + Ydb::Operations::GetOperationRequest getOperationRequest; + getOperationRequest.set_id(TStringType{OperationId_}); + + TRpcRequestSettings settings; + settings.PreferredEndpoint = TEndpointKey(Endpoint_, 0); + + Connection_->RunDeferred( + std::move(getOperationRequest), + std::move(UserResponseCb_), + &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, + DbDriverState_, + NextDelay_, + settings, + true, + std::move(Context_)); + } + +void TDeferredAction::OnError() { + Y_ABORT_UNLESS(Connection_); + NYdbGrpc::TGrpcStatus status = {"Deferred timer interrupted", -1, true}; + DbDriverState_->StatCollector.IncDiscoveryFailDueTransportError(); + + auto resp = new TGRpcErrorResponse( + std::move(status), + std::move(UserResponseCb_), + Connection_, + std::move(Context_), + Endpoint_); + Connection_->EnqueueResponse(resp); +} + +TPeriodicAction::TPeriodicAction( + TPeriodicCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDuration period) + : TAlarmActionBase(std::move(userCb), connection, std::move(context)) + , Period_(period) +{ + Deadline_ = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(Period_.MicroSeconds(), GPR_TIMESPAN)); +} + +void TPeriodicAction::OnAlarm() { + NYdb::NIssue::TIssues issues; + if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) { + return; + } + + auto ctx = Connection_->CreateContext(); + if (!ctx) + return; + Context_ = ctx; + + auto action = MakeIntrusive( + std::move(UserResponseCb_), + Connection_, + Context_, + Period_); + action->Start(); +} + +void TPeriodicAction::OnError() { + NYdb::NIssue::TIssues issues; + issues.AddIssue(NYdb::NIssue::TIssue("Deferred timer interrupted")); + UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.h new file mode 100644 index 000000000000..818b538bbe93 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/actions.h @@ -0,0 +1,232 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +using NYdbGrpc::IQueueClientContext; +using NYdbGrpc::IQueueClientEvent; + +class TGRpcConnectionsImpl; +struct TPlainStatus; + + +template +using TResponseCb = std::function; +using TDeferredOperationCb = std::function; + +template +class TGenericCbHolder { +protected: + TGenericCbHolder( + TCb&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr context) + : UserResponseCb_(std::move(userCb)) + , Connection_(connections) + , Context_(std::move(context)) + {} + + TCb UserResponseCb_; + TGRpcConnectionsImpl* Connection_; + std::shared_ptr Context_; +}; + +template +class TAlarmActionBase + : public TThrRefBase + , public TGenericCbHolder + , private IQueueClientEvent +{ +public: + using TPtr = TIntrusivePtr>; + using TGenericCbHolder::TGenericCbHolder; + + virtual void OnAlarm() = 0; + virtual void OnError() = 0; + + void Start() { + Y_ABORT_UNLESS(this->Context_, "Missing shared context"); + auto context = this->Context_->CreateContext(); + { + std::lock_guard lock(Mutex_); + LocalContext_ = context; + Alarm_.Set(this->Context_->CompletionQueue(), Deadline_, PrepareTag()); + } + context->SubscribeStop([self = TPtr(this)] { + self->Stop(); + }); + } + + void Stop() { + Alarm_.Cancel(); + } + +private: + IQueueClientEvent* PrepareTag() { + Ref(); + return this; + } + + bool Execute(bool ok) override { + { + std::lock_guard lock(Mutex_); + LocalContext_.reset(); + } + + if (ok) { + OnAlarm(); + } else { + OnError(); + } + + return false; + } + + void Destroy() override { + UnRef(); + } + +protected: + gpr_timespec Deadline_ = {}; + +private: + std::mutex Mutex_; + grpc::Alarm Alarm_; + std::shared_ptr LocalContext_; +}; + +template +class TGRpcErrorResponse + : public TGenericCbHolder> + , public IObjectInQueue +{ +public: + TGRpcErrorResponse( + NYdbGrpc::TGrpcStatus&& status, + TResponseCb&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr context, + const std::string& endpoint) + : TGenericCbHolder>(std::move(userCb), connections, std::move(context)) + , GRpcStatus_(std::move(status)) + , Endpoint_(endpoint) + { } + + void Process(void*) override { + TPlainStatus status(GRpcStatus_, Endpoint_, {}); + + if (!Endpoint_.empty()) { + std::string msg = "Grpc error response on endpoint "; + msg += Endpoint_; + status.Issues.AddIssue(NYdb::NIssue::TIssue(msg)); + } + + this->UserResponseCb_(nullptr, status); + delete this; + } + +private: + NYdbGrpc::TGrpcStatus GRpcStatus_; + std::string Endpoint_; +}; + +template +class TResult + : public TGenericCbHolder> + , public IObjectInQueue +{ +public: + TResult( + TResponse&& response, + NYdbGrpc::TGrpcStatus&& status, + TResponseCb&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr context, + const std::string& endpoint, + std::multimap&& metadata) + : TGenericCbHolder>(std::move(userCb), connections, std::move(context)) + , Response_(std::move(response)) + , GRpcStatus_(std::move(status)) + , Endpoint_(endpoint) + , Metadata_(std::move(metadata)) {} + + void Process(void*) override { + this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)}); + delete this; + } + +private: + TResponse Response_; + NYdbGrpc::TGrpcStatus GRpcStatus_; + const std::string Endpoint_; + std::multimap Metadata_; +}; + +class TSimpleCbResult + : public TGenericCbHolder + , public IObjectInQueue +{ +public: + TSimpleCbResult( + TSimpleCb&& cb, + TGRpcConnectionsImpl* connections, + std::shared_ptr context); + void Process(void*) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDeferredAction + : public TAlarmActionBase +{ +public: + using TPtr = TIntrusivePtr; + + TDeferredAction( + const std::string& operationId, + TDeferredOperationCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDuration timeout, + TDbDriverStatePtr dbState, + const std::string& endpoint); + + void OnAlarm() override; + void OnError() override; + +private: + TDuration NextDelay_; + TDbDriverStatePtr DbDriverState_; + const std::string OperationId_; + const std::string Endpoint_; +}; + +class TPeriodicAction + : public TAlarmActionBase +{ +public: + TPeriodicAction( + TPeriodicCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDuration period); + + void OnAlarm() override; + void OnError() override; +private: + TDuration Period_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp new file mode 100644 index 000000000000..a3eba4dc8a09 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -0,0 +1,427 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "grpc_connections.h" + +#include + +namespace NYdb::inline V3 { + +bool IsTokenCorrect(const std::string& in) { + for (char c : in) { + if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) + return false; + } + return true; +} + +std::string GetAuthInfo(TDbDriverStatePtr p) { + auto token = p->CredentialsProvider->GetAuthInfo(); + if (!IsTokenCorrect(token)) { + throw TContractViolation("token is incorrect, illegal characters found"); + } + return token; +} + +void SetDatabaseHeader(TCallMeta& meta, const std::string& database) { + // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters + meta.Aux.push_back({YDB_DATABASE_HEADER, database}); +} + +std::string CreateSDKBuildInfo() { + return std::string("ydb-cpp-sdk/") + GetSdkSemver(); +} + +template +class TScheduledObject : public TThrRefBase { + using TSelf = TScheduledObject; + using TPtr = TIntrusivePtr; + + Y_FORCE_INLINE TDerived* Derived() { + return static_cast(this); + } + +protected: + TScheduledObject() { } + + void Start(TDuration timeout, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + Derived()->OnComplete(false); + return; + } + + auto deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(timeout.MicroSeconds(), GPR_TIMESPAN)); + + { + std::lock_guard guard(Mutex); + Context = context; + Alarm.Set(context->CompletionQueue(), deadline, OnAlarmTag.Prepare()); + } + + context->SubscribeCancel([self = TPtr(this)] { + self->Alarm.Cancel(); + }); + } + +private: + void OnAlarm(bool ok) { + { + std::lock_guard guard(Mutex); + // Break circular dependencies + Context.reset(); + } + + Derived()->OnComplete(ok); + } + +private: + std::mutex Mutex; + IQueueClientContextPtr Context; + grpc::Alarm Alarm; + +private: + using TFixedEvent = NYdbGrpc::TQueueClientFixedEvent; + + TFixedEvent OnAlarmTag = { this, &TSelf::OnAlarm }; +}; + +class TScheduledCallback : public TScheduledObject { + using TBase = TScheduledObject; + +public: + using TCallback = std::function; + + TScheduledCallback(TCallback&& callback) + : Callback(std::move(callback)) + { } + + void Start(TDuration timeout, IQueueClientContextProvider* provider) { + TBase::Start(timeout, provider); + } + + void OnComplete(bool ok) { + Callback(ok); + Callback = { }; + } + +private: + TCallback Callback; +}; + +class TScheduledFuture : public TScheduledObject { + using TBase = TScheduledObject; + +public: + TScheduledFuture() + : Promise(NThreading::NewPromise()) + { } + + NThreading::TFuture Start(TDuration timeout, IQueueClientContextProvider* provider) { + auto future = Promise.GetFuture(); + + TBase::Start(timeout, provider); + + return future; + } + + void OnComplete(bool ok) { + Promise.SetValue(ok); + Promise = { }; + } + +private: + NThreading::TPromise Promise; +}; + +TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr params) + : MetricRegistryPtr_(nullptr) + , ResponseQueue_(CreateThreadPool(params->GetClientThreadsNum())) + , DefaultDiscoveryEndpoint_(params->GetEndpoint()) + , SslCredentials_(params->GetSslCredentials()) + , DefaultDatabase_(params->GetDatabase()) + , DefaultCredentialsProviderFactory_(params->GetCredentialsProviderFactory()) + , StateTracker_(this) + , DefaultDiscoveryMode_(params->GetDiscoveryMode()) + , MaxQueuedRequests_(params->GetMaxQueuedRequests()) + , DrainOnDtors_(params->GetDrinOnDtors()) + , BalancingSettings_(params->GetBalancingSettings()) + , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout()) + , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls()) + , MemoryQuota_(params->GetMemoryQuota()) + , MaxInboundMessageSize_(params->GetMaxInboundMessageSize()) + , MaxOutboundMessageSize_(params->GetMaxOutboundMessageSize()) + , MaxMessageSize_(params->GetMaxMessageSize()) + , QueuedRequests_(0) +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + , ChannelPool_(params->GetTcpKeepAliveSettings(), params->GetSocketIdleTimeout()) +#endif + , GRpcClientLow_(params->GetNetworkThreadsNum()) + , Log(params->GetLog()) +{ +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + if (params->GetSocketIdleTimeout() != TDuration::Max()) { + auto channelPoolUpdateWrapper = [this] + (NYdb::NIssue::TIssues&&, EStatus status) mutable + { + if (status != EStatus::SUCCESS) { + return false; + } + + ChannelPool_.DeleteExpiredStubsHolders(); + return true; + }; + AddPeriodicTask(channelPoolUpdateWrapper, params->GetSocketIdleTimeout() * 0.1); + } +#endif + //TAdaptiveThreadPool ignores params + ResponseQueue_->Start(params->GetClientThreadsNum(), params->GetMaxQueuedResponses()); + if (!DefaultDatabase_.empty()) { + DefaultState_ = StateTracker_.GetDriverState( + DefaultDatabase_, + DefaultDiscoveryEndpoint_, + DefaultDiscoveryMode_, + SslCredentials_, + DefaultCredentialsProviderFactory_ + ); + } +} + +TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { + GRpcClientLow_.Stop(true); + ResponseQueue_->Stop(); +} + +void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { + std::shared_ptr context; + if (!TryCreateContext(context)) { + NYdb::NIssue::TIssues issues; + cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); + } else { + auto action = MakeIntrusive( + std::move(cb), + this, + std::move(context), + period); + action->Start(); + } +} + +void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) { + auto cbLow = [this, fn = std::move(fn)](NYdb::NIssue::TIssues&&, EStatus status) mutable { + if (status != EStatus::SUCCESS) { + return false; + } + + std::shared_ptr context; + + if (!TryCreateContext(context)) { + // Shutting down, fn must handle it + fn(); + } else { + // Enqueue to user pool + auto resp = new TSimpleCbResult( + std::move(fn), + this, + std::move(context)); + EnqueueResponse(resp); + } + + return false; + }; + + if (timeout) { + AddPeriodicTask(std::move(cbLow), timeout); + } else { + cbLow(NYdb::NIssue::TIssues(), EStatus::SUCCESS); + } +} + +NThreading::TFuture TGRpcConnectionsImpl::ScheduleFuture( + TDuration timeout, + IQueueClientContextPtr context) +{ + IQueueClientContextProvider* provider = context.get(); + if (!provider) { + provider = &GRpcClientLow_; + } + + return MakeIntrusive() + ->Start(timeout, provider); +} + +void TGRpcConnectionsImpl::ScheduleCallback( + TDuration timeout, + std::function callback, + IQueueClientContextPtr context) +{ + IQueueClientContextProvider* provider = context.get(); + if (!provider) { + provider = &GRpcClientLow_; + } + + return MakeIntrusive(std::move(callback)) + ->Start(timeout, provider); +} + +TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState( + const std::optional& database, + const std::optional& discoveryEndpoint, + const std::optional& discoveryMode, + const std::optional& sslCredentials, + const std::optional>& credentialsProviderFactory +) { + return StateTracker_.GetDriverState( + database.value_or(DefaultDatabase_), + discoveryEndpoint.value_or(DefaultDiscoveryEndpoint_), + discoveryMode.value_or(DefaultDiscoveryMode_), + sslCredentials.value_or(SslCredentials_), + credentialsProviderFactory.value_or(DefaultCredentialsProviderFactory_)); +} + +IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() { + return GRpcClientLow_.CreateContext(); +} + +bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) { + if (!context) { + // Keep CQ running until the request is complete + context = CreateContext(); + if (!context) { + return false; + } + } + return true; +} + +void TGRpcConnectionsImpl::WaitIdle() { + GRpcClientLow_.WaitIdle(); +} + +void TGRpcConnectionsImpl::Stop(bool wait) { + StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait(); + GRpcClientLow_.Stop(wait); +} + +void TGRpcConnectionsImpl::SetGrpcKeepAlive(NYdbGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls) { + ui64 timeoutMs = timeout.MilliSeconds(); + config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3; + config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs; + config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0; + config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0; +} + +TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) { + Ydb::Discovery::ListEndpointsRequest request; + request.set_database(TStringType{dbState->Database}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::ListEndpointsResult result; + if (any) { + any->UnpackTo(&result); + } + TListEndpointsResult val{result, status}; + promise.SetValue(std::move(val)); + }; + + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = GET_ENDPOINTS_TIMEOUT; + + RunDeferred( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, + dbState->shared_from_this(), + INITIAL_DEFERRED_CALL_DELAY, + rpcSettings); + + std::weak_ptr weakState = dbState; + + return promise.GetFuture().Apply([this, weakState](NThreading::TFuture future){ + auto strong = weakState.lock(); + auto result = future.ExtractValue(); + if (strong && result.DiscoveryStatus.IsTransportError()) { + strong->StatCollector.IncDiscoveryFailDueTransportError(); + } + return NThreading::MakeFuture(MutateDiscovery(std::move(result), *strong)); + }); +} + +TListEndpointsResult TGRpcConnectionsImpl::MutateDiscovery(TListEndpointsResult result, const TDbDriverState& dbDriverState) { + std::lock_guard lock(ExtensionsLock_); + if (!DiscoveryMutatorCb) + return result; + + auto endpoint = result.DiscoveryStatus.Endpoint; + auto ydbStatus = NYdb::TStatus(std::move(result.DiscoveryStatus)); + + auto aux = IDiscoveryMutatorApi::TAuxInfo { + .Database = dbDriverState.Database, + .DiscoveryEndpoint = dbDriverState.DiscoveryEndpoint + }; + + ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), aux); + + auto issues = ydbStatus.GetIssues(); + + auto plainStatus = TPlainStatus(ydbStatus.GetStatus(), std::move(issues), endpoint, {}); + result.DiscoveryStatus = plainStatus; + return result; +} + +bool TGRpcConnectionsImpl::GetDrainOnDtors() const { + return DrainOnDtors_; +} + +TBalancingSettings TGRpcConnectionsImpl::GetBalancingSettings() const { + return BalancingSettings_; +} + +bool TGRpcConnectionsImpl::StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) { + { + std::lock_guard lock(ExtensionsLock_); + if (MetricRegistryPtr_) { + return false; + } + if (auto ptr = dynamic_cast(sensorsRegistry)) { + MetricRegistryPtr_ = ptr; + } else { + std::cerr << "Unknown IMetricRegistry impl" << std::endl; + return false; + } + } + + StateTracker_.SetMetricRegistry(MetricRegistryPtr_); + return true; +} + +NMonitoring::TMetricRegistry* TGRpcConnectionsImpl::GetMetricRegistry() { + std::lock_guard lock(ExtensionsLock_); + return MetricRegistryPtr_; +} + +void TGRpcConnectionsImpl::RegisterExtension(IExtension* extension) { + Extensions_.emplace_back(extension); +} + +void TGRpcConnectionsImpl::RegisterExtensionApi(IExtensionApi* api) { + ExtensionApis_.emplace_back(api); +} + +void TGRpcConnectionsImpl::SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb) { + std::lock_guard lock(ExtensionsLock_); + DiscoveryMutatorCb = std::move(cb); +} + +const TLog& TGRpcConnectionsImpl::GetLog() const { + return Log; +} + +void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) { + Y_ENSURE(ResponseQueue_->Add(action)); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h new file mode 100644 index 000000000000..c09b1932df83 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -0,0 +1,727 @@ +#pragma once + +#include +#include + +#include "actions.h" +#include "params.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3 { + +constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10); +constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call +constexpr TDuration GET_ENDPOINTS_TIMEOUT = TDuration::Seconds(10); // Time wait for ListEndpoints request, after this time we pass error to client + +using NYdbGrpc::TCallMeta; +using NYdbGrpc::IQueueClientContextPtr; +using NYdbGrpc::IQueueClientContextProvider; + +class ICredentialsProvider; + +// Deferred callbacks +using TDeferredResultCb = std::function; + +std::string GetAuthInfo(TDbDriverStatePtr p); +void SetDatabaseHeader(TCallMeta& meta, const std::string& database); +std::string CreateSDKBuildInfo(); + +class TGRpcConnectionsImpl + : public IQueueClientContextProvider + , public IInternalClient +{ + friend class TDeferredAction; +public: + TGRpcConnectionsImpl(std::shared_ptr params); + ~TGRpcConnectionsImpl(); + + void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; + void ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout); + NThreading::TFuture ScheduleFuture( + TDuration timeout, + IQueueClientContextPtr token = nullptr + ); + void ScheduleCallback( + TDuration timeout, + std::function callback, + IQueueClientContextPtr token = nullptr + ); + // The idea is: sometimes we need to work with multiple databases simultaneously + // This method returns DbDriverState (or just db state) for given database credentials pair + // this state is used to keep data related to particular database. + TDbDriverStatePtr GetDriverState( + const std::optional& database, + const std::optional& discoveryEndpoint, + const std::optional& discoveryMode, + const std::optional& sslCredentials, + const std::optional>& credentialsProviderFactory + ); + IQueueClientContextPtr CreateContext() override; + bool TryCreateContext(IQueueClientContextPtr& context); + void WaitIdle(); + void Stop(bool wait = false); + + template + using TServiceConnection = NYdbGrpc::TServiceConnection; + + static void SetGrpcKeepAlive(NYdbGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls); + + template + std::pair>, TEndpointKey> GetServiceConnection( + TDbDriverStatePtr dbState, const TEndpointKey& preferredEndpoint, + TRpcRequestSettings::TEndpointPolicy endpointPolicy) + { + auto clientConfig = NYdbGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint); + const auto& sslCredentials = dbState->SslCredentials; + clientConfig.SslCredentials = {.pem_root_certs = TStringType{sslCredentials.CaCert}, .pem_private_key = TStringType{sslCredentials.PrivateKey}, .pem_cert_chain = TStringType{sslCredentials.Cert}}; + clientConfig.EnableSsl = sslCredentials.IsEnabled; + + clientConfig.MemQuota = MemoryQuota_; + + if (MaxMessageSize_ > 0) { + clientConfig.MaxMessageSize = MaxMessageSize_; + } + if (MaxInboundMessageSize_ > 0) { + clientConfig.MaxInboundMessageSize = MaxInboundMessageSize_; + } + if (MaxOutboundMessageSize_ > 0) { + clientConfig.MaxOutboundMessageSize = MaxOutboundMessageSize_; + } + + if (std::is_same() + || dbState->Database.empty() + || endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint) + { + SetGrpcKeepAlive(clientConfig, GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY, GRpcKeepAlivePermitWithoutCalls_); + } else { + auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint, endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UsePreferredEndpointStrictly); + if (!endpoint) { + return {nullptr, TEndpointKey()}; + } + clientConfig.Locator = endpoint.Endpoint; + clientConfig.SslTargetNameOverride = endpoint.SslTargetNameOverride; + if (GRpcKeepAliveTimeout_) { + SetGrpcKeepAlive(clientConfig, GRpcKeepAliveTimeout_, GRpcKeepAlivePermitWithoutCalls_); + } + } + + std::unique_ptr> conn; +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + ChannelPool_.GetStubsHolderLocked( + clientConfig.Locator, clientConfig, [&conn, this](NYdbGrpc::TStubsHolder& holder) mutable { + conn.reset(GRpcClientLow_.CreateGRpcServiceConnection(holder).release()); + }); +#else + conn = std::move(GRpcClientLow_.CreateGRpcServiceConnection(clientConfig)); +#endif + return {std::move(conn), TEndpointKey(clientConfig.Locator, 0)}; + } + + template + using TSimpleRpc = + typename NYdbGrpc::TSimpleRequestProcessor< + typename TService::Stub, + TRequest, + TResponse>::TAsyncRequest; + + template + void Run( + TRequest&& request, + TResponseCb&& userResponseCb, + TSimpleRpc rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + std::shared_ptr context = nullptr) + { + using NYdbGrpc::TGrpcStatus; + using TConnection = std::unique_ptr>; + Y_ABORT_UNLESS(dbState); + + if (!TryCreateContext(context)) { + TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); + userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)}); + return; + } + + if (dbState->StatCollector.IsCollecting()) { + std::weak_ptr weakState = dbState; + const auto startTime = TInstant::Now(); + userResponseCb = std::move([cb = std::move(userResponseCb), weakState, startTime](TResponse* response, TPlainStatus status) { + const auto resultSize = response ? response->ByteSizeLong() : 0; + cb(response, status); + + if (auto state = weakState.lock()) { + state->StatCollector.IncRequestLatency(TInstant::Now() - startTime); + state->StatCollector.IncResultSize(resultSize); + } + }); + } + + WithServiceConnection( + [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), dbState] + (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable -> void { + if (!status.Ok()) { + userResponseCb( + nullptr, + std::move(status)); + return; + } + + TCallMeta meta; + meta.Timeout = requestSettings.ClientTimeout; + #ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; + #else + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const std::exception& e) { + userResponseCb( + nullptr, + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ) + ); + return; + } + } + #endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + + if (!dbState->Database.empty()) { + SetDatabaseHeader(meta, dbState->Database); + } + + static const std::string clientPid = GetClientPIDHeaderValue(); + + meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()}); + meta.Aux.push_back({YDB_CLIENT_PID, clientPid}); + meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end()); + + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + + NYdbGrpc::TAdvancedResponseCallback responseCbLow = + [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState] + (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + + if (NYdbGrpc::IsGRpcStatusGood(grpcStatus)) { + std::multimap metadata; + + for (const auto& [name, value] : ctx.GetServerInitialMetadata()) { + metadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } + for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) { + metadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } + + auto resp = new TResult( + std::move(response), + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint.GetEndpoint(), + std::move(metadata)); + + EnqueueResponse(resp); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + + auto resp = new TGRpcErrorResponse( + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint.GetEndpoint()); + + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + + EnqueueResponse(resp); + } + }; + + serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta, + context.get()); + }, dbState, requestSettings.PreferredEndpoint, requestSettings.EndpointPolicy); + } + + template + void RunDeferred( + TRequest&& request, + TDeferredOperationCb&& userResponseCb, + TSimpleRpc rpc, + TDbDriverStatePtr dbState, + TDuration deferredTimeout, + const TRpcRequestSettings& requestSettings, + bool poll = false, + std::shared_ptr context = nullptr) + { + if (!TryCreateContext(context)) { + TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); + userResponseCb(nullptr, status); + return; + } + + auto responseCb = [this, userResponseCb = std::move(userResponseCb), dbState, deferredTimeout, poll, context] + (TResponse* response, TPlainStatus status) mutable + { + if (response) { + Ydb::Operations::Operation* operation = response->mutable_operation(); + if (!operation->ready() && poll) { + auto action = MakeIntrusive( + operation->id(), + std::move(userResponseCb), + this, + std::move(context), + deferredTimeout, + dbState, + status.Endpoint); + + action->Start(); + } else { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(operation->issues(), opIssues); + userResponseCb(operation, TPlainStatus{static_cast(operation->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + } + } else { + userResponseCb(nullptr, status); + } + }; + + Run( + std::move(request), + responseCb, + rpc, + dbState, + requestSettings, + std::move(context)); + } + + // Run request using discovery endpoint. + // Mostly usefull to make calls from credential provider + template + static void RunOnDiscoveryEndpoint( + std::shared_ptr facility, + TRequest&& request, + TResponseCb&& responseCb, + TSimpleRpc rpc, + TRpcRequestSettings requestSettings) + { + requestSettings.EndpointPolicy = TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint; + requestSettings.UseAuth = false; + // TODO: Change implementation of Run, to use ICoreFacility and remove this cast + auto dbState = std::dynamic_pointer_cast(facility); + Y_ABORT_UNLESS(dbState); + auto self = dynamic_cast(dbState->Client); + Y_ABORT_UNLESS(self); + self->Run( + std::move(request), + std::move(responseCb), + rpc, + dbState, + requestSettings, + nullptr); + } + + template + void RunDeferred( + TRequest&& request, + TDeferredResultCb&& userResponseCb, + TSimpleRpc rpc, + TDbDriverStatePtr dbState, + TDuration deferredTimeout, + const TRpcRequestSettings& requestSettings, + std::shared_ptr context = nullptr) + { + auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable { + if (operation) { + status.SetCostInfo(std::move(*operation->mutable_cost_info())); + userResponseCb(operation->mutable_result(), std::move(status)); + } else { + userResponseCb(nullptr, std::move(status)); + } + }; + + RunDeferred( + std::move(request), + operationCb, + rpc, + dbState, + deferredTimeout, + requestSettings, + true, // poll + context); + } + + template class TStream> + using TStreamRpc = + typename TStream< + typename TService::Stub, + TRequest, + TResponse>::TAsyncRequest; + + template + void StartReadStream( + const TRequest& request, + TCallback responseCb, + TStreamRpc rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + std::shared_ptr context = nullptr) + { + using NYdbGrpc::TGrpcStatus; + using TConnection = std::unique_ptr>; + using TProcessor = typename NYdbGrpc::IStreamRequestReadProcessor::TPtr; + + if (!TryCreateContext(context)) { + responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + return; + } + + WithServiceConnection( + [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState](TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable { + if (!status.Ok()) { + responseCb(std::move(status), nullptr); + return; + } + + TCallMeta meta; + meta.Timeout = requestSettings.ClientTimeout; +#ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; +#else + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const std::exception& e) { + responseCb( + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ), + nullptr + ); + return; + } + } +#endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + + if (!dbState->Database.empty()) { + SetDatabaseHeader(meta, dbState->Database); + } + + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + + auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint] + (TGrpcStatus grpcStatus, TProcessor processor) mutable { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + + if (grpcStatus.Ok()) { + Y_ABORT_UNLESS(processor); + auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + } + }; + processor->AddFinishedCallback(std::move(finishedCallback)); + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + responseCb(std::move(status), std::move(processor)); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + } + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + responseCb(std::move(status), nullptr); + } + }; + + serviceConnection->template DoStreamRequest( + request, + std::move(lowCallback), + std::move(rpc), + std::move(meta), + context.get()); + }, dbState, requestSettings.PreferredEndpoint, requestSettings.EndpointPolicy); + } + + template + void StartBidirectionalStream( + TCallback connectedCallback, + TStreamRpc rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + std::shared_ptr context = nullptr) + { + using NYdbGrpc::TGrpcStatus; + using TConnection = std::unique_ptr>; + using TProcessor = typename NYdbGrpc::IStreamRequestReadWriteProcessor::TPtr; + + if (!TryCreateContext(context)) { + connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + return; + } + + WithServiceConnection( + [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState] + (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable { + if (!status.Ok()) { + connectedCallback(std::move(status), nullptr); + return; + } + + TCallMeta meta; + #ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; + #else + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const std::exception& e) { + connectedCallback( + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ), + nullptr + ); + return; + } + } + #endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + + if (!dbState->Database.empty()) { + SetDatabaseHeader(meta, dbState->Database); + } + + static const std::string clientPid = GetClientPIDHeaderValue(); + meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()}); + meta.Aux.push_back({YDB_CLIENT_PID, clientPid}); + meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end()); + + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + + auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint] + (TGrpcStatus grpcStatus, TProcessor processor) { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + + if (grpcStatus.Ok()) { + Y_ABORT_UNLESS(processor); + auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + } + }; + processor->AddFinishedCallback(std::move(finishedCallback)); + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + connectedCallback(std::move(status), std::move(processor)); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + } + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + connectedCallback(std::move(status), nullptr); + } + }; + + serviceConnection->template DoStreamRequest( + std::move(lowCallback), + std::move(rpc), + std::move(meta), + context.get()); + }, dbState, requestSettings.PreferredEndpoint, requestSettings.EndpointPolicy); + } + + TAsyncListEndpointsResult GetEndpoints(TDbDriverStatePtr dbState) override; + TListEndpointsResult MutateDiscovery(TListEndpointsResult result, const TDbDriverState& dbDriverState); + +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + void DeleteChannels(const std::vector& endpoints) override { + for (const auto& endpoint : endpoints) { + ChannelPool_.DeleteChannel(endpoint); + } + } +#endif + + bool GetDrainOnDtors() const; + TBalancingSettings GetBalancingSettings() const override; + bool StartStatCollecting(::NMonitoring::IMetricRegistry* sensorsRegistry) override; + ::NMonitoring::TMetricRegistry* GetMetricRegistry() override; + void RegisterExtension(IExtension* extension); + void RegisterExtensionApi(IExtensionApi* api); + void SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb); + const TLog& GetLog() const override; + +private: + template + void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState, + const TEndpointKey& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) + { + using TConnection = std::unique_ptr>; + TConnection serviceConnection; + TEndpointKey endpoint; + std::tie(serviceConnection, endpoint) = GetServiceConnection(dbState, preferredEndpoint, endpointPolicy); + if (!serviceConnection) { + if (dbState->DiscoveryMode == EDiscoveryMode::Sync) { + TStringStream errString; + errString << "Endpoint list is empty for database " << dbState->Database; + errString << ", cluster endpoint " << dbState->DiscoveryEndpoint; + TPlainStatus discoveryStatus; + { + std::shared_lock guard(dbState->LastDiscoveryStatusRWLock); + discoveryStatus = dbState->LastDiscoveryStatus; + } + + // Discovery returns success but no serviceConnection - in this case we unable to continue processing + if (discoveryStatus.Ok()) { + errString << " while last discovery returned success status. Unable to continue processing."; + discoveryStatus = TPlainStatus(EStatus::UNAVAILABLE, errString.Str()); + } else { + errString <<"."; + discoveryStatus.Issues.AddIssues({NYdb::NIssue::TIssue(errString.Str())}); + } + dbState->StatCollector.IncReqFailNoEndpoint(); + callback( + discoveryStatus, + TConnection{nullptr}, + TEndpointKey{ }); + } else { + int64_t newVal; + int64_t val; + do { + val = QueuedRequests_.load(); + if (val >= MaxQueuedRequests_) { + dbState->StatCollector.IncReqFailQueueOverflow(); + callback( + TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"), + TConnection{nullptr}, + TEndpointKey{ }); + return; + } + newVal = val + 1; + } while (!QueuedRequests_.compare_exchange_weak(val, newVal)); + + // UpdateAsync guarantee one update in progress for state + auto asyncResult = dbState->EndpointPool.UpdateAsync(); + const bool needUpdateChannels = asyncResult.second; + asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy] + (const NThreading::TFuture& future) mutable { + --QueuedRequests_; + const auto& updateResult = future.GetValue(); + if (needUpdateChannels) { +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + DeleteChannels(updateResult.Removed); +#endif + } + auto discoveryStatus = updateResult.DiscoveryStatus; + if (discoveryStatus.Status == EStatus::SUCCESS) { + WithServiceConnection(std::move(callback), dbState, preferredEndpoint, endpointPolicy); + } else { + callback( + TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), + TConnection{nullptr}, + TEndpointKey{ }); + } + }); + } + return; + } + + callback( + TPlainStatus{ }, + std::move(serviceConnection), + std::move(endpoint)); + } + + void EnqueueResponse(IObjectInQueue* action); + +private: + std::mutex ExtensionsLock_; + ::NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr; + + std::unique_ptr ResponseQueue_; + + const std::string DefaultDiscoveryEndpoint_; + const TSslCredentials SslCredentials_; + const std::string DefaultDatabase_; + std::shared_ptr DefaultCredentialsProviderFactory_; + TDbDriverStateTracker StateTracker_; + const EDiscoveryMode DefaultDiscoveryMode_; + const i64 MaxQueuedRequests_; + const bool DrainOnDtors_; + const TBalancingSettings BalancingSettings_; + const TDuration GRpcKeepAliveTimeout_; + const bool GRpcKeepAlivePermitWithoutCalls_; + const ui64 MemoryQuota_; + const ui64 MaxInboundMessageSize_; + const ui64 MaxOutboundMessageSize_; + const ui64 MaxMessageSize_; + + std::atomic_int64_t QueuedRequests_; +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + NYdbGrpc::TChannelPool ChannelPool_; +#endif + // State for default database, token pair + TDbDriverStatePtr DefaultState_; + + std::vector> Extensions_; + std::vector> ExtensionApis_; + + IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; + + // Must be the last member (first called destructor) + NYdbGrpc::TGRpcClientLow GRpcClientLow_; + TLog Log; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/params.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/params.h new file mode 100644 index 000000000000..b0670a8e484f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/params.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace NYdb::inline V3 { + +class IConnectionsParams { +public: + virtual ~IConnectionsParams() = default; + virtual std::string GetEndpoint() const = 0; + virtual size_t GetNetworkThreadsNum() const = 0; + virtual size_t GetClientThreadsNum() const = 0; + virtual size_t GetMaxQueuedResponses() const = 0; + virtual TSslCredentials GetSslCredentials() const = 0; + virtual std::string GetDatabase() const = 0; + virtual std::shared_ptr GetCredentialsProviderFactory() const = 0; + virtual EDiscoveryMode GetDiscoveryMode() const = 0; + virtual size_t GetMaxQueuedRequests() const = 0; + virtual NYdbGrpc::TTcpKeepAliveSettings GetTcpKeepAliveSettings() const = 0; + virtual bool GetDrinOnDtors() const = 0; + virtual TBalancingSettings GetBalancingSettings() const = 0; + virtual TDuration GetGRpcKeepAliveTimeout() const = 0; + virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0; + virtual TDuration GetSocketIdleTimeout() const = 0; + virtual const TLog& GetLog() const = 0; + virtual uint64_t GetMemoryQuota() const = 0; + virtual uint64_t GetMaxInboundMessageSize() const = 0; + virtual uint64_t GetMaxOutboundMessageSize() const = 0; + virtual uint64_t GetMaxMessageSize() const = 0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/ya.make new file mode 100644 index 000000000000..5ce41dc9a73b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + actions.cpp + grpc_connections.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/impl/ydb_internal/db_driver_state + ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status + ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool + ydb/public/sdk/cpp/src/client/impl/ydb_stats + ydb/public/sdk/cpp/src/client/resources + ydb/public/sdk/cpp/src/client/types/exceptions +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_client/client.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_client/client.h new file mode 100644 index 000000000000..b40003b50779 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_client/client.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +namespace NMonitoring { + class IMetricRegistry; + class TMetricRegistry; +} + +namespace NYdb::inline V3 { + +class TDbDriverState; +struct TListEndpointsResult; + +class IInternalClient { +public: + virtual NThreading::TFuture GetEndpoints(std::shared_ptr dbState) = 0; + virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + virtual void DeleteChannels(const std::vector& endpoints) = 0; +#endif + virtual TBalancingSettings GetBalancingSettings() const = 0; + virtual bool StartStatCollecting(::NMonitoring::IMetricRegistry* sensorsRegistry) = 0; + virtual ::NMonitoring::TMetricRegistry* GetMetricRegistry() = 0; + virtual const TLog& GetLog() const = 0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_header.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_header.h new file mode 100644 index 000000000000..acd8f23824ea --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/internal_header.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#if !defined(INCLUDE_YDB_INTERNAL_H) +#error "you are trying to use internal ydb header" +#endif // INCLUDE_YDB_INTERNAL_H + +// This macro allow to send credentials data via insecure channel +#define YDB_GRPC_UNSECURE_AUTH + +// This macro is used for grpc debug purpose only +//#define YDB_GRPC_BYPASS_CHANNEL_POOL + +// gRpc issue temporal workaround. +// In case of early TryCancel call and sharing shannel interfaces +// grpc doesn`t free allocated memory at the client shutdown +#if defined(_asan_enabled_) +#define YDB_GRPC_BYPASS_CHANNEL_POOL +#endif diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt new file mode 100644 index 000000000000..405cbd9653e5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(impl-ydb_internal-kqp_session_common) + +target_link_libraries(impl-ydb_internal-kqp_session_common PUBLIC + yutil + threading-future + library-operation_id + client-impl-ydb_endpoints +) + +target_sources(impl-ydb_internal-kqp_session_common PRIVATE + kqp_session_common.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-kqp_session_common) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp new file mode 100644 index 000000000000..c3f673e53efa --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.cpp @@ -0,0 +1,191 @@ +#include "kqp_session_common.h" + +#include + +#include + +namespace NYdb::inline V3 { + +using std::string; + +ui64 GetNodeIdFromSession(const std::string& sessionId) { + if (sessionId.empty()) { + return 0; + } + + try { + NKikimr::NOperationId::TOperationId opId(sessionId); + const auto& nodeIds = opId.GetValue("node_id"); + if (nodeIds.size() != 1) { + return 0; + } + + return FromStringWithDefault(*nodeIds[0], 0); + + } catch (...) { + return 0; + } + return 0; +} + +TKqpSessionCommon::TKqpSessionCommon( + const std::string& sessionId, const std::string& endpoint, + bool isOwnedBySessionPool) + : Lock_() + , SessionId_(sessionId) + , EndpointKey_(endpoint, GetNodeIdFromSession(sessionId)) + , IsOwnedBySessionPool_(isOwnedBySessionPool) + , State_(S_STANDALONE) + , TimeToTouch_(TInstant::Now()) + , TimeInPast_(TInstant::Now()) + , CloseHandler_(nullptr) + , NeedUpdateActiveCounter_(false) +{} + +TKqpSessionCommon::~TKqpSessionCommon() { + Unlink(); +} + +const std::string& TKqpSessionCommon::GetId() const { + return SessionId_; +} + +const string& TKqpSessionCommon::GetEndpoint() const { + return EndpointKey_.GetEndpoint(); +} + +const TEndpointKey& TKqpSessionCommon::GetEndpointKey() const { + return EndpointKey_; +} + +// Can be called from interceptor, need lock +void TKqpSessionCommon::MarkBroken() { + std::lock_guard guard(Lock_); + if (State_ == EState::S_ACTIVE) { + NeedUpdateActiveCounter_ = true; + } + State_ = EState::S_BROKEN; +} + +void TKqpSessionCommon::MarkAsClosing() { + std::lock_guard guard(Lock_); + if (State_ == EState::S_ACTIVE) { + NeedUpdateActiveCounter_ = true; + } + + State_ = EState::S_CLOSING; +} + +void TKqpSessionCommon::MarkActive() { + State_ = EState::S_ACTIVE; + NeedUpdateActiveCounter_ = false; +} + +void TKqpSessionCommon::MarkIdle() { + State_ = EState::S_IDLE; + NeedUpdateActiveCounter_ = false; +} + +bool TKqpSessionCommon::IsOwnedBySessionPool() const { + return IsOwnedBySessionPool_; +} + +TKqpSessionCommon::EState TKqpSessionCommon::GetState() const { + // See comments in InjectSessionStatusInterception about lock + std::lock_guard guard(const_cast(Lock_)); + return State_; +} + +void TKqpSessionCommon::SetNeedUpdateActiveCounter(bool flag) { + NeedUpdateActiveCounter_ = flag; +} + +bool TKqpSessionCommon::NeedUpdateActiveCounter() const { + return NeedUpdateActiveCounter_; +} + +// We need lock here because this method can be called from different thread +// if client makes simultaneous calls on one session. +// It should be possible to rewrite this part. +void TKqpSessionCommon::ScheduleTimeToTouch(TDuration interval, + bool updateTimeInPast) +{ + auto now = TInstant::Now(); + std::lock_guard guard(Lock_); + if (updateTimeInPast) { + TimeInPast_ = now; + } + TimeToTouch_.store(now + interval, std::memory_order_relaxed); +} + +void TKqpSessionCommon::ScheduleTimeToTouchFast(TDuration interval, + bool updateTimeInPast) +{ + auto now = TInstant::Now(); + if (updateTimeInPast) { + TimeInPast_ = now; + } + TimeToTouch_.store(now + interval, std::memory_order_relaxed); +} + +TInstant TKqpSessionCommon::GetTimeToTouchFast() const { + return TimeToTouch_.load(std::memory_order_relaxed); +} + +TInstant TKqpSessionCommon::GetTimeInPastFast() const { + return TimeInPast_; +} + +// SetTimeInterval/GetTimeInterval, are not atomic! +void TKqpSessionCommon::SetTimeInterval(TDuration interval) { + TimeInterval_ = interval; +} + +TDuration TKqpSessionCommon::GetTimeInterval() const { + return TimeInterval_; +} + +void TKqpSessionCommon::UpdateServerCloseHandler(IServerCloseHandler* handler) { + CloseHandler_.store(handler); +} + +void TKqpSessionCommon::CloseFromServer(std::weak_ptr client) noexcept { + auto strong = client.lock(); + if (!strong) { + // Session closed on the server after stopping client - do nothing + // moreover pool maybe destoyed now + return; + } + + IServerCloseHandler* h = CloseHandler_.load(); + if (h) { + h->OnCloseSession(this, strong); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +std::function TKqpSessionCommon::GetSmartDeleter( + std::shared_ptr client) +{ + return [client](TKqpSessionCommon* sessionImpl) { + switch (sessionImpl->GetState()) { + case TKqpSessionCommon::S_STANDALONE: + case TKqpSessionCommon::S_BROKEN: + case TKqpSessionCommon::S_CLOSING: + client->DeleteSession(sessionImpl); + break; + case TKqpSessionCommon::S_IDLE: + case TKqpSessionCommon::S_ACTIVE: { + if (!client->ReturnSession(sessionImpl)) { + client->DeleteSession(sessionImpl); + } + break; + } + default: + break; + } + }; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h new file mode 100644 index 000000000000..dac45a56bce7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/kqp_session_common.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +namespace NYdb::inline V3 { + +//////////////////////////////////////////////////////////////////////////////// +ui64 GetNodeIdFromSession(const std::string& sessionId); + +class TKqpSessionCommon; + +class IServerCloseHandler { +public: + virtual ~IServerCloseHandler() = default; + // called when session should be closed by server signal + virtual void OnCloseSession(const TKqpSessionCommon*, std::shared_ptr) = 0; +}; + +class TKqpSessionCommon : public TEndpointObj { +public: + TKqpSessionCommon(const std::string& sessionId, const std::string& endpoint, + bool isOwnedBySessionPool); + + enum EState { + S_STANDALONE, + S_IDLE, + S_BROKEN, + S_ACTIVE, + S_CLOSING + }; + +public: + ~TKqpSessionCommon(); + + const std::string& GetId() const; + const std::string& GetEndpoint() const; + const TEndpointKey& GetEndpointKey() const; + void MarkBroken(); + void MarkAsClosing(); + void MarkActive(); + void MarkIdle(); + bool IsOwnedBySessionPool() const; + EState GetState() const; + void SetNeedUpdateActiveCounter(bool flag); + bool NeedUpdateActiveCounter() const; + void InvalidateQueryInCache(const std::string& key); + void InvalidateQueryCache(); + void ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast); + void ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast); + TInstant GetTimeToTouchFast() const; + TInstant GetTimeInPastFast() const; + + // SetTimeInterval/GetTimeInterval, are not atomic! + void SetTimeInterval(TDuration interval); + TDuration GetTimeInterval() const; + + static std::function + GetSmartDeleter(std::shared_ptr client); + + // Shoult be called under session pool lock + void UpdateServerCloseHandler(IServerCloseHandler*); + + // Called asynchronously from grpc thread. + void CloseFromServer(std::weak_ptr client) noexcept; + +protected: + TAdaptiveLock Lock_; + +private: + const std::string SessionId_; + const TEndpointKey EndpointKey_; + const bool IsOwnedBySessionPool_; + + EState State_; + // This time is used during async close session handling which does not lock the session + // so we need to be able to read this value atomicaly + std::atomic TimeToTouch_; + TInstant TimeInPast_; + // Is used to implement progressive timeout for settler keep alive call + TDuration TimeInterval_; + + std::atomic CloseHandler_; + // Indicate session was in active state, but state was changed + // (need to decrement active session counter) + // TODO: suboptimal because need lock for atomic change from interceptor + // Rewrite with bit field + bool NeedUpdateActiveCounter_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/ya.make new file mode 100644 index 000000000000..629df1ea1384 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + kqp_session_common.cpp +) + +PEERDIR( + library/cpp/threading/future + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/impl/ydb_endpoints +) + + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/CMakeLists.txt new file mode 100644 index 000000000000..f078e7ebe41a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(impl-ydb_internal-logger) + +target_link_libraries(impl-ydb_internal-logger PUBLIC + yutil + logger +) + +target_sources(impl-ydb_internal-logger PRIVATE + log.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-logger) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.cpp new file mode 100644 index 000000000000..f8a8dcbcaf05 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.cpp @@ -0,0 +1,49 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "log.h" + +#include + +#include +#include + +namespace NYdb::inline V3 { + +static const std::string_view LogPrioritiesStrings[] = { + " :EMERG: ", + " :ALERT: ", + " :CRIT: ", + " :ERROR: ", + " :WARNING: ", + " :NOTICE: ", + " :INFO: ", + " :DEBUG: ", + " :TRACE: ", +}; + +std::string_view LogPriorityToString(ELogPriority priority) { + constexpr int maxPriority = static_cast(std::size(LogPrioritiesStrings) - 1); + const int index = static_cast(priority); + return LogPrioritiesStrings[ClampVal(index, 0, maxPriority)]; +} + +TLogFormatter GetPrefixLogFormatter(const std::string& prefix) { + return [prefix](ELogPriority priority, std::string_view message) -> std::string { + constexpr size_t timeLen = 27; // 2020-10-08T20:31:11.202588Z + constexpr size_t endlLen = 1; + + const std::string_view priorityString = LogPriorityToString(priority); + TStringBuilder result; + const size_t toReserve = prefix.size() + message.size() + timeLen + endlLen + priorityString.size(); + result.reserve(toReserve); + + result << TInstant::Now() << priorityString << prefix << message << Endl; + Y_ASSERT(result.size() == toReserve); + return std::move(result); + }; +} + +std::string GetDatabaseLogPrefix(const std::string& database) { + return "[" + database + "] "; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h new file mode 100644 index 000000000000..1a4c6e19f43d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +TLogFormatter GetPrefixLogFormatter(const std::string& prefix); +std::string GetDatabaseLogPrefix(const std::string& database); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/ya.make new file mode 100644 index 000000000000..25cadcab0ec4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + log.cpp +) + +PEERDIR( + library/cpp/logger +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/CMakeLists.txt new file mode 100644 index 000000000000..ce339e5cb7fb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(impl-ydb_internal-make_request) + +target_link_libraries(impl-ydb_internal-make_request PUBLIC + yutil + protobuf::libprotobuf + api-protos +) + +target_sources(impl-ydb_internal-make_request PRIVATE + make.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-make_request) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.cpp new file mode 100644 index 000000000000..e16b9ae8d433 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.cpp @@ -0,0 +1,12 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "make.h" + + +namespace NYdb::inline V3 { + +void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { + protoValue.set_seconds(duration.Seconds()); + protoValue.set_nanos(duration.NanoSecondsOfSecond()); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.h new file mode 100644 index 000000000000..ec38f3005b78 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/make.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue); + +template +void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params) { + auto& operationParams = *params.mutable_operation_params(); + + if (settings.CancelAfter_) { + SetDuration(settings.CancelAfter_, *operationParams.mutable_cancel_after()); + } + + if (settings.ForgetAfter_) { + SetDuration(settings.ForgetAfter_, *operationParams.mutable_forget_after()); + } + + if (settings.OperationTimeout_) { + SetDuration(settings.OperationTimeout_, *operationParams.mutable_operation_timeout()); + } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) { + SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout()); + } + + if (settings.ReportCostInfo_) { + operationParams.set_report_cost_info(Ydb::FeatureFlag::ENABLED); + } +} + +template +TProtoRequest MakeRequest() { + return TProtoRequest(); +} + +template +TProtoRequest MakeOperationRequest(const TRequestSettings& settings) { + auto request = MakeRequest(); + FillOperationParams(settings, request); + return request; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/ya.make new file mode 100644 index 000000000000..b0a40422de43 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + make.cpp +) + +PEERDIR( + contrib/libs/protobuf + ydb/public/api/protos +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/CMakeLists.txt new file mode 100644 index 000000000000..0e2dcc91ba32 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(impl-ydb_internal-plain_status) + +target_link_libraries(impl-ydb_internal-plain_status PUBLIC + yutil + protobuf::libprotobuf + grpc-client + yql-public-issue +) + +target_sources(impl-ydb_internal-plain_status PRIVATE + status.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-plain_status) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.cpp new file mode 100644 index 000000000000..bb29e6583e9e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.cpp @@ -0,0 +1,71 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "status.h" + +#include + +namespace NYdb::inline V3 { + +using std::string; + +TPlainStatus::TPlainStatus( + const NYdbGrpc::TGrpcStatus& grpcStatus, + const string& endpoint, + std::multimap&& metadata) + : Endpoint(endpoint) + , Metadata(std::move(metadata)) +{ + std::string msg; + if (grpcStatus.InternalError) { + Status = EStatus::CLIENT_INTERNAL_ERROR; + if (!grpcStatus.Msg.empty()) { + msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg; + } else { + msg = "Unknown internal client error"; + } + } else if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) { + switch (grpcStatus.GRpcStatusCode) { + case grpc::StatusCode::UNAVAILABLE: + Status = EStatus::TRANSPORT_UNAVAILABLE; + break; + case grpc::StatusCode::CANCELLED: + Status = EStatus::CLIENT_CANCELLED; + break; + case grpc::StatusCode::UNAUTHENTICATED: + Status = EStatus::CLIENT_UNAUTHENTICATED; + break; + case grpc::StatusCode::UNIMPLEMENTED: + Status = EStatus::CLIENT_CALL_UNIMPLEMENTED; + break; + case grpc::StatusCode::RESOURCE_EXHAUSTED: + Status = EStatus::CLIENT_RESOURCE_EXHAUSTED; + break; + case grpc::StatusCode::DEADLINE_EXCEEDED: + Status = EStatus::CLIENT_DEADLINE_EXCEEDED; + break; + case grpc::StatusCode::OUT_OF_RANGE: + Status = EStatus::CLIENT_OUT_OF_RANGE; + break; + default: + Status = EStatus::CLIENT_INTERNAL_ERROR; + break; + } + msg = TStringBuilder() << "GRpc error: (" << grpcStatus.GRpcStatusCode << "): " << grpcStatus.Msg; + } else { + Status = EStatus::SUCCESS; + } + if (!msg.empty()) { + Issues.AddIssue(NYdb::NIssue::TIssue(msg)); + } + for (const auto& [name, value] : grpcStatus.ServerTrailingMetadata) { + Metadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end()) + ); + } +} + +TPlainStatus TPlainStatus::Internal(const std::string& message) { + return { EStatus::CLIENT_INTERNAL_ERROR, "Internal client error: " + message }; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.h new file mode 100644 index 000000000000..cf5fe5110b81 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/status.h @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include + +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +struct TPlainStatus { + EStatus Status; + NYdb::NIssue::TIssues Issues; + std::string Endpoint; + std::multimap Metadata; + Ydb::CostInfo ConstInfo; + + TPlainStatus() + : Status(EStatus::SUCCESS) + { } + + TPlainStatus(EStatus status, NYdb::NIssue::TIssues&& issues) + : Status(status) + , Issues(std::move(issues)) + { } + + TPlainStatus(EStatus status, NYdb::NIssue::TIssues&& issues, const std::string& endpoint, + std::multimap&& metadata) + : Status(status) + , Issues(std::move(issues)) + , Endpoint(endpoint) + , Metadata(std::move(metadata)) + { } + + TPlainStatus(EStatus status, const std::string& message) + : Status(status) + { + if (!message.empty()) { + Issues.AddIssue(NYdb::NIssue::TIssue(message)); + } + } + + TPlainStatus( + const NYdbGrpc::TGrpcStatus& grpcStatus, const std::string& endpoint = std::string(), + std::multimap&& metadata = {}); + + template + void SetCostInfo(T&& costInfo) { + ConstInfo = std::forward(costInfo); + } + + bool Ok() const { + return Status == EStatus::SUCCESS; + } + + static TPlainStatus Internal(const std::string& message); + + bool IsTransportError() const { + auto status = static_cast(Status); + return TRANSPORT_STATUSES_FIRST <= status && status <= TRANSPORT_STATUSES_LAST; + } + + TStringBuilder ToDebugString() const { + TStringBuilder ret; + ret << "Status: " << Status; + if(!Ok()) + ret << ", Description: " << SubstGlobalCopy(Issues.ToString(), '\n', ' '); + return ret; + } + + +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/ya.make new file mode 100644 index 000000000000..630e2aab5655 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + status.cpp +) + +PEERDIR( + contrib/libs/protobuf + ydb/public/sdk/cpp/src/library/grpc/client + ydb/public/sdk/cpp/src/library/issue +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/CMakeLists.txt new file mode 100644 index 000000000000..03440c81d2b7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(impl-ydb_internal-retry) + +target_link_libraries(impl-ydb_internal-retry PUBLIC + yutil + impl-ydb_internal-grpc_connections +) + +target_sources(impl-ydb_internal-retry PRIVATE + retry.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-retry) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.cpp new file mode 100644 index 000000000000..dbeb516677d5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.cpp @@ -0,0 +1,41 @@ +#include "retry.h" + +#include + +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NRetry { + +constexpr ui32 MAX_BACKOFF_DURATION_MS = TDuration::Hours(1).MilliSeconds(); + +ui32 CalcBackoffTime(const TBackoffSettings& settings, ui32 retryNumber) { + ui32 backoffSlots = 1 << std::min(retryNumber, settings.Ceiling_); + TDuration maxDuration = settings.SlotDuration_ * backoffSlots; + + double uncertaintyRatio = std::max(std::min(settings.UncertainRatio_, 1.0), 0.0); + double uncertaintyMultiplier = RandomNumber() * uncertaintyRatio - uncertaintyRatio + 1.0; + + double durationMs = round(maxDuration.MilliSeconds() * uncertaintyMultiplier); + + return std::max(std::min(durationMs, (double)MAX_BACKOFF_DURATION_MS), 0.0); +} + +void Backoff(const NRetry::TBackoffSettings& settings, ui32 retryNumber) { + auto durationMs = CalcBackoffTime(settings, retryNumber); + Sleep(TDuration::MilliSeconds(durationMs)); +} + +void AsyncBackoff(std::shared_ptr client, const TBackoffSettings& settings, + ui32 retryNumber, const std::function& fn) +{ + auto durationMs = CalcBackoffTime(settings, retryNumber); + client->ScheduleTask(fn, TDuration::MilliSeconds(durationMs)); +} + +} + diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.h new file mode 100644 index 000000000000..d8f5f8bb792e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3 { +class IClientImplCommon; +} + +namespace NYdb::inline V3::NRetry { + +ui32 CalcBackoffTime(const TBackoffSettings& settings, ui32 retryNumber); +void Backoff(const NRetry::TBackoffSettings& settings, ui32 retryNumber); +void AsyncBackoff(std::shared_ptr client, const TBackoffSettings& settings, + ui32 retryNumber, const std::function& fn); + +enum class NextStep { + RetryImmediately, + RetryFastBackoff, + RetrySlowBackoff, + Finish, +}; + +class TRetryContextBase : TNonCopyable { +protected: + TRetryOperationSettings Settings_; + ui32 RetryNumber_; + TInstant RetryStartTime_; + +protected: + TRetryContextBase(const TRetryOperationSettings& settings) + : Settings_(settings) + , RetryNumber_(0) + , RetryStartTime_(TInstant::Now()) + {} + + virtual void Reset() {} + + void LogRetry(const TStatus& status) { + if (Settings_.Verbose_) { + std::cerr << "Previous query attempt was finished with unsuccessful status " + << ToString(status.GetStatus()) << ": " << status.GetIssues().ToString(true) << std::endl; + std::cerr << "Sending retry attempt " << RetryNumber_ << " of " << Settings_.MaxRetries_ << std::endl; + } + } + + NextStep GetNextStep(const TStatus& status) { + if (status.IsSuccess()) { + return NextStep::Finish; + } + if (RetryNumber_ >= Settings_.MaxRetries_) { + return NextStep::Finish; + } + if (TInstant::Now() - RetryStartTime_ >= Settings_.MaxTimeout_) { + return NextStep::Finish; + } + switch (status.GetStatus()) { + case EStatus::ABORTED: + return NextStep::RetryImmediately; + + case EStatus::OVERLOADED: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + return NextStep::RetrySlowBackoff; + + case EStatus::UNAVAILABLE: + return NextStep::RetryFastBackoff; + + case EStatus::BAD_SESSION: + case EStatus::SESSION_BUSY: + Reset(); + return NextStep::RetryImmediately; + + case EStatus::NOT_FOUND: + if (Settings_.RetryNotFound_) { + return NextStep::RetryImmediately; + } else { + return NextStep::Finish; + } + + case EStatus::UNDETERMINED: + if (Settings_.Idempotent_) { + return NextStep::RetryFastBackoff; + } else { + return NextStep::Finish; + } + + case EStatus::TRANSPORT_UNAVAILABLE: + if (Settings_.Idempotent_) { + Reset(); + return NextStep::RetryFastBackoff; + } else { + return NextStep::Finish; + } + + default: + return Settings_.RetryUndefined_ ? NextStep::RetrySlowBackoff : NextStep::Finish; + } + } + + TDuration GetRemainingTimeout() { + return Settings_.MaxTimeout_ - (TInstant::Now() - RetryStartTime_); + } +}; + +} // namespace NYdb::NRetry diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_async.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_async.h new file mode 100644 index 000000000000..f19a0bf3371b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_async.h @@ -0,0 +1,173 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3::NRetry::Async { + +template +class TRetryContext : public TThrRefBase, public TRetryContextBase { +public: + using TStatusType = typename TAsyncStatusType::value_type; + using TPtr = TIntrusivePtr>; + +protected: + TClient Client_; + NThreading::TPromise Promise_; + +public: + TAsyncStatusType Execute() { + this->RetryStartTime_ = TInstant::Now(); + this->Retry(); + return this->Promise_.GetFuture(); + } + +protected: + explicit TRetryContext(const TClient& client, const TRetryOperationSettings& settings) + : TRetryContextBase(settings) + , Client_(client) + , Promise_(NThreading::NewPromise()) + {} + + virtual void Retry() = 0; + + virtual TAsyncStatusType RunOperation() = 0; + + static void DoRetry(TPtr self) { + self->Retry(); + } + + static void DoBackoff(TPtr self, bool fast) { + auto backoffSettings = fast ? self->Settings_.FastBackoffSettings_ + : self->Settings_.SlowBackoffSettings_; + AsyncBackoff(self->Client_.Impl_, backoffSettings, self->RetryNumber_, + [self]() {DoRetry(self);}); + } + + static void HandleExceptionAsync(TPtr self, std::exception_ptr e) { + self->Promise_.SetException(e); + } + + static void HandleStatusAsync(TPtr self, const TStatusType& status) { + auto nextStep = self->GetNextStep(status); + if (nextStep != NextStep::Finish) { + self->RetryNumber_++; + self->Client_.Impl_->CollectRetryStatAsync(status.GetStatus()); + self->LogRetry(status); + } + switch (nextStep) { + case NextStep::RetryImmediately: + return DoRetry(self); + case NextStep::RetryFastBackoff: + return DoBackoff(self, true); + case NextStep::RetrySlowBackoff: + return DoBackoff(self, false); + case NextStep::Finish: + return self->Promise_.SetValue(status); + } + } + + static void DoRunOperation(TPtr self) { + self->RunOperation().Subscribe( + [self](const TAsyncStatusType& result) { + try { + HandleStatusAsync(self, result.GetValue()); + } catch (...) { + HandleExceptionAsync(self, std::current_exception()); + } + } + ); + } +}; + +template > +class TRetryWithoutSession : public TRetryContext { + using TRetryContext = TRetryContext; + using TPtr = typename TRetryContext::TPtr; + +private: + TOperation Operation_; + +public: + explicit TRetryWithoutSession( + const TClient& client, TOperation&& operation, const TRetryOperationSettings& settings) + : TRetryContext(client, settings) + , Operation_(operation) + {} + + void Retry() override { + TPtr self(this); + TRetryContext::DoRunOperation(self); + } + +protected: + TAsyncStatusType RunOperation() override { + if constexpr (TFunctionArgs::Length == 1) { + return Operation_(this->Client_); + } else { + return Operation_(this->Client_, this->GetRemainingTimeout()); + } + } +}; + +template > +class TRetryWithSession : public TRetryContext { + using TRetryContextAsync = TRetryContext; + using TPtr = typename TRetryContextAsync::TPtr; + using TStatusType = typename TRetryContextAsync::TStatusType; + using TSession = typename TClient::TSession; + using TCreateSessionSettings = typename TClient::TCreateSessionSettings; + using TAsyncCreateSessionResult = typename TClient::TAsyncCreateSessionResult; + +private: + TOperation Operation_; + std::optional Session_; + +public: + explicit TRetryWithSession( + const TClient& client, TOperation&& operation, const TRetryOperationSettings& settings) + : TRetryContextAsync(client, settings) + , Operation_(operation) + {} + + void Retry() override { + TPtr self(this); + if (!Session_) { + auto settings = TCreateSessionSettings().ClientTimeout(this->Settings_.GetSessionClientTimeout_); + this->Client_.GetSession(settings).Subscribe( + [self](const TAsyncCreateSessionResult& resultFuture) { + try { + auto& result = resultFuture.GetValue(); + if (!result.IsSuccess()) { + return TRetryContextAsync::HandleStatusAsync(self, TStatusType(TStatus(result))); + } + + auto* myself = dynamic_cast(self.Get()); + myself->Session_ = result.GetSession(); + myself->DoRunOperation(self); + } catch (...) { + return TRetryContextAsync::HandleExceptionAsync(self, std::current_exception()); + } + } + ); + } else { + TRetryContextAsync::DoRunOperation(self); + } + } + +private: + void Reset() override { + Session_.reset(); + } + + TAsyncStatusType RunOperation() override { + if constexpr (TFunctionArgs::Length == 1) { + return Operation_(this->Session_.value()); + } else { + return Operation_(this->Session_.value(), this->GetRemainingTimeout()); + } + } +}; + +} // namespace NYdb::NRetry::Async diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_sync.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_sync.h new file mode 100644 index 000000000000..b0c63187bddd --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/retry_sync.h @@ -0,0 +1,131 @@ +#pragma once + +#include +#include +#include + +namespace NYdb::inline V3::NRetry::Sync { + +template +class TRetryContext : public TRetryContextBase { +protected: + TClient& Client_; + +public: + TStatusType Execute() { + this->RetryStartTime_ = TInstant::Now(); + TStatusType status = Retry(); // first attempt + for (this->RetryNumber_ = 0; this->RetryNumber_ <= this->Settings_.MaxRetries_;) { + auto nextStep = this->GetNextStep(status); + switch (nextStep) { + case NextStep::RetryImmediately: + break; + case NextStep::RetryFastBackoff: + DoBackoff(true); + break; + case NextStep::RetrySlowBackoff: + DoBackoff(false); + break; + case NextStep::Finish: + return status; + } + // make next retry + this->RetryNumber_++; + this->LogRetry(status); + this->Client_.Impl_->CollectRetryStatSync(status.GetStatus()); + status = Retry(); + } + return status; + } + +protected: + TRetryContext(TClient& client, const TRetryOperationSettings& settings) + : TRetryContextBase(settings) + , Client_(client) + {} + + virtual TStatusType Retry() = 0; + + virtual TStatusType RunOperation() = 0; + + void DoBackoff(bool fast) { + const auto &settings = fast ? this->Settings_.FastBackoffSettings_ + : this->Settings_.SlowBackoffSettings_; + Backoff(settings, this->RetryNumber_); + } +}; + +template> +class TRetryWithoutSession : public TRetryContext { +private: + const TOperation& Operation_; + +public: + TRetryWithoutSession(TClient& client, const TOperation& operation, const TRetryOperationSettings& settings) + : TRetryContext(client, settings) + , Operation_(operation) + {} + +protected: + TStatusType Retry() override { + return RunOperation(); + } + + TStatusType RunOperation() override { + if constexpr (TFunctionArgs::Length == 1) { + return Operation_(this->Client_); + } else { + return Operation_(this->Client_, this->GetRemainingTimeout()); + } + } +}; + +template> +class TRetryWithSession : public TRetryContext { + using TSession = typename TClient::TSession; + using TCreateSessionSettings = typename TClient::TCreateSessionSettings; + +private: + const TOperation& Operation_; + std::optional Session_; + +public: + TRetryWithSession(TClient& client, const TOperation& operation, const TRetryOperationSettings& settings) + : TRetryContext(client, settings) + , Operation_(operation) + {} + +protected: + TStatusType Retry() override { + std::optional status; + + if (!Session_) { + auto settings = TCreateSessionSettings().ClientTimeout(this->Settings_.GetSessionClientTimeout_); + auto sessionResult = this->Client_.GetSession(settings).GetValueSync(); + if (sessionResult.IsSuccess()) { + Session_ = sessionResult.GetSession(); + } + status = TStatusType(TStatus(sessionResult)); + } + + if (Session_) { + status = RunOperation(); + } + + return *status; + } + + TStatusType RunOperation() override { + if constexpr (TFunctionArgs::Length == 1) { + return Operation_(this->Session_.value()); + } else { + return Operation_(this->Session_.value(), this->GetRemainingTimeout()); + } + } + + void Reset() override { + Session_.reset(); + } +}; + +} // namespace NYdb::NRetry::Sync diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/ya.make new file mode 100644 index 000000000000..da50cdb6a29f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + retry.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections +) + +END() + diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/rpc_request_settings/settings.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/rpc_request_settings/settings.h new file mode 100644 index 000000000000..515d20f64226 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/rpc_request_settings/settings.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { + +struct TRpcRequestSettings { + std::string TraceId; + std::string RequestType; + std::vector> Header; + TEndpointKey PreferredEndpoint = {}; + enum class TEndpointPolicy { + UsePreferredEndpointOptionally, // Try to use the preferred endpoint + UsePreferredEndpointStrictly, // Use only the preferred endpoint + UseDiscoveryEndpoint // Use single discovery endpoint + } EndpointPolicy = TEndpointPolicy::UsePreferredEndpointOptionally; + bool UseAuth = true; + TDuration ClientTimeout; + + template + static TRpcRequestSettings Make(const TRequestSettings& settings, const TEndpointKey& preferredEndpoint = {}, TEndpointPolicy endpointPolicy = TEndpointPolicy::UsePreferredEndpointOptionally) { + TRpcRequestSettings rpcSettings; + rpcSettings.TraceId = settings.TraceId_; + rpcSettings.RequestType = settings.RequestType_; + rpcSettings.Header = settings.Header_; + rpcSettings.PreferredEndpoint = preferredEndpoint; + rpcSettings.EndpointPolicy = endpointPolicy; + rpcSettings.UseAuth = true; + rpcSettings.ClientTimeout = settings.ClientTimeout_; + return rpcSettings; + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/scheme_helpers/helpers.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/scheme_helpers/helpers.h new file mode 100644 index 000000000000..5a4c8528b199 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/scheme_helpers/helpers.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { + +template +inline void PermissionToSchemeEntry(const TFrom& from, std::vector* to) { + for (const auto& effPerm : from) { + to->push_back(NScheme::TPermissions{effPerm.subject(), std::vector()}); + for (const auto& permName : effPerm.permission_names()) { + to->back().PermissionNames.push_back(permName); + } + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_client/session_client.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_client/session_client.h new file mode 100644 index 000000000000..db920a7ab90a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_client/session_client.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +class TKqpSessionCommon; + +class ISessionClient { +public: + virtual ~ISessionClient() = default; + virtual void DeleteSession(TKqpSessionCommon* sessionImpl) = 0; + // TODO: Try to remove from ISessionClient + virtual bool ReturnSession(TKqpSessionCommon* sessionImpl) = 0; +}; + +} diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/CMakeLists.txt new file mode 100644 index 000000000000..91577a39695a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/CMakeLists.txt @@ -0,0 +1,15 @@ +_ydb_sdk_add_library(impl-ydb_internal-session_pool) + +target_link_libraries(impl-ydb_internal-session_pool PUBLIC + yutil + threading-future + api-protos + client-impl-ydb_endpoints + client-ydb_types-operation +) + +target_sources(impl-ydb_internal-session_pool PRIVATE + session_pool.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-session_pool) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.cpp new file mode 100644 index 000000000000..ff9a6db5adec --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.cpp @@ -0,0 +1,386 @@ +#include "session_pool.h" + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NSessionPool { + +using namespace NThreading; + +constexpr ui64 KEEP_ALIVE_RANDOM_FRACTION = 4; +static const TStatus CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT = TStatus( + TPlainStatus( + EStatus::CLIENT_RESOURCE_EXHAUSTED, + "Active sessions limit exceeded" + ) + ); + +TStatus GetStatus(const TOperation& operation) { + return operation.Status(); +} + +TStatus GetStatus(const TStatus& status) { + return status; +} + +TDuration RandomizeThreshold(TDuration duration) { + TDuration::TValue value = duration.GetValue(); + if (KEEP_ALIVE_RANDOM_FRACTION) { + const i64 randomLimit = value / KEEP_ALIVE_RANDOM_FRACTION; + if (randomLimit < 2) + return duration; + value += static_cast(RandomNumber(randomLimit)); + } + return TDuration::FromValue(value); +} + +bool IsSessionCloseRequested(const TStatus& status) { + const auto& meta = status.GetResponseMetadata(); + auto hints = meta.equal_range(NYdb::YDB_SERVER_HINTS); + for(auto it = hints.first; it != hints.second; ++it) { + if (it->second == NYdb::YDB_SESSION_CLOSE) { + return true; + } + } + + return false; +} + +TSessionPool::TWaitersQueue::TWaitersQueue(ui32 maxQueueSize, TDuration maxWaitSessionTimeout) + : MaxQueueSize_(maxQueueSize) + , MaxWaitSessionTimeout_(maxWaitSessionTimeout) +{ +} + +bool TSessionPool::TWaitersQueue::TryPush(std::unique_ptr& p) { + if (Waiters_.size() < MaxQueueSize_) { + Waiters_.insert(std::make_pair(TInstant::Now(), std::move(p))); + return true; + } + return false; +} + +std::unique_ptr TSessionPool::TWaitersQueue::TryGet() { + if (Waiters_.empty()) { + return {}; + } + auto it = Waiters_.begin(); + auto result = std::move(it->second); + Waiters_.erase(it); + return result; +} + +void TSessionPool::TWaitersQueue::GetOld(TInstant now, std::vector>& oldWaiters) { + auto it = Waiters_.begin(); + while (it != Waiters_.end()) { + if (now < it->first + MaxWaitSessionTimeout_) + break; + + oldWaiters.emplace_back(std::move(it->second)); + + Waiters_.erase(it++); + } +} + +ui32 TSessionPool::TWaitersQueue::Size() const { + return Waiters_.size(); +} + + +TSessionPool::TSessionPool(ui32 maxActiveSessions) + : Closed_(false) + , WaitersQueue_(maxActiveSessions * 10) + , ActiveSessions_(0) + , MaxActiveSessions_(maxActiveSessions) +{} + +static void CloseAndDeleteSession(std::unique_ptr&& impl, + std::shared_ptr client) { + auto deleter = TKqpSessionCommon::GetSmartDeleter(client); + TKqpSessionCommon* p = impl.release(); + p->MarkBroken(); + deleter(p); +} + +void TSessionPool::ReplySessionToUser( + TKqpSessionCommon* session, + std::unique_ptr ctx) +{ + Y_ABORT_UNLESS(session->GetState() == TKqpSessionCommon::S_IDLE); + Y_ABORT_UNLESS(!session->GetTimeInterval()); + session->MarkActive(); + session->SetNeedUpdateActiveCounter(true); + ctx->ReplySessionToUser(session); +} + +void TSessionPool::GetSession(std::unique_ptr ctx) +{ + std::unique_ptr sessionImpl; + enum class TSessionSource { + Pool, + Waiter, + Error + } sessionSource = TSessionSource::Pool; + + { + std::lock_guard guard(Mtx_); + + if (MaxActiveSessions_ == 0 || ActiveSessions_ < MaxActiveSessions_) { + IncrementActiveCounterUnsafe(); + } else if (WaitersQueue_.TryPush(ctx)) { + sessionSource = TSessionSource::Waiter; + } else { + sessionSource = TSessionSource::Error; + } + if (!Sessions_.empty()) { + auto it = std::prev(Sessions_.end()); + it->second->UpdateServerCloseHandler(nullptr); + sessionImpl = std::move(it->second); + Sessions_.erase(it); + } + + UpdateStats(); + } + + if (sessionSource == TSessionSource::Waiter) { + // Nothing to do here + } else if (sessionSource == TSessionSource::Error) { + FakeSessionsCounter_.Inc(); + ctx->ReplyError(CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT); + } else if (sessionImpl) { + ReplySessionToUser(sessionImpl.release(), std::move(ctx)); + } else { + ctx->ReplyNewSession(); + } +} + +bool TSessionPool::CheckAndFeedWaiterNewSession(bool active) { + std::unique_ptr getSessionCtx; + { + std::lock_guard guard(Mtx_); + if (Closed_) + return false; + + if (auto maybeCtx = WaitersQueue_.TryGet()) { + getSessionCtx = std::move(maybeCtx); + } else { + return false; + } + } + + if (!active) { + // Session was IDLE. It means session has been closed during + // keep-alive activity inside session pool. In this case + // we must not touch ActiveSession counter. + // Moreover it is unsafe to recreate the session because session + // may be closed during balancing or draining routine. + // But we must feed waiters if we have some otherwise deadlock + // is possible. + // The above mentioned conditions is quite rare so + // we can just return an error. In this case uplevel code should + // start retry. + getSessionCtx->ReplyError(CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT); + return true; + } + + getSessionCtx->ReplyNewSession(); + return true; +} + +bool TSessionPool::ReturnSession(TKqpSessionCommon* impl, bool active) { + // Do not call ReplySessionToUser under the session pool lock + std::unique_ptr getSessionCtx; + { + std::lock_guard guard(Mtx_); + if (Closed_) + return false; + + if (auto maybeCtx = WaitersQueue_.TryGet()) { + getSessionCtx = std::move(maybeCtx); + if (!active) + IncrementActiveCounterUnsafe(); + } else { + impl->UpdateServerCloseHandler(this); + Sessions_.emplace(std::make_pair( + impl->GetTimeToTouchFast(), + impl)); + + if (active) { + Y_ABORT_UNLESS(ActiveSessions_); + ActiveSessions_--; + impl->SetNeedUpdateActiveCounter(false); + } + } + UpdateStats(); + } + + if (getSessionCtx) { + ReplySessionToUser(impl, std::move(getSessionCtx)); + } + + return true; +} + +void TSessionPool::DecrementActiveCounter() { + std::lock_guard guard(Mtx_); + Y_ABORT_UNLESS(ActiveSessions_); + ActiveSessions_--; + UpdateStats(); +} + +void TSessionPool::IncrementActiveCounterUnsafe() { + ActiveSessions_++; + UpdateStats(); +} + +void TSessionPool::Drain(std::function&&)> cb, bool close) { + std::lock_guard guard(Mtx_); + Closed_ = close; + for (auto it = Sessions_.begin(); it != Sessions_.end();) { + it->second->UpdateServerCloseHandler(nullptr); + const bool cont = cb(std::move(it->second)); + it = Sessions_.erase(it); + if (!cont) + break; + } + UpdateStats(); +} + +TPeriodicCb TSessionPool::CreatePeriodicTask(std::weak_ptr weakClient, + TKeepAliveCmd&& cmd, TDeletePredicate&& deletePredicate) +{ + auto periodicCb = [this, weakClient, cmd=std::move(cmd), deletePredicate=std::move(deletePredicate)](NYdb::NIssue::TIssues&&, EStatus status) { + if (status != EStatus::SUCCESS) { + return false; + } + + auto strongClient = weakClient.lock(); + if (!strongClient) { + // No more clients alive - no need to run periodic, + // moreover it is unsafe to touch this ptr! + return false; + } else { + auto keepAliveBatchSize = PERIODIC_ACTION_BATCH_SIZE; + std::vector> sessionsToTouch; + sessionsToTouch.reserve(keepAliveBatchSize); + std::vector> sessionsToDelete; + sessionsToDelete.reserve(keepAliveBatchSize); + std::vector> waitersToReplyError; + waitersToReplyError.reserve(keepAliveBatchSize); + const auto now = TInstant::Now(); + { + std::lock_guard guard(Mtx_); + { + auto& sessions = Sessions_; + + auto it = sessions.begin(); + while (it != sessions.end() && keepAliveBatchSize--) { + if (now < it->second->GetTimeToTouchFast()) + break; + + if (deletePredicate(it->second.get(), sessions.size())) { + it->second->UpdateServerCloseHandler(nullptr); + sessionsToDelete.emplace_back(std::move(it->second)); + sessions.erase(it++); + } else if (cmd) { + it->second->UpdateServerCloseHandler(nullptr); + sessionsToTouch.emplace_back(std::move(it->second)); + sessions.erase(it++); + } else { + it++; + } + } + } + + WaitersQueue_.GetOld(now, waitersToReplyError); + + UpdateStats(); + } + + for (auto& sessionImpl : sessionsToTouch) { + if (sessionImpl) { + Y_ABORT_UNLESS(sessionImpl->GetState() == TKqpSessionCommon::S_IDLE); + cmd(sessionImpl.release()); + } + } + + for (auto& sessionImpl : sessionsToDelete) { + if (sessionImpl) { + Y_ABORT_UNLESS(sessionImpl->GetState() == TKqpSessionCommon::S_IDLE); + CloseAndDeleteSession(std::move(sessionImpl), strongClient); + } + } + + for (auto& waiter : waitersToReplyError) { + FakeSessionsCounter_.Inc(); + waiter->ReplyError(CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT); + } + } + + return true; + }; + return periodicCb; +} + +i64 TSessionPool::GetActiveSessions() const { + std::lock_guard guard(Mtx_); + return ActiveSessions_; +} + +i64 TSessionPool::GetActiveSessionsLimit() const { + return MaxActiveSessions_; +} + +i64 TSessionPool::GetCurrentPoolSize() const { + std::lock_guard guard(Mtx_); + return Sessions_.size(); +} + +void TSessionPool::OnCloseSession(const TKqpSessionCommon* s, std::shared_ptr client) { + std::unique_ptr session; + { + std::lock_guard guard(Mtx_); + const auto timeToTouch = s->GetTimeToTouchFast(); + const auto id = s->GetId(); + auto it = Sessions_.find(timeToTouch); + // Sessions_ is multimap of sessions sorted by scheduled time to run periodic task + // Scan sessions with same scheduled time to find needed one. In most cases only one session here + while (it != Sessions_.end() && it->first == timeToTouch) { + if (id != it->second->GetId()) { + it++; + continue; + } + session = std::move(it->second); + Sessions_.erase(it); + break; + } + } + + if (session) { + Y_ABORT_UNLESS(session->GetState() == TKqpSessionCommon::S_IDLE); + CloseAndDeleteSession(std::move(session), client); + } +} + +void TSessionPool::SetStatCollector(NSdkStats::TStatCollector::TSessionPoolStatCollector statCollector) { + ActiveSessionsCounter_.Set(statCollector.ActiveSessions); + InPoolSessionsCounter_.Set(statCollector.InPoolSessions); + FakeSessionsCounter_.Set(statCollector.FakeSessions); + SessionWaiterCounter_.Set(statCollector.Waiters); +} + +void TSessionPool::UpdateStats() { + ActiveSessionsCounter_.Apply(ActiveSessions_); + InPoolSessionsCounter_.Apply(Sessions_.size()); + SessionWaiterCounter_.Apply(WaitersQueue_.Size()); +} + +} +} diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.h new file mode 100644 index 000000000000..78394bbcc126 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/session_pool.h @@ -0,0 +1,149 @@ +#pragma once + +#include +#include + + +namespace NYdb::inline V3 { + +class TOperation; +namespace NSessionPool { + +class IGetSessionCtx : private TNonCopyable { +public: + virtual ~IGetSessionCtx() = default; + // Transfer ownership to ctx + virtual void ReplySessionToUser(TKqpSessionCommon* session) = 0; + virtual void ReplyError(TStatus status) = 0; + virtual void ReplyNewSession() = 0; +}; + +//How often run session pool keep alive check +constexpr TDuration PERIODIC_ACTION_INTERVAL = TDuration::Seconds(5); +constexpr TDuration MAX_WAIT_SESSION_TIMEOUT = TDuration::Seconds(5); //Max time to wait session +constexpr ui64 PERIODIC_ACTION_BATCH_SIZE = 10; //Max number of tasks to perform during one interval +constexpr TDuration CREATE_SESSION_INTERNAL_TIMEOUT = TDuration::Seconds(2); //Timeout for createSession call inside session pool + +TStatus GetStatus(const TOperation& operation); +TStatus GetStatus(const TStatus& status); + +TDuration RandomizeThreshold(TDuration duration); +bool IsSessionCloseRequested(const TStatus& status); + +template +NThreading::TFuture InjectSessionStatusInterception( + std::shared_ptr<::NYdb::TKqpSessionCommon> impl, NThreading::TFuture asyncResponse, + bool updateTimeout, + TDuration timeout, + std::function cb = {}) +{ + auto promise = NThreading::NewPromise(); + asyncResponse.Subscribe([impl, promise, cb, updateTimeout, timeout](NThreading::TFuture future) mutable { + Y_ABORT_UNLESS(future.HasValue()); + + // TResponse can hold refcounted user provided data (TSession for example) + // and we do not want to have copy of it (for example it can cause delay dtor call) + // so using move semantic here is mandatory. + // Also we must reset captured shared pointer to session impl + TResponse value = std::move(future.ExtractValue()); + + const TStatus& status = GetStatus(value); + // Exclude CLIENT_RESOURCE_EXHAUSTED from transport errors which can cause to session disconnect + // since we have guarantee this request wasn't been started to execute. + + if (status.IsTransportError() + && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED && status.GetStatus() != EStatus::CLIENT_OUT_OF_RANGE) + { + impl->MarkBroken(); + } else if (status.GetStatus() == EStatus::SESSION_BUSY) { + impl->MarkBroken(); + } else if (status.GetStatus() == EStatus::BAD_SESSION) { + impl->MarkBroken(); + } else if (IsSessionCloseRequested(status)) { + impl->MarkAsClosing(); + } else { + // NOTE: About GetState and lock + // Simultanious call multiple requests on the same session make no sence, due to server limitation. + // But user can perform this call, right now we do not protect session from this, it may cause + // raise on session state if respoise is not success. + // It should not be a problem - in case of this race we close session + // or put it in to settler. + if (updateTimeout && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED) { + impl->ScheduleTimeToTouch(RandomizeThreshold(timeout), impl->GetState() == TKqpSessionCommon::EState::S_ACTIVE); + } + } + + if (cb) { + cb(value, *impl); + } + impl.reset(); + promise.SetValue(std::move(value)); + }); + return promise.GetFuture(); +} + +class TSessionPool : public IServerCloseHandler { +private: + class TWaitersQueue { + public: + TWaitersQueue(ui32 maxQueueSize, TDuration maxWaitSessionTimeout=MAX_WAIT_SESSION_TIMEOUT); + + // returns true and gets ownership if queue size less than limit + // otherwise returns false and doesn't not touch ctx + bool TryPush(std::unique_ptr& p); + std::unique_ptr TryGet(); + void GetOld(TInstant now, std::vector>& oldWaiters); + ui32 Size() const; + + private: + const ui32 MaxQueueSize_; + const TDuration MaxWaitSessionTimeout_; + std::multimap> Waiters_; + }; +public: + using TKeepAliveCmd = std::function; + using TDeletePredicate = std::function; + TSessionPool(ui32 maxActiveSessions); + + // Extracts session from pool or creates new one ising given ctx + void GetSession(std::unique_ptr ctx); + + // Returns true if session returned to pool successfully + bool ReturnSession(TKqpSessionCommon* impl, bool active); + + // Returns trun if has waiter and scheduled to create new session + // too feed it + bool CheckAndFeedWaiterNewSession(bool active); + + TPeriodicCb CreatePeriodicTask(std::weak_ptr weakClient, TKeepAliveCmd&& cmd, TDeletePredicate&& predicate); + i64 GetActiveSessions() const; + i64 GetActiveSessionsLimit() const; + i64 GetCurrentPoolSize() const; + void DecrementActiveCounter(); + void IncrementActiveCounterUnsafe(); + + void Drain(std::function&&)> cb, bool close); + void SetStatCollector(NSdkStats::TStatCollector::TSessionPoolStatCollector collector); + + void OnCloseSession(const TKqpSessionCommon*, std::shared_ptr client) override; + +private: + void UpdateStats(); + static void ReplySessionToUser(TKqpSessionCommon* session, std::unique_ptr ctx); + + mutable std::mutex Mtx_; + bool Closed_; + + std::multimap> Sessions_; + TWaitersQueue WaitersQueue_; + + i64 ActiveSessions_; + const ui32 MaxActiveSessions_; + NSdkStats::TSessionCounter ActiveSessionsCounter_; + NSdkStats::TSessionCounter InPoolSessionsCounter_; + NSdkStats::TSessionCounter SessionWaiterCounter_; + NSdkStats::TAtomicCounter<::NMonitoring::TRate> FakeSessionsCounter_; +}; + +} +} diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/ya.make new file mode 100644 index 000000000000..f12dffa2fc33 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + session_pool.cpp +) + +PEERDIR( + library/cpp/threading/future + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/impl/ydb_endpoints + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/stats_extractor/extractor.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/stats_extractor/extractor.h new file mode 100644 index 000000000000..964f30fea0bf --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/stats_extractor/extractor.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +class TStatsExtractor: public NSdkStats::IStatApi { +public: + + TStatsExtractor(std::shared_ptr client) + : Client_(client) + { } + + virtual void SetMetricRegistry(::NMonitoring::IMetricRegistry* sensorsRegistry) override { + auto strong = Client_.lock(); + if (strong) { + strong->StartStatCollecting(sensorsRegistry); + } else { + return; + } + } + + void Accept(NMonitoring::IMetricConsumer* consumer) const override { + + auto strong = Client_.lock(); + if (strong) { + auto sensorsRegistry = strong->GetMetricRegistry(); + Y_ABORT_UNLESS(sensorsRegistry, "TMetricRegistry is null in Stats Extractor"); + sensorsRegistry->Accept(TInstant::Zero(), consumer); + } else { + throw NSdkStats::DestroyedClientException(); + } + } +private: + std::weak_ptr Client_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/table_helpers/helpers.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/table_helpers/helpers.h new file mode 100644 index 000000000000..f28e7cc16601 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/table_helpers/helpers.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include +#include + +namespace NYdb::inline V3 { + +inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(std::optional mode) { + if (mode.has_value()) { + switch (*mode) { + case NTable::ECollectQueryStatsMode::None: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_NONE; + case NTable::ECollectQueryStatsMode::Basic: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC; + case NTable::ECollectQueryStatsMode::Full: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL; + case NTable::ECollectQueryStatsMode::Profile: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_PROFILE; + default: + break; + } + } + + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_UNSPECIFIED; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/CMakeLists.txt new file mode 100644 index 000000000000..386db7a454f6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(impl-ydb_internal-thread_pool) + +target_link_libraries(impl-ydb_internal-thread_pool PUBLIC + yutil +) + +target_sources(impl-ydb_internal-thread_pool PRIVATE + pool.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-thread_pool) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.cpp new file mode 100644 index 000000000000..7fbf33277f0c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.cpp @@ -0,0 +1,2 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "pool.h" diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.h new file mode 100644 index 000000000000..f29a7bb62381 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/pool.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +inline std::unique_ptr CreateThreadPool(size_t threads) { + std::unique_ptr queue; + if (threads) { + queue.reset(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false))); + } else { + queue.reset(new TAdaptiveThreadPool()); + } + return queue; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/ya.make new file mode 100644 index 000000000000..f5e706f1e54f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/thread_pool/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + pool.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/CMakeLists.txt new file mode 100644 index 000000000000..00591fe67c9e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(impl-ydb_internal-value_helpers) + +target_link_libraries(impl-ydb_internal-value_helpers PUBLIC + yutil + api-protos + client-ydb_types-fatal_error_handlers +) + +target_sources(impl-ydb_internal-value_helpers PRIVATE + helpers.cpp +) + +_ydb_sdk_install_targets(TARGETS impl-ydb_internal-value_helpers) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.cpp new file mode 100644 index 000000000000..ebfbe83c8320 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.cpp @@ -0,0 +1,112 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "helpers.h" +#include + +#include + +namespace NYdb::inline V3 { + +bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) { + if (t1.type_case() != t2.type_case()) { + return false; + } + + switch (t1.type_case()) { + case Ydb::Type::kTypeId: + return t1.type_id() == t2.type_id(); + case Ydb::Type::kDecimalType: + return t1.decimal_type().precision() == t2.decimal_type().precision() + && t1.decimal_type().scale() == t2.decimal_type().scale(); + case Ydb::Type::kPgType: + return t1.pg_type().type_name() == t2.pg_type().type_name() + && t1.pg_type().type_modifier() == t2.pg_type().type_modifier(); + case Ydb::Type::kOptionalType: + return TypesEqual(t1.optional_type().item(), t2.optional_type().item()); + case Ydb::Type::kTaggedType: + return t1.tagged_type().tag() == t2.tagged_type().tag() && + TypesEqual(t1.tagged_type().type(), t2.tagged_type().type()); + case Ydb::Type::kListType: + return TypesEqual(t1.list_type().item(), t2.list_type().item()); + case Ydb::Type::kTupleType: + if (t1.tuple_type().elements_size() != t2.tuple_type().elements_size()) { + return false; + } + for (size_t i = 0; i < (size_t)t1.tuple_type().elements_size(); ++i) { + if (!TypesEqual(t1.tuple_type().elements(i), t2.tuple_type().elements(i))) { + return false; + } + } + return true; + case Ydb::Type::kStructType: + if (t1.struct_type().members_size() != t2.struct_type().members_size()) { + return false; + } + for (size_t i = 0; i < (size_t)t1.struct_type().members_size(); ++i) { + auto& m1 = t1.struct_type().members(i); + auto& m2 = t2.struct_type().members(i); + if (m1.name() != m2.name()) { + return false; + } + + if (!TypesEqual(m1.type(), m2.type())) { + return false; + } + } + return true; + case Ydb::Type::kDictType: + return TypesEqual(t1.dict_type().key(), t2.dict_type().key()) + && TypesEqual(t1.dict_type().payload(), t2.dict_type().payload()); + case Ydb::Type::kVariantType: { + const auto& v1 = t1.variant_type(); + const auto& v2 = t2.variant_type(); + if (v1.type_case() != v2.type_case()) { + return false; + } + switch (v1.type_case()) { + case Ydb::VariantType::kTupleItems: + if (v1.tuple_items().elements_size() != v2.tuple_items().elements_size()) { + return false; + } + for (size_t i = 0; i < (size_t)v1.tuple_items().elements_size(); ++i) { + if (!TypesEqual(v1.tuple_items().elements(i), v2.tuple_items().elements(i))) { + return false; + } + } + return true; + case Ydb::VariantType::kStructItems: + if (v1.struct_items().members_size() != v2.struct_items().members_size()) { + return false; + } + for (size_t i = 0; i < (size_t)v1.struct_items().members_size(); ++i) { + auto& m1 = v1.struct_items().members(i); + auto& m2 = v2.struct_items().members(i); + if (m1.name() != m2.name()) { + return false; + } + + if (!TypesEqual(m1.type(), m2.type())) { + return false; + } + } + return true; + default: + ThrowFatalError(TStringBuilder() << "Unexpected Variant type case " << static_cast(t1.type_case())); + return false; + } + return false; + } + case Ydb::Type::kVoidType: + return true; + case Ydb::Type::kNullType: + return true; + case Ydb::Type::kEmptyListType: + return true; + case Ydb::Type::kEmptyDictType: + return true; + default: + ThrowFatalError(TStringBuilder() << "Unexpected type case " << static_cast(t1.type_case())); + return false; + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.h b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.h new file mode 100644 index 000000000000..e282083e71eb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/helpers.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/ya.make new file mode 100644 index 000000000000..f92e5abd156e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + helpers.cpp +) + +PEERDIR( + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/types/fatal_error_handlers +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_stats/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/CMakeLists.txt new file mode 100644 index 000000000000..498104196cd4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(client-impl-ydb_stats) + +target_link_libraries(client-impl-ydb_stats PUBLIC + yutil + grpc-client + monlib-metrics +) + +target_sources(client-impl-ydb_stats PRIVATE + stats.cpp +) + +_ydb_sdk_install_targets(TARGETS client-impl-ydb_stats) diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.cpp b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.cpp new file mode 100644 index 000000000000..4139ba9dd933 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.cpp @@ -0,0 +1,45 @@ +#include "stats.h" + +#include + +namespace NYdb::inline V3 { +namespace NSdkStats { + +using std::string; + +const NMonitoring::TLabel SESSIONS_ON_KQP_HOST_LABEL = NMonitoring::TLabel {"sensor", "SessionsByYdbHost"}; +const NMonitoring::TLabel TRANSPORT_ERRORS_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "TransportErrorsByYdbHost"}; +const NMonitoring::TLabel GRPC_INFLIGHT_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "Grpc/InFlightByYdbHost"}; + +void TStatCollector::IncSessionsOnHost(const string& host) { + if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { + ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + +void TStatCollector::DecSessionsOnHost(const string& host) { + if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { + ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Dec(); + } +} + +void TStatCollector::IncTransportErrorsByHost(const string& host) { + if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { + ptr->Rate({ DatabaseLabel_, TRANSPORT_ERRORS_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + +void TStatCollector::IncGRpcInFlightByHost(const string& host) { + if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { + ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + +void TStatCollector::DecGRpcInFlightByHost(const string& host) { + if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { + ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Dec(); + } +} + +} // namespace NSdkStats +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.h b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.h new file mode 100644 index 000000000000..9226872c9a72 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/stats.h @@ -0,0 +1,403 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3 { +namespace NSdkStats { + +// works only for case normal (foo_bar) underscore + +inline std::string UnderscoreToUpperCamel(const std::string& in) { + std::string result; + result.reserve(in.size()); + + if (in.empty()) + return {}; + + result.push_back(toupper(in[0])); + + size_t i = 1; + + while (i < in.size()) { + if (in[i] == '_') { + if (++i < in.size()) { + result.push_back(toupper(in[i++])); + } else { + break; + } + } else { + result.push_back(tolower(in[i++])); + } + } + return result; +} + +template +class TAtomicPointer { +public: + + TAtomicPointer(TPointer* pointer = nullptr) { + Set(pointer); + } + + TAtomicPointer(const TAtomicPointer& other) { + Set(other.Get()); + } + + TAtomicPointer& operator=(const TAtomicPointer& other) { + Set(other.Get()); + return *this; + } + + TPointer* Get() const { + return Pointer_.load(); + } + + void Set(TPointer* pointer) { + Pointer_.store(pointer); + } + +private: + std::atomic Pointer_; +}; + +template +class TAtomicCounter: public TAtomicPointer { + public: + void Add(ui64 value) { + if (auto counter = this->Get()) { + counter->Add(value); + } + } + + void Inc() { + if (auto counter = this->Get()) { + counter->Inc(); + } + } + + void Dec() { + if (auto counter = this->Get()) { + counter->Dec(); + } + } + + void SetValue(ui64 value) { + if (auto counter = this->Get()) { + counter->Set(value); + } + } +}; + +template +class FastLocalCounter { +public: + FastLocalCounter(TAtomicCounter& counter) + : Counter(counter), Value(0) + { } + + ~FastLocalCounter() { + Counter.Add(Value); + } + + FastLocalCounter& operator++ () { + ++Value; + return *this; + } + + TAtomicCounter& Counter; + ui64 Value; +}; + +template +class TAtomicHistogram: public TAtomicPointer { +public: + + void Record(i64 value) { + if (auto histogram = this->Get()) { + histogram->Record(value); + } + } + + bool IsCollecting() { + return this->Get() != nullptr; + } +}; + +// Sessions count for all clients +// Every client has 3 TSessionCounter for active, in session pool, in settler sessions +// TSessionCounters in different clients with same role share one sensor +class TSessionCounter: public TAtomicPointer<::NMonitoring::TIntGauge> { +public: + + // Call with mutex + void Apply(i64 newValue) { + if (auto gauge = this->Get()) { + gauge->Add(newValue - oldValue); + oldValue = newValue; + } + } + + ~TSessionCounter() { + ::NMonitoring::TIntGauge* gauge = this->Get(); + if (gauge) { + gauge->Add(-oldValue); + } + } + +private: + i64 oldValue = 0; +}; + +struct TStatCollector { + using TMetricRegistry = ::NMonitoring::TMetricRegistry; + +public: + + struct TEndpointElectorStatCollector { + + TEndpointElectorStatCollector(::NMonitoring::TIntGauge* endpointCount = nullptr + , ::NMonitoring::TIntGauge* pessimizationRatio = nullptr + , ::NMonitoring::TIntGauge* activeEndpoints = nullptr) + : EndpointCount(endpointCount) + , PessimizationRatio(pessimizationRatio) + , EndpointActive(activeEndpoints) + { } + + ::NMonitoring::TIntGauge* EndpointCount; + ::NMonitoring::TIntGauge* PessimizationRatio; + ::NMonitoring::TIntGauge* EndpointActive; + }; + + struct TSessionPoolStatCollector { + TSessionPoolStatCollector(::NMonitoring::TIntGauge* activeSessions = nullptr + , ::NMonitoring::TIntGauge* inPoolSessions = nullptr + , ::NMonitoring::TRate* fakeSessions = nullptr + , ::NMonitoring::TIntGauge* waiters = nullptr) + : ActiveSessions(activeSessions) + , InPoolSessions(inPoolSessions) + , FakeSessions(fakeSessions) + , Waiters(waiters) + { } + + ::NMonitoring::TIntGauge* ActiveSessions; + ::NMonitoring::TIntGauge* InPoolSessions; + ::NMonitoring::TRate* FakeSessions; + ::NMonitoring::TIntGauge* Waiters; + }; + + struct TClientRetryOperationStatCollector { + + TClientRetryOperationStatCollector() : MetricRegistry_(), Database_() {} + + TClientRetryOperationStatCollector(::NMonitoring::TMetricRegistry* registry, const std::string& database) + : MetricRegistry_(registry), Database_(database) + { } + + void IncSyncRetryOperation(const EStatus& status) { + if (auto registry = MetricRegistry_.Get()) { + std::string statusName = TStringBuilder() << status; + std::string sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); + registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); + } + } + + void IncAsyncRetryOperation(const EStatus& status) { + if (auto registry = MetricRegistry_.Get()) { + std::string statusName = TStringBuilder() << status; + std::string sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); + registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); + } + } + + private: + TAtomicPointer<::NMonitoring::TMetricRegistry> MetricRegistry_; + std::string Database_; + }; + + struct TClientStatCollector { + + TClientStatCollector(::NMonitoring::TRate* cacheMiss = nullptr + , ::NMonitoring::THistogram* querySize = nullptr + , ::NMonitoring::THistogram* paramsSize = nullptr + , ::NMonitoring::TRate* sessionRemoved = nullptr + , ::NMonitoring::TRate* requestMigrated = nullptr + , TClientRetryOperationStatCollector retryOperationStatCollector = TClientRetryOperationStatCollector()) + : CacheMiss(cacheMiss) + , QuerySize(querySize) + , ParamsSize(paramsSize) + , SessionRemovedDueBalancing(sessionRemoved) + , RequestMigrated(requestMigrated) + , RetryOperationStatCollector(retryOperationStatCollector) + { } + + ::NMonitoring::TRate* CacheMiss; + ::NMonitoring::THistogram* QuerySize; + ::NMonitoring::THistogram* ParamsSize; + ::NMonitoring::TRate* SessionRemovedDueBalancing; + ::NMonitoring::TRate* RequestMigrated; + TClientRetryOperationStatCollector RetryOperationStatCollector; + }; + + TStatCollector(const std::string& database, TMetricRegistry* sensorsRegistry) + : Database_(database) + , DatabaseLabel_({"database", database}) + { + if (sensorsRegistry) { + SetMetricRegistry(sensorsRegistry); + } + } + + void SetMetricRegistry(TMetricRegistry* sensorsRegistry) { + Y_ABORT_UNLESS(sensorsRegistry, "TMetricRegistry is null in stats collector."); + MetricRegistryPtr_.Set(sensorsRegistry); + DiscoveryDuePessimization_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/TooManyBadEndpoints"} })); + DiscoveryDueExpiration_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/Regular"} })); + DiscoveryFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/FailedTransportError"} })); + RequestFailDueQueueOverflow_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedDiscoveryQueueOverflow"} })); + RequestFailDueNoEndpoint_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedNoEndpoint"} })); + RequestFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedTransportError"} })); + CacheMiss_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/ClientQueryCacheMiss"} })); + ActiveSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InUse"} })); + InPoolSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InPool"} })); + Waiters_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/WaitForReturn"} })); + SessionCV_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "SessionBalancer/Variation"} })); + SessionRemovedDueBalancing_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/SessionsRemoved"} })); + RequestMigrated_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/RequestsMigrated"} })); + FakeSessions_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Sessions/SessionsLimitExceeded"} })); + GRpcInFlight_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Grpc/InFlight"} })); + + RequestLatency_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/Latency"} }, + ::NMonitoring::ExponentialHistogram(20, 2, 1))); + QuerySize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/QuerySize"} }, + ::NMonitoring::ExponentialHistogram(20, 2, 32))); + ParamsSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ParamsSize"} }, + ::NMonitoring::ExponentialHistogram(10, 2, 32))); + ResultSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ResultSize"} }, + ::NMonitoring::ExponentialHistogram(20, 2, 32))); + } + + void IncDiscoveryDuePessimization() { + DiscoveryDuePessimization_.Inc(); + } + + void IncDiscoveryDueExpiration() { + DiscoveryDueExpiration_.Inc(); + } + + void IncDiscoveryFailDueTransportError() { + DiscoveryFailDueTransportError_.Inc(); + } + + void IncReqFailQueueOverflow() { + RequestFailDueQueueOverflow_.Inc(); + } + + void IncReqFailNoEndpoint() { + RequestFailDueNoEndpoint_.Inc(); + } + + void IncReqFailDueTransportError() { + RequestFailDueTransportError_.Inc(); + } + + void IncRequestLatency(TDuration duration) { + RequestLatency_.Record(duration.MilliSeconds()); + } + + void IncResultSize(const size_t& size) { + ResultSize_.Record(size); + } + + void IncCounter(const std::string& sensor) { + if (auto registry = MetricRegistryPtr_.Get()) { + registry->Counter({ {"database", Database_}, {"sensor", sensor} })->Inc(); + } + } + + void SetSessionCV(ui32 cv) { + SessionCV_.SetValue(cv); + } + + void IncGRpcInFlight () { + GRpcInFlight_.Inc(); + } + + void DecGRpcInFlight () { + GRpcInFlight_.Dec(); + } + + TEndpointElectorStatCollector GetEndpointElectorStatCollector() { + if (auto registry = MetricRegistryPtr_.Get()) { + auto endpointCoint = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Total"} }); + auto pessimizationRatio = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/BadRatio"} }); + auto activeEndpoints = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Good"} }); + return TEndpointElectorStatCollector(endpointCoint, pessimizationRatio, activeEndpoints); + } else { + return TEndpointElectorStatCollector(); + } + } + + TSessionPoolStatCollector GetSessionPoolStatCollector() { + if (!IsCollecting()) { + return TSessionPoolStatCollector(); + } + + return TSessionPoolStatCollector(ActiveSessions_.Get(), InPoolSessions_.Get(), FakeSessions_.Get(), Waiters_.Get()); + } + + TClientStatCollector GetClientStatCollector() { + if (IsCollecting()) { + return TClientStatCollector(CacheMiss_.Get(), QuerySize_.Get(), ParamsSize_.Get(), + SessionRemovedDueBalancing_.Get(), RequestMigrated_.Get(), + TClientRetryOperationStatCollector(MetricRegistryPtr_.Get(), Database_)); + } else { + return TClientStatCollector(); + } + } + + bool IsCollecting() { + return MetricRegistryPtr_.Get() != nullptr; + } + + void IncSessionsOnHost(const std::string& host); + void DecSessionsOnHost(const std::string& host); + + void IncTransportErrorsByHost(const std::string& host); + + void IncGRpcInFlightByHost(const std::string& host); + void DecGRpcInFlightByHost(const std::string& host); +private: + const std::string Database_; + const ::NMonitoring::TLabel DatabaseLabel_; + TAtomicPointer MetricRegistryPtr_; + TAtomicCounter<::NMonitoring::TRate> DiscoveryDuePessimization_; + TAtomicCounter<::NMonitoring::TRate> DiscoveryDueExpiration_; + TAtomicCounter<::NMonitoring::TRate> RequestFailDueQueueOverflow_; + TAtomicCounter<::NMonitoring::TRate> RequestFailDueNoEndpoint_; + TAtomicCounter<::NMonitoring::TRate> RequestFailDueTransportError_; + TAtomicCounter<::NMonitoring::TRate> DiscoveryFailDueTransportError_; + TAtomicPointer<::NMonitoring::TIntGauge> ActiveSessions_; + TAtomicPointer<::NMonitoring::TIntGauge> InPoolSessions_; + TAtomicPointer<::NMonitoring::TIntGauge> Waiters_; + TAtomicCounter<::NMonitoring::TIntGauge> SessionCV_; + TAtomicCounter<::NMonitoring::TRate> SessionRemovedDueBalancing_; + TAtomicCounter<::NMonitoring::TRate> RequestMigrated_; + TAtomicCounter<::NMonitoring::TRate> FakeSessions_; + TAtomicCounter<::NMonitoring::TRate> CacheMiss_; + TAtomicCounter<::NMonitoring::TIntGauge> GRpcInFlight_; + TAtomicHistogram<::NMonitoring::THistogram> RequestLatency_; + TAtomicHistogram<::NMonitoring::THistogram> QuerySize_; + TAtomicHistogram<::NMonitoring::THistogram> ParamsSize_; + TAtomicHistogram<::NMonitoring::THistogram> ResultSize_; +}; + +} // namespace NSdkStats +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/impl/ydb_stats/ya.make b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/ya.make new file mode 100644 index 000000000000..3b9f01f6d59c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/impl/ydb_stats/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + stats.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client + library/cpp/monlib/metrics +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/import/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/import/CMakeLists.txt new file mode 100644 index 000000000000..237fe5f7f075 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/import/CMakeLists.txt @@ -0,0 +1,26 @@ +_ydb_sdk_add_library(client-ydb_import) + +target_link_libraries(client-ydb_import PUBLIC + yutil + enum_serialization_runtime + api-grpc + api-protos + client-ydb_common_client-impl + client-ydb_driver + client-ydb_proto + client-ydb_types-operation +) + +target_sources(client-ydb_import + PRIVATE + import.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_import + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/import/import.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/import/import.h +) + +_ydb_sdk_make_client_component(Import client-ydb_import) diff --git a/ydb/public/sdk/cpp/src/client/import/import.cpp b/ydb/public/sdk/cpp/src/client/import/import.cpp new file mode 100644 index 000000000000..c705ebe00afd --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/import/import.cpp @@ -0,0 +1,175 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NImport { + +using namespace NThreading; +using namespace Ydb::Import; + +/// Common +namespace { + +std::vector ItemsProgressFromProto(const google::protobuf::RepeatedPtrField& proto) { + std::vector result; + result.reserve(proto.size()); + + for (const auto& protoItem : proto) { + auto& item = result.emplace_back(); + item.PartsTotal = protoItem.parts_total(); + item.PartsCompleted = protoItem.parts_completed(); + item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); + item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); + } + + return result; +} + +} // anonymous + +/// S3 +TImportFromS3Response::TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ImportFromS3Metadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Endpoint(metadata.settings().endpoint()); + Metadata_.Settings.Scheme(TProtoAccessor::FromProto(metadata.settings().scheme())); + Metadata_.Settings.Bucket(metadata.settings().bucket()); + Metadata_.Settings.AccessKey(metadata.settings().access_key()); + Metadata_.Settings.SecretKey(metadata.settings().secret_key()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_prefix(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TImportFromS3Response::TMetadata& TImportFromS3Response::Metadata() const { + return Metadata_; +} + +/// Data +TImportDataResult::TImportDataResult(TStatus&& status) + : TStatus(std::move(status)) +{} + +//////////////////////////////////////////////////////////////////////////////// + +class TImportClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TFuture ImportFromS3(ImportFromS3Request&& request, const TImportFromS3Settings& settings) { + return RunOperation( + std::move(request), + &V1::ImportService::Stub::AsyncImportFromS3, + TRpcRequestSettings::Make(settings)); + } + + template + TAsyncImportDataResult ImportData(ImportDataRequest&& request, const TSettings& settings) { + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any*, TPlainStatus status) mutable { + TImportDataResult result(TStatus(std::move(status))); + promise.SetValue(std::move(result)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &V1::ImportService::Stub::AsyncImportData, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + template + TAsyncImportDataResult ImportData(const std::string& table, TData&& data, const TImportYdbDumpDataSettings& settings) { + auto request = MakeOperationRequest(settings); + + request.set_path(TStringType{table}); + request.set_data(TStringType{std::forward(data)}); + + for (const auto& column : settings.Columns_) { + request.mutable_ydb_dump()->add_columns(TStringType{column}); + } + + return ImportData(std::move(request), settings); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TImportClient::TImportClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +TFuture TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { + auto request = MakeOperationRequest(settings); + + request.mutable_settings()->set_endpoint(TStringType{settings.Endpoint_}); + request.mutable_settings()->set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); + request.mutable_settings()->set_bucket(TStringType{settings.Bucket_}); + request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); + request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_prefix(TStringType{item.Src}); + protoItem.set_destination_path(TStringType{item.Dst}); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.value()); + } + + if (settings.NoACL_) { + request.mutable_settings()->set_no_acl(settings.NoACL_.value()); + } + + request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); + + return Impl_->ImportFromS3(std::move(request), settings); +} + +TAsyncImportDataResult TImportClient::ImportData(const std::string& table, std::string&& data, const TImportYdbDumpDataSettings& settings) { + return Impl_->ImportData(table, std::move(data), settings); +} + +TAsyncImportDataResult TImportClient::ImportData(const std::string& table, const std::string& data, const TImportYdbDumpDataSettings& settings) { + return Impl_->ImportData(table, data, settings); +} + +} // namespace NImport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/import/out.cpp b/ydb/public/sdk/cpp/src/client/import/out.cpp new file mode 100644 index 000000000000..af523528dd8e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/import/out.cpp @@ -0,0 +1,5 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::NImport::TImportDataResult, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/import/ya.make b/ydb/public/sdk/cpp/src/client/import/ya.make new file mode 100644 index 000000000000..76d170c09b7f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/import/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + import.cpp + out.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/monitoring/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/monitoring/CMakeLists.txt new file mode 100644 index 000000000000..5c458ebabcf4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/monitoring/CMakeLists.txt @@ -0,0 +1,22 @@ +_ydb_sdk_add_library(client-ydb_monitoring) + +target_link_libraries(client-ydb_monitoring PUBLIC + yutil + enum_serialization_runtime + client-ydb_proto + impl-ydb_internal-make_request + client-ydb_common_client-impl + client-ydb_driver +) + +target_sources(client-ydb_monitoring PRIVATE + monitoring.cpp +) + +generate_enum_serilization(client-ydb_monitoring + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/monitoring/monitoring.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/monitoring/monitoring.h +) + +_ydb_sdk_make_client_component(Monitoring client-ydb_monitoring) diff --git a/ydb/public/sdk/cpp/src/client/monitoring/monitoring.cpp b/ydb/public/sdk/cpp/src/client/monitoring/monitoring.cpp new file mode 100644 index 000000000000..3f088a945ee2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/monitoring/monitoring.cpp @@ -0,0 +1,98 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NMonitoring { + +using namespace NThreading; + +class TSelfCheckResult::TImpl { +public: + TImpl(Ydb::Monitoring::SelfCheckResult&& result) + : Result(std::move(result)) + {} + Ydb::Monitoring::SelfCheckResult Result; +}; + +TSelfCheckResult::TSelfCheckResult(TStatus&& status, Ydb::Monitoring::SelfCheckResult&& result) + : TStatus(std::move(status)) + , Impl_(std::make_shared(std::move(result))) +{} + +class TMonitoringClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + {} + + TAsyncSelfCheckResult SelfCheck(const TSelfCheckSettings& settings) { + auto request = MakeOperationRequest(settings); + + if (settings.ReturnVerboseStatus_) { + request.set_return_verbose_status(settings.ReturnVerboseStatus_.value()); + } + + if (settings.MinimumStatus_) { + request.set_minimum_status((::Ydb::Monitoring::StatusFlag_Status)settings.MinimumStatus_.value()); + } + + if (settings.MaximumLevel_) { + request.set_maximum_level(settings.MaximumLevel_.value()); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Monitoring::SelfCheckResult result; + if (any) { + any->UnpackTo(&result); + } + TSelfCheckResult val( + TStatus(std::move(status)), + std::move(result)); + + promise.SetValue(std::move(val)); + }; + + using Ydb::Monitoring::SelfCheckRequest; + using Ydb::Monitoring::SelfCheckResponse; + + auto requestSettings = TRpcRequestSettings::Make(settings); + requestSettings.EndpointPolicy = TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Monitoring::V1::MonitoringService::Stub::AsyncSelfCheck, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + requestSettings); + + return promise.GetFuture(); + } +}; + +TMonitoringClient::TMonitoringClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{} + +TAsyncSelfCheckResult TMonitoringClient::SelfCheck(const TSelfCheckSettings& settings) { + return Impl_->SelfCheck(settings); +} + +} + +const Ydb::Monitoring::SelfCheckResult& TProtoAccessor::GetProto(const NYdb::NMonitoring::TSelfCheckResult& selfCheckResult) { + return selfCheckResult.Impl_->Result; +} + +} diff --git a/ydb/public/sdk/cpp/src/client/monitoring/ya.make b/ydb/public/sdk/cpp/src/client/monitoring/ya.make new file mode 100644 index 000000000000..c3e253273493 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/monitoring/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + monitoring.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/monitoring/monitoring.h) + +PEERDIR( + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver +) + +END() + diff --git a/ydb/public/sdk/cpp/src/client/operation/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/operation/CMakeLists.txt new file mode 100644 index 000000000000..d34e2e92da89 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/operation/CMakeLists.txt @@ -0,0 +1,20 @@ +_ydb_sdk_add_library(client-ydb_operation) + +target_link_libraries(client-ydb_operation PUBLIC + yutil + api-grpc + library-operation_id + client-ydb_query + client-ydb_common_client-impl + client-ydb_driver + client-ydb_export + client-ydb_import + client-ss_tasks + client-ydb_types-operation +) + +target_sources(client-ydb_operation PRIVATE + operation.cpp +) + +_ydb_sdk_make_client_component(Operation client-ydb_operation) diff --git a/ydb/public/sdk/cpp/src/client/operation/operation.cpp b/ydb/public/sdk/cpp/src/client/operation/operation.cpp new file mode 100644 index 000000000000..fc6a1bb17fc4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/operation/operation.cpp @@ -0,0 +1,210 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +/* Headers below used to instantiate concrete 'Get' & 'List' methods */ +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NOperation { + +constexpr TDuration OPERATION_CLIENT_TIMEOUT = TDuration::Seconds(5); + +using namespace NThreading; +using namespace Ydb::Operation; +using namespace Ydb::Operations; + +class TOperationClient::TImpl : public TClientImplCommon { + template + using TSimpleRpc = TGRpcConnectionsImpl::TSimpleRpc; + + template + TAsyncStatus Run(TRequest&& request, TSimpleRpc rpc) { + auto promise = NewPromise(); + + auto extractor = [promise] + (TResponse* response, TPlainStatus status) mutable { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus st(static_cast(response->status()), std::move(opIssues)); + promise.SetValue(std::move(st)); + } else { + TStatus st(std::move(status)); + promise.SetValue(std::move(st)); + } + }; + + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = OPERATION_CLIENT_TIMEOUT; + + Connections_->Run( + std::move(request), + extractor, + rpc, + DbDriverState_, + rpcSettings); + + return promise.GetFuture(); + } + +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + template + TFuture Get(GetOperationRequest&& request) { + return RunOperation( + std::move(request), + &V1::OperationService::Stub::AsyncGetOperation); + } + + TAsyncStatus Cancel(CancelOperationRequest&& request) { + return Run(std::move(request), + &V1::OperationService::Stub::AsyncCancelOperation); + } + + TAsyncStatus Forget(ForgetOperationRequest&& request) { + return Run(std::move(request), + &V1::OperationService::Stub::AsyncForgetOperation); + } + + template + TFuture> List(ListOperationsRequest&& request) { + auto promise = NewPromise>(); + + auto extractor = [promise] + (ListOperationsResponse* response, TPlainStatus status) mutable { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus st(static_cast(response->status()), std::move(opIssues)); + + std::vector operations; + operations.reserve(response->operations_size()); + for (auto& operation : *response->mutable_operations()) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(operation.issues(), opIssues); + operations.emplace_back( + TStatus(static_cast(operation.status()), std::move(opIssues)), + std::move(operation)); + } + + promise.SetValue(TOperationsList(std::move(st), std::move(operations), response->next_page_token())); + } else { + TStatus st(std::move(status)); + promise.SetValue(TOperationsList(std::move(st))); + } + }; + + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = OPERATION_CLIENT_TIMEOUT; + + Connections_->Run( + std::move(request), + extractor, + &V1::OperationService::Stub::AsyncListOperations, + DbDriverState_, + rpcSettings); + + return promise.GetFuture(); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TOperationClient::TOperationClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +template +TFuture TOperationClient::Get(const TOperation::TOperationId& id) { + auto request = MakeRequest(); + request.set_id(TStringType{id.ToString()}); + + return Impl_->Get(std::move(request)); +} + +TAsyncStatus TOperationClient::Cancel(const TOperation::TOperationId& id) { + auto request = MakeRequest(); + request.set_id(TStringType{id.ToString()}); + + return Impl_->Cancel(std::move(request)); +} + +TAsyncStatus TOperationClient::Forget(const TOperation::TOperationId& id) { + auto request = MakeRequest(); + request.set_id(TStringType{id.ToString()}); + + return Impl_->Forget(std::move(request)); +} + +template +TFuture> TOperationClient::List(const std::string& kind, ui64 pageSize, const std::string& pageToken) { + auto request = MakeRequest(); + + request.set_kind(TStringType{kind}); + if (pageSize) { + request.set_page_size(pageSize); + } + if (!pageToken.empty()) { + request.set_page_token(TStringType{pageToken}); + } + + return Impl_->List(std::move(request)); +} + +// Instantiations +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("ss/backgrounds", pageSize, pageToken); +} + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + // TODO: export -> export/yt + return List("export", pageSize, pageToken); +} + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("export/s3", pageSize, pageToken); +} + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("import/s3", pageSize, pageToken); +} + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("buildindex", pageSize, pageToken); +} + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("scriptexec", pageSize, pageToken); +} + +} // namespace NOperation +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/operation/ya.make b/ydb/public/sdk/cpp/src/client/operation/ya.make new file mode 100644 index 000000000000..d23866bbdc8e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/operation/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + operation.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/import + ydb/public/sdk/cpp/src/client/ss_tasks + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/params/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/params/CMakeLists.txt new file mode 100644 index 000000000000..798fb859743d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/params/CMakeLists.txt @@ -0,0 +1,15 @@ +_ydb_sdk_add_library(client-ydb_params) + +target_link_libraries(client-ydb_params PUBLIC + yutil + api-protos + client-ydb_types-fatal_error_handlers + client-ydb_value +) + +target_sources(client-ydb_params PRIVATE + params.cpp + impl.cpp +) + +_ydb_sdk_make_client_component(Params client-ydb_params) diff --git a/ydb/public/sdk/cpp/src/client/params/impl.cpp b/ydb/public/sdk/cpp/src/client/params/impl.cpp new file mode 100644 index 000000000000..99164d73d243 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/params/impl.cpp @@ -0,0 +1,47 @@ +#include "impl.h" + +#include +#include + + +namespace NYdb::inline V3 { + +TParams::TImpl::TImpl(::google::protobuf::Map&& paramsMap) { + ParamsMap_.swap(paramsMap); +} + +bool TParams::TImpl::Empty() const { + return ParamsMap_.empty(); +} + +std::map TParams::TImpl::GetValues() const { + std::map valuesMap; + for (auto it = ParamsMap_.begin(); it != ParamsMap_.end(); ++it) { + auto paramType = TType(it->second.type()); + auto paramValue = TValue(paramType, it->second.value()); + + valuesMap.emplace(it->first, paramValue); + } + + return valuesMap; +} + +std::optional TParams::TImpl::GetValue(const std::string& name) const { + auto it = ParamsMap_.find(name); + if (it != ParamsMap_.end()) { + auto paramType = TType(it->second.type()); + return TValue(paramType, it->second.value()); + } + + return std::optional(); +} + +::google::protobuf::Map* TParams::TImpl::GetProtoMapPtr() { + return &ParamsMap_; +} + +const ::google::protobuf::Map& TParams::TImpl::GetProtoMap() const { + return ParamsMap_; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/params/impl.h b/ydb/public/sdk/cpp/src/client/params/impl.h new file mode 100644 index 000000000000..8594f955f742 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/params/impl.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +class TParams::TImpl { +public: + TImpl(::google::protobuf::Map&& paramsMap); + + bool Empty() const; + std::map GetValues() const; + std::optional GetValue(const std::string& name) const; + ::google::protobuf::Map* GetProtoMapPtr(); + const ::google::protobuf::Map& GetProtoMap() const; + +private: + ::google::protobuf::Map ParamsMap_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/params/params.cpp b/ydb/public/sdk/cpp/src/client/params/params.cpp new file mode 100644 index 000000000000..4c077a27f171 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/params/params.cpp @@ -0,0 +1,179 @@ +#include "impl.h" + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +//////////////////////////////////////////////////////////////////////////////// + +TParams::TParams(::google::protobuf::Map&& protoMap) + : Impl_(new TImpl(std::move(protoMap))) {} + +::google::protobuf::Map* TParams::GetProtoMapPtr() { + return Impl_->GetProtoMapPtr(); +} + +const ::google::protobuf::Map& TParams::GetProtoMap() const { + return Impl_->GetProtoMap(); +} + +bool TParams::Empty() const { + return Impl_->Empty(); +} + +std::map TParams::GetValues() const { + return Impl_->GetValues(); +} + +std::optional TParams::GetValue(const std::string& name) const { + return Impl_->GetValue(name); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TParamsBuilder::TImpl { +public: + TImpl() = default; + + TImpl(const ::google::protobuf::Map& typeInfo) + : HasTypeInfo_(true) + { + for (const auto& pair : typeInfo) { + ParamsMap_[pair.first].mutable_type()->CopyFrom(pair.second); + } + } + + TImpl(const std::map& typeInfo) + : HasTypeInfo_(true) + { + for (const auto& pair : typeInfo) { + ParamsMap_[pair.first].mutable_type()->CopyFrom(pair.second.GetProto()); + } + } + + bool HasTypeInfo() const { + return HasTypeInfo_; + } + + TParamValueBuilder& AddParam(TParamsBuilder& owner, const std::string& name) { + auto param = GetParam(name); + Y_ABORT_UNLESS(param); + + auto result = ValueBuildersMap_.emplace(name, TParamValueBuilder(owner, *param->mutable_type(), + *param->mutable_value())); + + return result.first->second; + } + + void AddParam(const std::string& name, const TValue& value) { + auto param = GetParam(name); + Y_ABORT_UNLESS(param); + + if (HasTypeInfo()) { + if (!TypesEqual(param->type(), value.GetType().GetProto())) { + FatalError(TStringBuilder() << "Type mismatch for parameter: " << name << ", expected: " + << FormatType(TType(param->type())) << ", actual: " << FormatType(value.GetType())); + } + } else { + param->mutable_type()->CopyFrom(value.GetType().GetProto()); + } + + param->mutable_value()->CopyFrom(value.GetProto()); + } + + TParams Build() { + for (auto& pair : ValueBuildersMap_) { + if (!pair.second.Finished()) { + FatalError(TStringBuilder() << "Incomplete value for parameter: " << pair.first + << ", call Build() on parameter value builder"); + } + } + + ValueBuildersMap_.clear(); + + ::google::protobuf::Map paramsMap; + paramsMap.swap(ParamsMap_); + return TParams(std::move(paramsMap)); + } + +private: + Ydb::TypedValue* GetParam(const std::string& name) { + if (HasTypeInfo()) { + auto it = ParamsMap_.find(name); + if (it == ParamsMap_.end()) { + FatalError(TStringBuilder() << "Parameter not found: " << name); + return nullptr; + } + + return &it->second; + } else { + return &ParamsMap_[name]; + } + } + + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TParamsBuilder: " << msg); + } + +private: + bool HasTypeInfo_ = false; + ::google::protobuf::Map ParamsMap_; + std::map ValueBuildersMap_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TParamValueBuilder::TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto) + : TValueBuilderBase(typeProto, valueProto) + , Owner_(owner) + , Finished_(false) {} + +bool TParamValueBuilder::Finished() { + return Finished_; +} + +TParamsBuilder& TParamValueBuilder::Build() { + CheckValue(); + + Finished_ = true; + return Owner_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TParamsBuilder::TParamsBuilder(TParamsBuilder&&) = default; +TParamsBuilder::~TParamsBuilder() = default; + +TParamsBuilder::TParamsBuilder() + : Impl_(new TImpl()) {} + +TParamsBuilder::TParamsBuilder(const std::map& typeInfo) + : Impl_(new TImpl(typeInfo)) {} + +TParamsBuilder::TParamsBuilder(const ::google::protobuf::Map& typeInfo) + : Impl_(new TImpl(typeInfo)) {} + +bool TParamsBuilder::HasTypeInfo() const { + return Impl_->HasTypeInfo(); +} + +TParamValueBuilder& TParamsBuilder::AddParam(const std::string& name) { + return Impl_->AddParam(*this, name); +} + +TParamsBuilder& TParamsBuilder::AddParam(const std::string& name, const TValue& value) { + Impl_->AddParam(name, value); + return *this; +} + +TParams TParamsBuilder::Build() { + return Impl_->Build(); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/params/ya.make b/ydb/public/sdk/cpp/src/client/params/ya.make new file mode 100644 index 000000000000..0609d93b38f6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/params/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + params.cpp + impl.cpp +) + +PEERDIR( + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/types/fatal_error_handlers + ydb/public/sdk/cpp/src/client/value +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/persqueue_public/CMakeLists.txt new file mode 100644 index 000000000000..d7ad2dfca95e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/CMakeLists.txt @@ -0,0 +1,35 @@ +add_subdirectory(codecs) +add_subdirectory(impl) + +_ydb_sdk_add_library(cpp-client-ydb_persqueue_public INTERFACE) + +target_link_libraries(cpp-client-ydb_persqueue_public INTERFACE + yutil + cpp-client-ydb_persqueue_core + client-ydb_persqueue_public-impl + client-ydb_persqueue_public-codecs +) + +generate_enum_serilization(cpp-client-ydb_persqueue_public + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/control_plane.h + INCLUDE_HEADERS + src/client/persqueue_public/include/control_plane.h +) + +generate_enum_serilization(cpp-client-ydb_persqueue_public + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/read_events.h + INCLUDE_HEADERS + src/client/persqueue_public/include/read_events.h +) + +generate_enum_serilization(cpp-client-ydb_persqueue_public + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_events.h + INCLUDE_HEADERS + src/client/persqueue_public/include/write_events.h +) + +generate_enum_serilization(cpp-client-ydb_persqueue_public + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_session.h + INCLUDE_HEADERS + src/client/persqueue_public/include/write_session.h +) diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/CMakeLists.txt new file mode 100644 index 000000000000..66fbed79a088 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/CMakeLists.txt @@ -0,0 +1,5 @@ +_ydb_sdk_add_library(client-ydb_persqueue_public-codecs INTERFACE) + +target_link_libraries(client-ydb_persqueue_public-codecs INTERFACE + client-ydb_topic-codecs +) diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/codecs.h b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/codecs.h new file mode 100644 index 000000000000..bf9b2b2a8480 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/codecs.h @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/ya.make new file mode 100644 index 000000000000..05c88b8a209b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/codecs/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + codecs.h +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/topic/codecs +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/CMakeLists.txt new file mode 100644 index 000000000000..74dd435f4c14 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/CMakeLists.txt @@ -0,0 +1,26 @@ +_ydb_sdk_add_library(client-ydb_persqueue_public-impl) + +target_link_libraries(client-ydb_persqueue_public-impl PUBLIC + yutil + containers-disjoint_interval_tree + grpc-client + monlib-dynamic_counters + monlib-metrics + string_utils-url + persqueue-obfuscate + api-grpc-draft + impl-ydb_internal-make_request + client-ydb_common_client-impl + client-ydb_driver + string_utils-misc +) + +target_sources(client-ydb_persqueue_public-impl PRIVATE + common.cpp + persqueue_impl.cpp + persqueue.cpp + read_session.cpp + read_session_messages.cpp + write_session_impl.cpp + write_session.cpp +) diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/aliases.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/aliases.h new file mode 100644 index 000000000000..96de4ce1c8c1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/aliases.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include + +namespace NYdb::inline V3::NPersQueue { + +// codecs +using NTopic::ICodec; +using NTopic::TCodecMap; +using NTopic::TGzipCodec; +using NTopic::TZstdCodec; +using NTopic::TUnsupportedCodec; + +// callback_context +using NTopic::TCallbackContext; +using NTopic::TEnableSelfContext; +using NTopic::TContextOwner; + +// common +// using NTopic::GetRetryErrorClass; +using NTopic::ISessionConnectionProcessorFactory; +using NTopic::CreateConnectionProcessorFactory; +using NTopic::TBaseSessionEventsQueue; +using NTopic::TWaiter; +using NTopic::IssuesSingleLineString; +using NTopic::MakeIssueWithSubIssues; +using NTopic::ApplyClusterEndpoint; +using NTopic::Cancel; +using NTopic::IsErrorMessage; +using NTopic::MakeErrorFromProto; + +// counters_logger +using TCountersLogger = NTopic::TCountersLogger; + +// read_session_impl +using NTopic::HasNullCounters; +using NTopic::MakeCountersNotNull; +using TDeferredActions = NTopic::TDeferredActions; +using TCallbackContextPtr = NTopic::TCallbackContextPtr; +using TReadSessionEventsQueue = NTopic::TReadSessionEventsQueue; +using TPartitionStreamImpl = NTopic::TPartitionStreamImpl; +using TSingleClusterReadSessionImpl = NTopic::TSingleClusterReadSessionImpl; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.cpp new file mode 100644 index 000000000000..18df160d2e99 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.cpp @@ -0,0 +1,58 @@ +#include "common.h" + +#include + +namespace NYdb::inline V3::NPersQueue { + +ERetryErrorClass GetRetryErrorClass(EStatus status) { + switch (status) { + case EStatus::SUCCESS: // NoRetry? + case EStatus::INTERNAL_ERROR: // NoRetry? + case EStatus::ABORTED: + case EStatus::UNAVAILABLE: + case EStatus::GENERIC_ERROR: // NoRetry? + case EStatus::BAD_SESSION: // NoRetry? + case EStatus::SESSION_EXPIRED: + case EStatus::CANCELLED: + case EStatus::UNDETERMINED: + case EStatus::SESSION_BUSY: + case EStatus::CLIENT_INTERNAL_ERROR: + case EStatus::CLIENT_CANCELLED: + case EStatus::CLIENT_OUT_OF_RANGE: + return ERetryErrorClass::ShortRetry; + + case EStatus::OVERLOADED: + case EStatus::TIMEOUT: + case EStatus::TRANSPORT_UNAVAILABLE: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::CLIENT_DEADLINE_EXCEEDED: + case EStatus::CLIENT_LIMITS_REACHED: + case EStatus::CLIENT_DISCOVERY_FAILED: + return ERetryErrorClass::LongRetry; + + case EStatus::SCHEME_ERROR: + case EStatus::STATUS_UNDEFINED: + case EStatus::BAD_REQUEST: + case EStatus::UNAUTHORIZED: + case EStatus::PRECONDITION_FAILED: + case EStatus::UNSUPPORTED: + case EStatus::ALREADY_EXISTS: + case EStatus::NOT_FOUND: + case EStatus::EXTERNAL_ERROR: + case EStatus::CLIENT_UNAUTHENTICATED: + case EStatus::CLIENT_CALL_UNIMPLEMENTED: + return ERetryErrorClass::NoRetry; + } +} + +ERetryErrorClass GetRetryErrorClassV2(EStatus status) { + switch (status) { + case EStatus::SCHEME_ERROR: + return ERetryErrorClass::NoRetry; + default: + return GetRetryErrorClass(status); + + } +} + +} // namespace NYdb::NPersQueue \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.h new file mode 100644 index 000000000000..b010474d260f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/common.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + ERetryErrorClass GetRetryErrorClass(EStatus status); + ERetryErrorClass GetRetryErrorClassV2(EStatus status); +} diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue.cpp new file mode 100644 index 000000000000..4fad6d95f637 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue.cpp @@ -0,0 +1,202 @@ +#include +#include +#include + +#include + +#include +#include +#include + +namespace NYdb::inline V3::NPersQueue { + +class TCommonCodecsProvider { +public: + TCommonCodecsProvider() { + TCodecMap::GetTheCodecMap().Set((ui32)NYdb::NPersQueue::ECodec::GZIP, std::make_unique()); + TCodecMap::GetTheCodecMap().Set((ui32)NYdb::NPersQueue::ECodec::ZSTD, std::make_unique()); + } +}; +TCommonCodecsProvider COMMON_CODECS_PROVIDER; + +const std::vector& GetDefaultCodecs() { + static const std::vector codecs = {}; + return codecs; +} + +using TTopicSettingsCreate = TTopicSettings; +using TTopicSettingsAlter = TTopicSettings; + +TCredentials::TCredentials(const Ydb::PersQueue::V1::Credentials& settings) + : Credentials_(settings) +{ + switch (Credentials_.credentials_case()) { + case Ydb::PersQueue::V1::Credentials::kOauthToken: { + Mode_ = EMode::OAUTH_TOKEN; + break; + } + case Ydb::PersQueue::V1::Credentials::kJwtParams: { + Mode_ = EMode::JWT_PARAMS; + break; + } + case Ydb::PersQueue::V1::Credentials::kIam: { + Mode_ = EMode::IAM; + break; + } + case Ydb::PersQueue::V1::Credentials::CREDENTIALS_NOT_SET: { + Mode_ = EMode::NOT_SET; + break; + } + default: { + ythrow yexception() << "unsupported credentials type " << ::NPersQueue::ObfuscateString(ToString(Credentials_)); + } + } +} + +TCredentials::EMode TCredentials::GetMode() const { + return Mode_; +} + +std::string TCredentials::GetOauthToken() const { + Y_ENSURE(GetMode() == EMode::OAUTH_TOKEN); + return Credentials_.oauth_token(); +} + +std::string TCredentials::GetJwtParams() const { + Y_ENSURE(GetMode() == EMode::JWT_PARAMS); + return Credentials_.jwt_params(); +} + +std::string TCredentials::GetIamEndpoint() const { + Y_ENSURE(GetMode() == EMode::IAM); + return Credentials_.iam().endpoint(); +} + +std::string TCredentials::GetIamServiceAccountKey() const { + Y_ENSURE(GetMode() == EMode::IAM); + return Credentials_.iam().service_account_key(); +} + +TDescribeTopicResult::TDescribeTopicResult(TStatus status, const Ydb::PersQueue::V1::DescribeTopicResult& result) + : TStatus(std::move(status)) + , TopicSettings_(result.settings()) + , Proto_(result) +{ +} + +TDescribeTopicResult::TTopicSettings::TTopicSettings(const Ydb::PersQueue::V1::TopicSettings& settings) { + RetentionPeriod_ = TDuration::MilliSeconds(settings.retention_period_ms()); + SupportedFormat_ = static_cast(settings.supported_format()); + + if (settings.has_auto_partitioning_settings()) { + PartitionsCount_ = settings.auto_partitioning_settings().min_active_partitions(); + MaxPartitionsCount_ = settings.auto_partitioning_settings().max_active_partitions(); + StabilizationWindow_ = TDuration::Seconds(settings.auto_partitioning_settings().partition_write_speed().stabilization_window().seconds()); + UpUtilizationPercent_ = settings.auto_partitioning_settings().partition_write_speed().up_utilization_percent(); + DownUtilizationPercent_ = settings.auto_partitioning_settings().partition_write_speed().down_utilization_percent(); + AutoPartitioningStrategy_ = settings.auto_partitioning_settings().strategy(); + } else { + PartitionsCount_ = settings.partitions_count(); + } + + for (const auto& codec : settings.supported_codecs()) { + SupportedCodecs_.push_back(static_cast(codec)); + } + MaxPartitionStorageSize_ = settings.max_partition_storage_size(); + MaxPartitionWriteSpeed_ = settings.max_partition_write_speed(); + MaxPartitionWriteBurst_ = settings.max_partition_write_burst(); + ClientWriteDisabled_ = settings.client_write_disabled(); + AllowUnauthenticatedRead_ = AllowUnauthenticatedWrite_ = false; + + for (auto& pair : settings.attributes()) { + if (pair.first == "_partitions_per_tablet") { + PartitionsPerTablet_ = FromString(pair.second); + } else if (pair.first == "_allow_unauthenticated_read") { + AllowUnauthenticatedRead_ = FromString(pair.second); + } else if (pair.first == "_allow_unauthenticated_write") { + AllowUnauthenticatedWrite_ = FromString(pair.second); + } else if (pair.first == "_abc_id") { + AbcId_ = FromString(pair.second); + } else if (pair.first == "_abc_slug") { + AbcSlug_ = pair.second; + } else if (pair.first == "_federation_account") { + FederationAccount_ = pair.second; + } + } + for (const auto& readRule : settings.read_rules()) { + ReadRules_.emplace_back(readRule); + } + if (settings.has_remote_mirror_rule()) { + RemoteMirrorRule_ = settings.remote_mirror_rule(); + } +} + + +TDescribeTopicResult::TTopicSettings::TReadRule::TReadRule(const Ydb::PersQueue::V1::TopicSettings::ReadRule& settings) { + + ConsumerName_ = settings.consumer_name(); + Important_ = settings.important(); + StartingMessageTimestamp_ = TInstant::MilliSeconds(settings.starting_message_timestamp_ms()); + + SupportedFormat_ = static_cast(settings.supported_format()); + for (const auto& codec : settings.supported_codecs()) { + SupportedCodecs_.push_back(static_cast(codec)); + } + Version_ = settings.version(); + ServiceType_ = settings.service_type(); +} + +TDescribeTopicResult::TTopicSettings::TRemoteMirrorRule::TRemoteMirrorRule(const Ydb::PersQueue::V1::TopicSettings::RemoteMirrorRule& settings) + : Credentials_(settings.credentials()) +{ + Endpoint_ = settings.endpoint(); + TopicPath_ = settings.topic_path(); + ConsumerName_ = settings.consumer_name(); + StartingMessageTimestamp_ = TInstant::MilliSeconds(settings.starting_message_timestamp_ms()); + Database_ = settings.database(); +} + +TPersQueueClient::TPersQueueClient(const TDriver& driver, const TPersQueueClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncStatus TPersQueueClient::CreateTopic(const std::string& path, const TCreateTopicSettings& settings) { + return Impl_->CreateTopic(path, settings); +} + +TAsyncStatus TPersQueueClient::AlterTopic(const std::string& path, const TAlterTopicSettings& settings) { + return Impl_->AlterTopic(path, settings); +} + +TAsyncStatus TPersQueueClient::DropTopic(const std::string& path, const TDropTopicSettings& settings) { + return Impl_->DropTopic(path, settings); +} + +TAsyncStatus TPersQueueClient::AddReadRule(const std::string& path, const TAddReadRuleSettings& settings) { + return Impl_->AddReadRule(path, settings); +} + +TAsyncStatus TPersQueueClient::RemoveReadRule(const std::string& path, const TRemoveReadRuleSettings& settings) { + return Impl_->RemoveReadRule(path, settings); +} + +TAsyncDescribeTopicResult TPersQueueClient::DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings) { + return Impl_->DescribeTopic(path, settings); +} + +std::shared_ptr TPersQueueClient::CreateReadSession(const TReadSessionSettings& settings) { + return Impl_->CreateReadSession(settings); +} + +std::shared_ptr TPersQueueClient::CreateWriteSession(const TWriteSessionSettings& settings) { + return Impl_->CreateWriteSession(settings); +} + +std::shared_ptr TPersQueueClient::CreateSimpleBlockingWriteSession( + const TWriteSessionSettings& settings +) { + return Impl_->CreateSimpleWriteSession(settings); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.cpp new file mode 100644 index 000000000000..0ce4ff9efa49 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.cpp @@ -0,0 +1,88 @@ +#include "persqueue_impl.h" + +#include "read_session.h" +#include "write_session.h" + +namespace NYdb::inline V3::NPersQueue { + +std::shared_ptr TPersQueueClient::TImpl::CreateReadSession(const TReadSessionSettings& settings) { + std::optional maybeSettings; + if (!settings.DecompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_) { + maybeSettings = settings; + std::lock_guard guard(Lock); + if (!settings.DecompressionExecutor_) { + maybeSettings->DecompressionExecutor(Settings.DefaultCompressionExecutor_); + } + if (!settings.EventHandlers_.HandlersExecutor_) { + maybeSettings->EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + } + } + auto session = std::make_shared(maybeSettings.value_or(settings), shared_from_this(), Connections_, DbDriverState_); + session->Start(); + return std::move(session); +} + +std::shared_ptr TPersQueueClient::TImpl::CreateWriteSession( + const TWriteSessionSettings& settings +) { + std::optional maybeSettings; + if (!settings.CompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_) { + maybeSettings = settings; + std::lock_guard guard(Lock); + if (!settings.CompressionExecutor_) { + maybeSettings->CompressionExecutor(Settings.DefaultCompressionExecutor_); + } + if (!settings.EventHandlers_.HandlersExecutor_) { + maybeSettings->EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + } + } + auto session = std::make_shared( + maybeSettings.value_or(settings), shared_from_this(), Connections_, DbDriverState_ + ); + session->Start(TDuration::Zero()); + return std::move(session); +} + +std::shared_ptr TPersQueueClient::TImpl::CreateSimpleWriteSession( + const TWriteSessionSettings& settings +) { + auto alteredSettings = settings; + { + std::lock_guard guard(Lock); + alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + if (!settings.CompressionExecutor_) { + alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_); + } + } + + auto session = std::make_shared( + alteredSettings, shared_from_this(), Connections_, DbDriverState_ + ); + return std::move(session); +} + +std::shared_ptr TPersQueueClient::TImpl::GetClientForEndpoint(const std::string& clusterEndoint) { + std::lock_guard guard(Lock); + Y_ABORT_UNLESS(CustomEndpoint.empty()); + std::shared_ptr& client = Subclients[clusterEndoint]; + if (!client) { + client = std::make_shared(clusterEndoint, Connections_, Settings); + } + return client; +} + +std::shared_ptr TPersQueueClient::TImpl::CreateReadSessionConnectionProcessorFactory() { + using TService = Ydb::PersQueue::V1::PersQueueService; + using TRequest = Ydb::PersQueue::V1::MigrationStreamingReadClientMessage; + using TResponse = Ydb::PersQueue::V1::MigrationStreamingReadServerMessage; + return CreateConnectionProcessorFactory(&TService::Stub::AsyncMigrationStreamingRead, Connections_, DbDriverState_); +} + +std::shared_ptr TPersQueueClient::TImpl::CreateWriteSessionConnectionProcessorFactory() { + using TService = Ydb::PersQueue::V1::PersQueueService; + using TRequest = Ydb::PersQueue::V1::StreamingWriteClientMessage; + using TResponse = Ydb::PersQueue::V1::StreamingWriteServerMessage; + return CreateConnectionProcessorFactory(&TService::Stub::AsyncStreamingWrite, Connections_, DbDriverState_); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.h new file mode 100644 index 000000000000..084978bc1b35 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/persqueue_impl.h @@ -0,0 +1,275 @@ +#pragma once + +#include +#include + +#include +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +namespace NYdb::inline V3::NPersQueue { + +class TPersQueueClient::TImpl : public TClientImplCommon { +public: + // Constructor for main client. + TImpl(std::shared_ptr connections, const TPersQueueClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + , Settings(settings) + { + } + + // Constructor for subclients with endpoints discovered by cluster discovery. + // Async discovery mode is used because this client is created inside SDK threads. + // See YDB-1231 and YDB-1232. + TImpl(const std::string& clusterEndpoint, std::shared_ptr connections, const TPersQueueClientSettings& settings) + : TClientImplCommon(std::move(connections), settings.Database_, clusterEndpoint, EDiscoveryMode::Async, settings.SslCredentials_, settings.CredentialsProviderFactory_) + , Settings(settings) + , CustomEndpoint(clusterEndpoint) + { + } + + template + static void ConvertToProtoReadRule(const TReadRule& readRule, Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps) { + rrProps.set_consumer_name(TStringType{readRule.ConsumerName_}); + rrProps.set_important(readRule.Important_); + rrProps.set_starting_message_timestamp_ms(readRule.StartingMessageTimestamp_.MilliSeconds()); + rrProps.set_version(readRule.Version_); + rrProps.set_supported_format(static_cast(readRule.SupportedFormat_)); + for (const auto& codec : readRule.SupportedCodecs_) { + rrProps.add_supported_codecs((static_cast(codec))); + } + rrProps.set_service_type(TStringType{readRule.ServiceType_}); + } + + template + static TRequest MakePropsCreateOrAlterRequest(const std::string& path, const TSettings& settings) { + TRequest request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + Ydb::PersQueue::V1::TopicSettings& props = *request.mutable_settings(); + + props.set_partitions_count(settings.PartitionsCount_); + + bool autoPartitioningSettingsDefined = false; + if (settings.MaxPartitionsCount_.has_value()) { + props.mutable_auto_partitioning_settings()->set_max_active_partitions(*settings.MaxPartitionsCount_); + autoPartitioningSettingsDefined = true; + } + if (settings.AutoPartitioningStrategy_.has_value()) { + props.mutable_auto_partitioning_settings()->set_strategy(*settings.AutoPartitioningStrategy_); + autoPartitioningSettingsDefined = true; + } + if (settings.DownUtilizationPercent_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_down_utilization_percent(*settings.DownUtilizationPercent_); + autoPartitioningSettingsDefined = true; + } + if (settings.UpUtilizationPercent_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_up_utilization_percent(*settings.UpUtilizationPercent_); + autoPartitioningSettingsDefined = true; + } + if (settings.StabilizationWindow_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->mutable_stabilization_window()->set_seconds((*settings.StabilizationWindow_).Seconds()); + autoPartitioningSettingsDefined = true; + } + + if (autoPartitioningSettingsDefined) { + props.mutable_auto_partitioning_settings()->set_min_active_partitions(settings.PartitionsCount_); + } + + props.set_retention_period_ms(settings.RetentionPeriod_.MilliSeconds()); + props.set_supported_format(static_cast(settings.SupportedFormat_)); + for (const auto& codec : settings.SupportedCodecs_) { + props.add_supported_codecs((static_cast(codec))); + } + props.set_max_partition_storage_size(settings.MaxPartitionStorageSize_); + props.set_max_partition_write_speed(settings.MaxPartitionWriteSpeed_); + props.set_max_partition_write_burst(settings.MaxPartitionWriteBurst_); + props.set_client_write_disabled(settings.ClientWriteDisabled_); + (*props.mutable_attributes())["_allow_unauthenticated_read"] = settings.AllowUnauthenticatedRead_ ? "true" : "false"; + (*props.mutable_attributes())["_allow_unauthenticated_write"] = settings.AllowUnauthenticatedWrite_ ? "true" : "false"; + if (settings.PartitionsPerTablet_) (*props.mutable_attributes())["_partitions_per_tablet"] = ToString(*settings.PartitionsPerTablet_); + if (settings.AbcId_) (*props.mutable_attributes())["_abc_id"] = ToString(*settings.AbcId_); + if (settings.AbcSlug_) (*props.mutable_attributes())["_abc_slug"] = *settings.AbcSlug_; + if (settings.FederationAccount_) (*props.mutable_attributes())["_federation_account"] = ToString(*settings.FederationAccount_); + + for (const auto& readRule : settings.ReadRules_) { + Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps = *props.add_read_rules(); + ConvertToProtoReadRule(readRule, rrProps); + } + + if (settings.RemoteMirrorRule_) { + auto rmr = props.mutable_remote_mirror_rule(); + rmr->set_endpoint(TStringType{settings.RemoteMirrorRule_.value().Endpoint_}); + rmr->set_topic_path(TStringType{settings.RemoteMirrorRule_.value().TopicPath_}); + rmr->set_consumer_name(TStringType{settings.RemoteMirrorRule_.value().ConsumerName_}); + rmr->set_starting_message_timestamp_ms(settings.RemoteMirrorRule_.value().StartingMessageTimestamp_.MilliSeconds()); + const auto& credentials = settings.RemoteMirrorRule_.value().Credentials_; + switch (credentials.GetMode()) { + case TCredentials::EMode::OAUTH_TOKEN: { + rmr->mutable_credentials()->set_oauth_token(credentials.GetOauthToken()); + break; + } + case TCredentials::EMode::JWT_PARAMS: { + rmr->mutable_credentials()->set_jwt_params(credentials.GetJwtParams()); + break; + } + case TCredentials::EMode::IAM: { + rmr->mutable_credentials()->mutable_iam()->set_endpoint(credentials.GetIamEndpoint()); + rmr->mutable_credentials()->mutable_iam()->set_service_account_key(credentials.GetIamServiceAccountKey()); + break; + } + case TCredentials::EMode::NOT_SET: { + break; + } + default: { + ythrow yexception() << "unsupported credentials type for remote mirror rule"; + } + } + rmr->set_database(TStringType{settings.RemoteMirrorRule_.value().Database_}); + } + return request; + } + + void ProvideCodec(ECodec codecId, std::unique_ptr&& codecImpl) { + with_lock(Lock) { + if (ProvidedCodecs->contains(codecId)) { + throw yexception() << "codec with id " << ui32(codecId) << " already provided"; + } + (*ProvidedCodecs)[codecId] = std::move(codecImpl); + } + } + + void OverrideCodec(ECodec codecId, std::unique_ptr&& codecImpl) { + with_lock(Lock) { + (*ProvidedCodecs)[codecId] = std::move(codecImpl); + } + } + + const ICodec* GetCodecImplOrThrow(ECodec codecId) const { + with_lock(Lock) { + if (!ProvidedCodecs->contains(codecId)) { + throw yexception() << "codec with id " << ui32(codecId) << " not provided"; + } + return ProvidedCodecs->at(codecId).get(); + } + } + + std::shared_ptr>> GetProvidedCodecs() const { + return ProvidedCodecs; + } + + TAsyncStatus CreateTopic(const std::string& path, const TCreateTopicSettings& settings) { + auto request = MakePropsCreateOrAlterRequest(path, + settings.PartitionsPerTablet_ ? settings : TCreateTopicSettings(settings).PartitionsPerTablet(2)); + + return RunSimple( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncCreateTopic, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AlterTopic(const std::string& path, const TAlterTopicSettings& settings) { + auto request = MakePropsCreateOrAlterRequest(path, settings); + + return RunSimple( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncAlterTopic, + TRpcRequestSettings::Make(settings)); + } + + + TAsyncStatus DropTopic(const std::string& path, const TDropTopicSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + return RunSimple( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncDropTopic, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AddReadRule(const std::string& path, const TAddReadRuleSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + ConvertToProtoReadRule(settings.ReadRule_, *request.mutable_read_rule()); + return RunSimple( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncAddReadRule, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveReadRule(const std::string& path, const TRemoveReadRuleSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + request.set_consumer_name(TStringType{settings.ConsumerName_}); + return RunSimple( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncRemoveReadRule, + TRpcRequestSettings::Make(settings)); + } + + + TAsyncDescribeTopicResult DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::PersQueue::V1::DescribeTopicResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeTopicResult val(TStatus(std::move(status)), result); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncDescribeTopic, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + // Runtime API. + std::shared_ptr CreateReadSession(const TReadSessionSettings& settings); + std::shared_ptr CreateSimpleWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TWriteSessionSettings& settings); + + std::shared_ptr GetClientForEndpoint(const std::string& clusterEndoint); + + using IReadSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory; + + std::shared_ptr CreateReadSessionConnectionProcessorFactory(); + + using IWriteSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory< + Ydb::PersQueue::V1::StreamingWriteClientMessage, + Ydb::PersQueue::V1::StreamingWriteServerMessage>; + + std::shared_ptr CreateWriteSessionConnectionProcessorFactory(); + + NYdbGrpc::IQueueClientContextPtr CreateContext() { + return Connections_->CreateContext(); + } + +private: + const TPersQueueClientSettings Settings; + const std::string CustomEndpoint; + TAdaptiveLock Lock; + std::shared_ptr>> ProvidedCodecs = std::make_shared>>(); + std::unordered_map> Subclients; // Endpoint -> Subclient. +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.cpp new file mode 100644 index 000000000000..3e0e9dda3a91 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.cpp @@ -0,0 +1,929 @@ +#include "read_session.h" + +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +static const std::string DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; + +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index) { + if (dataReceivedEvent.IsCompressedMessages()) { + const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; + return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1}; + } + const auto& msg = dataReceivedEvent.GetMessages()[index]; + return {msg.GetOffset(), msg.GetOffset() + 1}; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TReadSession + +TStringBuilder TReadSession::GetLogPrefix() const { + return TStringBuilder() << GetDatabaseLogPrefix(DbDriverState->Database) << "[" << SessionId << "] "; +} + +TReadSession::TReadSession(const TReadSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : Settings(settings) + , SessionId(CreateGuidAsString()) + , Log(settings.Log_.value_or(dbDriverState->Log)) + , Client(std::move(client)) + , Connections(std::move(connections)) + , DbDriverState(std::move(dbDriverState)) +{ + if (!Settings.RetryPolicy_) { + Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); + } + + MakeCountersIfNeeded(); +} + +TReadSession::~TReadSession() { + Close(TDuration::Zero()); + + { + TDeferredActions deferred; + NYdb::NIssue::TIssues issues; + issues.AddIssue("Aborted"); + EventsQueue->Close(TSessionClosedEvent(EStatus::ABORTED, std::move(issues)), deferred); + } + + Abort(); + ClearAllEvents(); + + for (const auto& ctx : CbContexts) { + ctx->Cancel(); + } + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + } +} + +Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TReadSession::MakeClusterDiscoveryRequest() const { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; + for (const TTopicReadSettings& topic : Settings.Topics_) { + auto* params = req.add_read_sessions(); + params->set_topic(TStringType{topic.Path_}); + params->mutable_all_original(); // set all_original + } + return req; +} + +void TReadSession::Start() { + EventsQueue = std::make_shared(Settings); + + if (!ValidateSettings()) { + return; + } + + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Starting read session"); + if (Settings.DisableClusterDiscovery_) { + ProceedWithoutClusterDiscovery(); + } else { + StartClusterDiscovery(); + } +} + +bool TReadSession::ValidateSettings() { + NYdb::NIssue::TIssues issues; + if (Settings.Topics_.empty()) { + issues.AddIssue("Empty topics list."); + } + + if (Settings.ConsumerName_.empty()) { + issues.AddIssue("No consumer specified."); + } + + if (Settings.MaxMemoryUsageBytes_ < 1_MB) { + issues.AddIssue("Too small max memory usage. Valid values start from 1 megabyte."); + } + + if (issues) { + { + TDeferredActions deferred; + EventsQueue->Close(TSessionClosedEvent(EStatus::BAD_REQUEST, MakeIssueWithSubIssues("Invalid read session settings", issues)), deferred); + } + Abort(); + return false; + } else { + return true; + } +} + +void TReadSession::StartClusterDiscovery() { + { + std::lock_guard guard(Lock); + if (Aborting) { + return; + } + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Starting cluster discovery"); + ClusterDiscoveryDelayContext = nullptr; + } + + auto extractor = [self = weak_from_this()] + (google::protobuf::Any* any, TPlainStatus status) mutable { + auto selfShared = self.lock(); + if (!selfShared) { + return; + } + + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; + if (any) { + any->UnpackTo(&result); + } + TStatus st(std::move(status)); + selfShared->OnClusterDiscovery(st, result); + }; + + auto rpcSettings = TRpcRequestSettings::Make(Settings); + rpcSettings.ClientTimeout = TDuration::Seconds(5); // TODO: make client timeout setting + Connections->RunDeferred( + MakeClusterDiscoveryRequest(), + std::move(extractor), + &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, + DbDriverState, + INITIAL_DEFERRED_CALL_DELAY, + rpcSettings); // TODO: make client timeout setting +} + + +void TReadSession::ProceedWithoutClusterDiscovery() { + TDeferredActions deferred; + with_lock(Lock) { + if (Aborting) { + return; + } + + std::string normalizedName = "null"; + auto clusterSessionInfoIter = ClusterSessions.emplace(normalizedName, normalizedName).first; + TClusterSessionInfo& clusterSessionInfo = clusterSessionInfoIter->second; + clusterSessionInfo.ClusterEndpoint = DbDriverState->DiscoveryEndpoint; + clusterSessionInfo.Topics = Settings.Topics_; + CreateClusterSessionsImpl(deferred); + } + SetupCountersLogger(); +} + +void TReadSession::CreateClusterSessionsImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + // Create cluster sessions. + ui64 partitionStreamIdStart = 1; + const size_t clusterSessionsCount = ClusterSessions.size(); + for (auto& [clusterName, clusterSessionInfo] : ClusterSessions) { + TReadSessionSettings sessionSettings = Settings; + sessionSettings.Topics_ = clusterSessionInfo.Topics; + if (sessionSettings.MaxMemoryUsageBytes_ > clusterSessionsCount && sessionSettings.MaxMemoryUsageBytes_ != std::numeric_limits::max()) { + sessionSettings.MaxMemoryUsageBytes_ /= clusterSessionsCount; + } + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Starting session to cluster " << clusterName + << " (" << clusterSessionInfo.ClusterEndpoint << ")" + ); + auto subclient = Client->GetClientForEndpoint(clusterSessionInfo.ClusterEndpoint); + auto context = subclient->CreateContext(); + if (!context) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return; + } + CbContexts.push_back(MakeWithCallbackContext( + sessionSettings, + DbDriverState->Database, + SessionId, + clusterName, + Log, + subclient->CreateReadSessionConnectionProcessorFactory(), + EventsQueue, + context, + partitionStreamIdStart++, + clusterSessionsCount + )); + + clusterSessionInfo.Session = CbContexts.back()->TryGet(); + deferred.DeferStartSession(CbContexts.back()); + } +} + +void TReadSession::OnClusterDiscovery(const TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result) { + TDeferredActions deferred; + with_lock(Lock) { + if (Aborting) { + return; + } + + if (!status.IsSuccess()) { + ++*Settings.Counters_->Errors; + if (!ClusterDiscoveryRetryState) { + ClusterDiscoveryRetryState = Settings.RetryPolicy_->CreateRetryState(); + } + auto retryDelay = ClusterDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); + if (retryDelay) { + LOG_LAZY(Log, + TLOG_INFO, + GetLogPrefix() << "Cluster discovery request failed. Status: " << status.GetStatus() + << ". Issues: \"" << IssuesSingleLineString(status.GetIssues()) << "\"" + ); + RestartClusterDiscoveryImpl(*retryDelay, deferred); + } else { + AbortImpl(status.GetStatus(), MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues()), deferred); + } + return; + } + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Cluster discovery request succeeded"); + ClusterDiscoveryRetryState = nullptr; + + // Init ClusterSessions. + if (static_cast(result.read_sessions_clusters_size()) != Settings.Topics_.size()) { + ++*Settings.Counters_->Errors; + AbortImpl(EStatus::INTERNAL_ERROR, TStringBuilder() << "Unexpected reply from cluster discovery. Sizes of topics arrays don't match: " + << result.read_sessions_clusters_size() << " vs " << Settings.Topics_.size(), deferred); + return; + } + + const bool explicitlySpecifiedClusters = !Settings.Clusters_.empty(); + if (explicitlySpecifiedClusters) { + for (const std::string& cluster : Settings.Clusters_) { + std::string normalizedName = cluster; + NUtils::ToLower(normalizedName); + ClusterSessions.emplace(normalizedName, normalizedName); + } + } + + NYdb::NIssue::TIssues issues; + EStatus errorStatus = EStatus::INTERNAL_ERROR; + for (size_t topicIndex = 0; topicIndex < Settings.Topics_.size(); ++topicIndex) { + const TTopicReadSettings& topicSettings = Settings.Topics_[topicIndex]; + const Ydb::PersQueue::ClusterDiscovery::ReadSessionClusters& readSessionClusters = result.read_sessions_clusters(topicIndex); + for (const Ydb::PersQueue::ClusterDiscovery::ClusterInfo& cluster : readSessionClusters.clusters()) { + std::string normalizedName = cluster.name(); + NUtils::ToLower(normalizedName); + decltype(ClusterSessions)::iterator clusterSessionInfoIter; + if (explicitlySpecifiedClusters) { + clusterSessionInfoIter = ClusterSessions.find(normalizedName); + if (clusterSessionInfoIter == ClusterSessions.end()) { // User hasn't specified this cluster, so it isn't in our interest. + continue; + } + } else { + clusterSessionInfoIter = ClusterSessions.emplace(normalizedName, normalizedName).first; + } + TClusterSessionInfo& clusterSessionInfo = clusterSessionInfoIter->second; + if (cluster.endpoint().empty()) { + issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster " + << normalizedName); + } + auto fullEndpoint = ApplyClusterEndpoint(DbDriverState->DiscoveryEndpoint, cluster.endpoint()); + if (!clusterSessionInfo.ClusterEndpoint.empty() && clusterSessionInfo.ClusterEndpoint != fullEndpoint) { + issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Different endpoints for one cluster name. Cluster: " + << normalizedName << ". \"" << clusterSessionInfo.ClusterEndpoint << "\" vs \"" + << fullEndpoint << "\""); + } + if (clusterSessionInfo.ClusterEndpoint.empty()) { + clusterSessionInfo.ClusterEndpoint = fullEndpoint; + } + clusterSessionInfo.Topics.reserve(Settings.Topics_.size()); + clusterSessionInfo.Topics.push_back(topicSettings); + } + } + + // Check clusters. + for (const auto& [cluster, clusterInfo] : ClusterSessions) { + if (clusterInfo.Topics.empty()) { // If specified explicitly by user. + errorStatus = EStatus::BAD_REQUEST; + issues.AddIssue(TStringBuilder() << "Unsupported cluster: " << cluster); + } + } + + if (issues) { + ++*Settings.Counters_->Errors; + AbortImpl(errorStatus, std::move(issues), deferred); + return; + } + + CreateClusterSessionsImpl(deferred); + } + SetupCountersLogger(); +} + +void TReadSession::RestartClusterDiscoveryImpl(TDuration delay, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (Aborting || Closing) { + return; + } + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Restart cluster discovery in " << delay); + auto startCallback = [self = weak_from_this()](bool ok) { + if (ok) { + if (auto sharedSelf = self.lock()) { + sharedSelf->StartClusterDiscovery(); + } + } + }; + + ClusterDiscoveryDelayContext = Connections->CreateContext(); + if (!ClusterDiscoveryDelayContext) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return; + } + Connections->ScheduleCallback(delay, + std::move(startCallback), + ClusterDiscoveryDelayContext); +} + +bool TReadSession::Close(TDuration timeout) { + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Closing read session. Close timeout: " << timeout); + + + // the program may not have reached SetupCountersLogger + if (CountersLogger) { + // Log final counters. + CountersLogger->Stop(); + } + + std::vector sessions; + NThreading::TPromise promise = NThreading::NewPromise(); + std::shared_ptr> count = std::make_shared>(0); + auto callback = [=]() mutable { + if (--*count == 0) { + promise.TrySetValue(true); + } + }; + + TDeferredActions deferred; + { + std::lock_guard guard(Lock); + if (Closing || Aborting) { + return false; + } + + if (!timeout) { + AbortImpl(EStatus::ABORTED, "Close with zero timeout", deferred); + return false; + } + + Closing = true; + sessions.reserve(ClusterSessions.size()); + for (auto& [cluster, sessionInfo] : ClusterSessions) { + if (sessionInfo.Session) { + sessions.emplace_back(sessionInfo.Session); + } + } + } + *count = sessions.size() + 1; + for (const auto& session : sessions) { + session->Close(callback); + } + + callback(); // For the case when there are no subsessions yet. + + auto timeoutCallback = [=](bool) mutable { + promise.TrySetValue(false); + }; + + auto timeoutContext = Connections->CreateContext(); + if (!timeoutContext) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return false; + } + Connections->ScheduleCallback(timeout, + std::move(timeoutCallback), + timeoutContext); + + // Wait. + NThreading::TFuture resultFuture = promise.GetFuture(); + const bool result = resultFuture.GetValueSync(); + if (result) { + Cancel(timeoutContext); + + NYdb::NIssue::TIssues issues; + issues.AddIssue("Session was gracefully closed"); + EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); + } else { + ++*Settings.Counters_->Errors; + for (const auto& session : sessions) { + session->Abort(); + } + + NYdb::NIssue::TIssues issues; + issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); + EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); + } + + std::lock_guard guard(Lock); + Aborting = true; // Set abort flag for doing nothing on destructor. + return result; +} + +void TReadSession::AbortImpl(TDeferredActions&) { + + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Aborting) { + Y_ABORT_UNLESS(EventsQueue->IsClosed()); + Aborting = true; + if (ClusterDiscoveryDelayContext) { + ClusterDiscoveryDelayContext->Cancel(); + ClusterDiscoveryDelayContext.reset(); + } + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + DumpCountersContext.reset(); + } + for (auto& [cluster, sessionInfo] : ClusterSessions) { + if (sessionInfo.Session) { + sessionInfo.Session->Abort(); + } + } + } +} + +void TReadSession::AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + auto closeEvent = TSessionClosedEvent(statusCode, std::move(issues)); + LOG_LAZY(Log, TLOG_NOTICE, GetLogPrefix() << "Aborting read session. Description: " << closeEvent.DebugString()); + + EventsQueue->Close(std::move(closeEvent), deferred); + AbortImpl(deferred); +} + +void TReadSession::AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + + AbortImpl(statusCode, std::move(issues), deferred); +} + +void TReadSession::Abort() { + TDeferredActions deferred; + std::lock_guard guard(Lock); + AbortImpl(EStatus::ABORTED, "Aborted", deferred); +} + +void TReadSession::ClearAllEvents() { + EventsQueue->ClearAllEvents(); +} + +NThreading::TFuture TReadSession::WaitEvent() { + return EventsQueue->WaitEvent(); +} + +std::vector TReadSession::GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) { + auto res = EventsQueue->GetEvents(block, maxEventsCount, maxByteSize); + if (EventsQueue->IsClosed()) { + Abort(); + } + return res; +} + +std::optional TReadSession::GetEvent(bool block, size_t maxByteSize) { + auto res = EventsQueue->GetEvent(block, maxByteSize); + if (EventsQueue->IsClosed()) { + Abort(); + } + return res; +} + +void TReadSession::StopReadingData() { + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Stop reading data"); + std::lock_guard guard(Lock); + if (!DataReadingSuspended) { + DataReadingSuspended = true; + + for (auto& [cluster, sessionInfo] : ClusterSessions) { + if (sessionInfo.Session) { + sessionInfo.Session->StopReadingData(); + } + } + } +} + +void TReadSession::ResumeReadingData() { + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Resume reading data"); + std::lock_guard guard(Lock); + if (DataReadingSuspended) { + DataReadingSuspended = false; + + for (auto& [cluster, sessionInfo] : ClusterSessions) { + if (sessionInfo.Session) { + sessionInfo.Session->ResumeReadingData(); + } + } + } +} + +void TReadSession::MakeCountersIfNeeded() { + if (!Settings.Counters_ || HasNullCounters(*Settings.Counters_)) { + TReaderCounters::TPtr counters = MakeIntrusive(); + if (Settings.Counters_) { + *counters = *Settings.Counters_; // Copy all counters that have been set by user. + } + MakeCountersNotNull(*counters); + Settings.Counters(counters); + } +} + +void TReadSession::SetupCountersLogger() { + std::lock_guard guard(Lock); + CountersLogger = std::make_shared(Connections, CbContexts, Settings.Counters_, Log, + GetLogPrefix(), StartSessionTime); + DumpCountersContext = CountersLogger->MakeCallbackContext(); + CountersLogger->Start(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NPersQueue::TReadSessionEvent + +TReadSessionEvent::TCreatePartitionStreamEvent::TCreatePartitionStreamEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset, ui64 endOffset) + : PartitionStream(std::move(partitionStream)) + , CommittedOffset(committedOffset) + , EndOffset(endOffset) +{ +} + +void TReadSessionEvent::TCreatePartitionStreamEvent::Confirm(std::optional readOffset, std::optional commitOffset) { + if (PartitionStream) { + static_cast(PartitionStream.Get())->ConfirmCreate(readOffset, commitOffset); + } +} + +TReadSessionEvent::TDestroyPartitionStreamEvent::TDestroyPartitionStreamEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset) + : PartitionStream(std::move(partitionStream)) + , CommittedOffset(committedOffset) +{ +} + +void TReadSessionEvent::TDestroyPartitionStreamEvent::Confirm() { + if (PartitionStream) { + static_cast(PartitionStream.Get())->ConfirmDestroy(); + } +} + +TReadSessionEvent::TPartitionStreamClosedEvent::TPartitionStreamClosedEvent(TPartitionStream::TPtr partitionStream, EReason reason) + : PartitionStream(std::move(partitionStream)) + , Reason(reason) +{ +} + +TReadSessionEvent::TDataReceivedEvent::TDataReceivedEvent(std::vector messages, + std::vector compressedMessages, + TPartitionStream::TPtr partitionStream) + : Messages(std::move(messages)) + , CompressedMessages(std::move(compressedMessages)) + , PartitionStream(std::move(partitionStream)) +{ + for (size_t i = 0; i < GetMessagesCount(); ++i) { + auto [from, to] = GetMessageOffsetRange(*this, i); + if (OffsetRanges.empty() || OffsetRanges.back().second != from) { + OffsetRanges.emplace_back(from, to); + } else { + OffsetRanges.back().second = to; + } + } +} + +void TReadSessionEvent::TDataReceivedEvent::Commit() { + for (auto [from, to] : OffsetRanges) { + static_cast(PartitionStream.Get())->Commit(from, to); + } +} + +TReadSessionEvent::TCommitAcknowledgementEvent::TCommitAcknowledgementEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset) + : PartitionStream(std::move(partitionStream)) + , CommittedOffset(committedOffset) +{ +} + +std::string DebugString(const TReadSessionEvent::TEvent& event) { + return std::visit([](const auto& ev) { return ev.DebugString(); }, event); +} + +std::string TReadSessionEvent::TDataReceivedEvent::DebugString(bool printData) const { + TStringBuilder ret; + ret << "DataReceived { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " PartitionId: " << GetPartitionStream()->GetPartitionId(); + for (const auto& message : Messages) { + ret << " "; + message.DebugString(ret, printData); + } + for (const auto& message : CompressedMessages) { + ret << " "; + message.DebugString(ret, printData); + } + ret << " }"; + return std::move(ret); +} + +std::string TReadSessionEvent::TCommitAcknowledgementEvent::DebugString() const { + return TStringBuilder() << "CommitAcknowledgement { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " PartitionId: " << GetPartitionStream()->GetPartitionId() + << " CommittedOffset: " << GetCommittedOffset() + << " }"; +} + +std::string TReadSessionEvent::TCreatePartitionStreamEvent::DebugString() const { + return TStringBuilder() << "CreatePartitionStream { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " TopicPath: " << GetPartitionStream()->GetTopicPath() + << " Cluster: " << GetPartitionStream()->GetCluster() + << " PartitionId: " << GetPartitionStream()->GetPartitionId() + << " CommittedOffset: " << GetCommittedOffset() + << " EndOffset: " << GetEndOffset() + << " }"; +} + +std::string TReadSessionEvent::TDestroyPartitionStreamEvent::DebugString() const { + return TStringBuilder() << "DestroyPartitionStream { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " PartitionId: " << GetPartitionStream()->GetPartitionId() + << " CommittedOffset: " << GetCommittedOffset() + << " }"; +} + +std::string TReadSessionEvent::TPartitionStreamStatusEvent::DebugString() const { + return TStringBuilder() << "PartitionStreamStatus { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " PartitionId: " << GetPartitionStream()->GetPartitionId() + << " CommittedOffset: " << GetCommittedOffset() + << " ReadOffset: " << GetReadOffset() + << " EndOffset: " << GetEndOffset() + << " WriteWatermark: " << GetWriteWatermark() + << " }"; +} + +std::string TReadSessionEvent::TPartitionStreamClosedEvent::DebugString() const { + return TStringBuilder() << "PartitionStreamClosed { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() + << " PartitionId: " << GetPartitionStream()->GetPartitionId() + << " Reason: " << GetReason() + << " }"; +} + +TReadSessionEvent::TPartitionStreamStatusEvent::TPartitionStreamStatusEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset, ui64 readOffset, ui64 endOffset, TInstant writeWatermark) + : PartitionStream(std::move(partitionStream)) + , CommittedOffset(committedOffset) + , ReadOffset(readOffset) + , EndOffset(endOffset) + , WriteWatermark(writeWatermark) +{ +} + +class TGracefulReleasingSimpleDataHandlers : public TThrRefBase { +public: + explicit TGracefulReleasingSimpleDataHandlers(std::function dataHandler, bool commitAfterProcessing) + : DataHandler(std::move(dataHandler)) + , CommitAfterProcessing(commitAfterProcessing) + { + } + + void OnDataReceived(TReadSessionEvent::TDataReceivedEvent& event) { + Y_ASSERT(event.GetMessagesCount()); + TDeferredCommit deferredCommit; + { + std::lock_guard guard(Lock); + auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()]; + // Messages could contain holes in offset, but later commit ack will tell us right border. + // So we can easily insert the whole interval with holes included. + // It will be removed from set by specifying proper right border. + auto firstMessageOffsets = GetMessageOffsetRange(event, 0); + auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1); + + offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second); + + if (CommitAfterProcessing) { + deferredCommit.Add(event); + } + } + DataHandler(event); + deferredCommit.Commit(); + } + + void OnCommitAcknowledgement(TReadSessionEvent::TCommitAcknowledgementEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + auto& offsetSet = PartitionStreamToUncommittedOffsets[partitionStreamId]; + if (offsetSet.EraseInterval(0, event.GetCommittedOffset() + 1)) { // Remove some offsets. + if (offsetSet.Empty()) { // No offsets left. + auto unconfirmedDestroyIt = UnconfirmedDestroys.find(partitionStreamId); + if (unconfirmedDestroyIt != UnconfirmedDestroys.end()) { + // Confirm and forget about this partition stream. + unconfirmedDestroyIt->second.Confirm(); + UnconfirmedDestroys.erase(unconfirmedDestroyIt); + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + } + } + } + } + + void OnCreatePartitionStream(TReadSessionEvent::TCreatePartitionStreamEvent& event) { + { + std::lock_guard guard(Lock); + Y_ABORT_UNLESS(PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()].Empty()); + } + event.Confirm(); + } + + void OnDestroyPartitionStream(TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + Y_ABORT_UNLESS(UnconfirmedDestroys.find(partitionStreamId) == UnconfirmedDestroys.end()); + if (PartitionStreamToUncommittedOffsets[partitionStreamId].Empty()) { + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + event.Confirm(); + } else { + UnconfirmedDestroys.emplace(partitionStreamId, std::move(event)); + } + } + + void OnPartitionStreamClosed(TReadSessionEvent::TPartitionStreamClosedEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + UnconfirmedDestroys.erase(partitionStreamId); + } + +private: + TAdaptiveLock Lock; // For the case when user gave us multithreaded executor. + const std::function DataHandler; + const bool CommitAfterProcessing; + std::unordered_map> PartitionStreamToUncommittedOffsets; // Partition stream id -> set of offsets. + std::unordered_map UnconfirmedDestroys; // Partition stream id -> destroy events. +}; + +TReadSessionSettings::TEventHandlers& TReadSessionSettings::TEventHandlers::SimpleDataHandlers(std::function dataHandler, + bool commitDataAfterProcessing, + bool gracefulReleaseAfterCommit) { + Y_ASSERT(dataHandler); + + PartitionStreamStatusHandler([](TReadSessionEvent::TPartitionStreamStatusEvent&){}); + + if (gracefulReleaseAfterCommit) { + auto handlers = MakeIntrusive(std::move(dataHandler), commitDataAfterProcessing); + DataReceivedHandler([handlers](TReadSessionEvent::TDataReceivedEvent& event) { + handlers->OnDataReceived(event); + }); + CreatePartitionStreamHandler([handlers](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + handlers->OnCreatePartitionStream(event); + }); + DestroyPartitionStreamHandler([handlers](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + handlers->OnDestroyPartitionStream(event); + }); + CommitAcknowledgementHandler([handlers](TReadSessionEvent::TCommitAcknowledgementEvent& event) { + handlers->OnCommitAcknowledgement(event); + }); + PartitionStreamClosedHandler([handlers](TReadSessionEvent::TPartitionStreamClosedEvent& event) { + handlers->OnPartitionStreamClosed(event); + }); + } else { + if (commitDataAfterProcessing) { + DataReceivedHandler([dataHandler = std::move(dataHandler)](TReadSessionEvent::TDataReceivedEvent& event) { + TDeferredCommit deferredCommit; + deferredCommit.Add(event); + dataHandler(event); + deferredCommit.Commit(); + }); + } else { + DataReceivedHandler(std::move(dataHandler)); + } + CreatePartitionStreamHandler([](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + event.Confirm(); + }); + DestroyPartitionStreamHandler([](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + event.Confirm(); + }); + CommitAcknowledgementHandler([](TReadSessionEvent::TCommitAcknowledgementEvent&){}); + PartitionStreamClosedHandler([](TReadSessionEvent::TPartitionStreamClosedEvent&){}); + } + return *this; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TDeferredCommit + +class TDeferredCommit::TImpl { +public: + + void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset); + void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset); + + void Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message); + void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); + + void Commit(); + +private: + static void Add(const TPartitionStream::TPtr& partitionStream, TDisjointIntervalTree& offsetSet, ui64 startOffset, ui64 endOffset); + +private: + std::unordered_map, THash> Offsets; // Partition stream -> offsets set. +}; + +TDeferredCommit::TDeferredCommit() { +} + +TDeferredCommit::TDeferredCommit(TDeferredCommit&&) = default; + +TDeferredCommit& TDeferredCommit::operator=(TDeferredCommit&&) = default; + +TDeferredCommit::~TDeferredCommit() { +} + +#define GET_IMPL() \ + if (!Impl) { \ + Impl = std::make_unique(); \ + } \ + Impl + +void TDeferredCommit::Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) { + GET_IMPL()->Add(partitionStream, startOffset, endOffset); +} + +void TDeferredCommit::Add(const TPartitionStream::TPtr& partitionStream, ui64 offset) { + GET_IMPL()->Add(partitionStream, offset); +} + +void TDeferredCommit::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) { + GET_IMPL()->Add(message); +} + +void TDeferredCommit::Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent) { + GET_IMPL()->Add(dataReceivedEvent); +} + +#undef GET_IMPL + +void TDeferredCommit::Commit() { + if (Impl) { + Impl->Commit(); + } +} + +void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) { + Y_ASSERT(message.GetPartitionStream()); + Add(message.GetPartitionStream(), message.GetOffset()); +} + +void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, TDisjointIntervalTree& offsetSet, ui64 startOffset, ui64 endOffset) { + if (offsetSet.Intersects(startOffset, endOffset)) { + ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval [" + << startOffset << "; " << endOffset + << ") for partition stream with id " << partitionStream->GetPartitionStreamId()); + } else { + offsetSet.InsertInterval(startOffset, endOffset); + } +} + +void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) { + Y_ASSERT(partitionStream); + Add(partitionStream, Offsets[partitionStream], startOffset, endOffset); +} + +void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, ui64 offset) { + Y_ASSERT(partitionStream); + auto& offsetSet = Offsets[partitionStream]; + if (offsetSet.Has(offset)) { + ThrowFatalError(TStringBuilder() << "Commit set already has offset " << offset + << " for partition stream with id " << partitionStream->GetPartitionStreamId()); + } else { + offsetSet.Insert(offset); + } +} + +void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent) { + const TPartitionStream::TPtr& partitionStream = dataReceivedEvent.GetPartitionStream(); + Y_ASSERT(partitionStream); + auto& offsetSet = Offsets[partitionStream]; + auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0); + for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) { + auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i); + if (msgOffsetRange.first == endOffset) { + endOffset= msgOffsetRange.second; + } else { + Add(partitionStream, offsetSet, startOffset, endOffset); + startOffset = msgOffsetRange.first; + endOffset = msgOffsetRange.second; + } + } + Add(partitionStream, offsetSet, startOffset, endOffset); +} + +void TDeferredCommit::TImpl::Commit() { + for (auto&& [partitionStream, offsetRanges] : Offsets) { + for (auto&& [startOffset, endOffset] : offsetRanges) { + static_cast(partitionStream.Get())->Commit(startOffset, endOffset); + } + } + Offsets.clear(); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.h new file mode 100644 index 000000000000..df67c60d2149 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +// High level class that manages several read session impls. +// Each one of them works with single cluster. +// This class communicates with cluster discovery service and then creates +// sessions to each cluster. +class TReadSession : public IReadSession, + public std::enable_shared_from_this { + struct TClusterSessionInfo { + TClusterSessionInfo(const std::string& cluster) + : ClusterName(cluster) + { + } + + std::string ClusterName; // In lower case + TSingleClusterReadSessionImpl::TPtr Session; + std::vector Topics; + std::string ClusterEndpoint; + }; + +public: + TReadSession(const TReadSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + ~TReadSession(); + + void Start(); + + NThreading::TFuture WaitEvent() override; + std::vector GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) override; + std::optional GetEvent(bool block, size_t maxByteSize) override; + + bool Close(TDuration timeout) override; + + std::string GetSessionId() const override { + return SessionId; + } + + TReaderCounters::TPtr GetCounters() const override { + return Settings.Counters_; // Always not nullptr. + } + + void AddTopic(const TTopicReadSettings& topicReadSettings) /*override*/ { + Y_UNUSED(topicReadSettings); + // TODO: implement. + ThrowFatalError("Method \"AddTopic\" is not implemented"); + } + + void RemoveTopic(const std::string& path) /*override*/ { + Y_UNUSED(path); + // TODO: implement. + ThrowFatalError("Method \"RemoveTopic\" is not implemented"); + } + + void RemoveTopic(const std::string& path, const std::vector& partitionGruops) /*override*/ { + Y_UNUSED(path); + Y_UNUSED(partitionGruops); + // TODO: implement. + ThrowFatalError("Method \"RemoveTopic\" is not implemented"); + } + + void StopReadingData() override; + void ResumeReadingData() override; + + void Abort(); + + void ClearAllEvents(); + +private: + TStringBuilder GetLogPrefix() const; + + // Start + bool ValidateSettings(); + + // Cluster discovery + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest MakeClusterDiscoveryRequest() const; + void StartClusterDiscovery(); + void OnClusterDiscovery(const TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result); + void ProceedWithoutClusterDiscovery(); + void RestartClusterDiscoveryImpl(TDuration delay, TDeferredActions& deferred); + void CreateClusterSessionsImpl(TDeferredActions& deferred); + + void AbortImpl(TDeferredActions& deferred); + void AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred); + + void MakeCountersIfNeeded(); + void SetupCountersLogger(); + +private: + TReadSessionSettings Settings; + const std::string SessionId; + const TInstant StartSessionTime = TInstant::Now(); + TLog Log; + std::shared_ptr Client; + std::shared_ptr Connections; + TDbDriverStatePtr DbDriverState; + TAdaptiveLock Lock; + std::shared_ptr EventsQueue; + std::unordered_map ClusterSessions; // Cluster name (in lower case) -> TClusterSessionInfo + NYdbGrpc::IQueueClientContextPtr ClusterDiscoveryDelayContext; + IRetryPolicy::IRetryState::TPtr ClusterDiscoveryRetryState; + bool DataReadingSuspended = false; + + // Exiting. + bool Aborting = false; + bool Closing = false; + + std::vector CbContexts; + + std::shared_ptr CountersLogger; + std::shared_ptr> DumpCountersContext; +}; + +} // namespace NYdb::NPersQueue \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session_messages.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session_messages.cpp new file mode 100644 index 000000000000..4d7a82631d44 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/read_session_messages.cpp @@ -0,0 +1,244 @@ +#include + +#include +#include +#include + +namespace NYdb::inline V3::NPersQueue { + +TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation( + ui64 offset, + std::string messageGroupId, + ui64 seqNo, + TInstant createTime, + TInstant writeTime, + std::string ip, + TWriteSessionMeta::TPtr meta, + ui64 uncompressedSize +) + : Offset(offset) + , MessageGroupId(messageGroupId) + , SeqNo(seqNo) + , CreateTime(createTime) + , WriteTime(writeTime) + , Ip(ip) + , Meta(meta) + , UncompressedSize(uncompressedSize) +{} + +static void DebugStringImpl(const TReadSessionEvent::TDataReceivedEvent::TMessageInformation& info, TStringBuilder& ret) { + ret << " Information: {" + << " Offset: " << info.Offset + << " SeqNo: " << info.SeqNo + << " MessageGroupId: \"" << info.MessageGroupId << "\"" + << " CreateTime: " << info.CreateTime + << " WriteTime: " << info.WriteTime + << " Ip: \"" << info.Ip << "\"" + << " UncompressedSize: " << info.UncompressedSize; + ret << " Meta: {"; + bool firstKey = true; + for (const auto& [k, v] : info.Meta->Fields) { + ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\""; + firstKey = false; + } + ret << " } }"; +} + +template +static void DebugStringImpl(TStringBuilder& ret, + const std::string& name, + const TReadSessionEvent::TDataReceivedEvent::IMessage& msg, + bool printData, + TSerializeInformationFunc serializeInformationFunc, + std::optional codec = std::nullopt) +{ + ret << name << " {"; + try { + const std::string& data = msg.GetData(); + if (printData) { + ret << " Data: \"" << data << "\""; + } else { + ret << " Data: .." << data.size() << " bytes.."; + } + } catch (...) { + ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\""; + } + auto partitionStream = msg.GetPartitionStream(); + ret << " Partition stream id: " << partitionStream->GetPartitionStreamId() + << " Cluster: \"" << partitionStream->GetCluster() << "\". Topic: \"" << partitionStream->GetTopicPath() << "\"" + << " Partition: " << partitionStream->GetPartitionId() + << " PartitionKey: \"" << msg.GetPartitionKey() << "\""; + if (codec) { + ret << " Codec: " << codec.value(); + } + serializeInformationFunc(ret); + ret << " }"; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::IMessage::GetData() const { + return Data; +} + +const TPartitionStream::TPtr& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionStream() const { + return PartitionStream; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionKey() const { + return PartitionKey; +} + +const std::string TReadSessionEvent::TDataReceivedEvent::IMessage::GetExplicitHash() const { + return ExplicitHash; +} + +std::string TReadSessionEvent::TDataReceivedEvent::IMessage::DebugString(bool printData) const { + TStringBuilder ret; + DebugString(ret, printData); + return std::move(ret); +} + +TReadSessionEvent::TDataReceivedEvent::IMessage::IMessage(const std::string& data, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash) + : Data(data) + , PartitionStream(partitionStream) + , PartitionKey(partitionKey) + , ExplicitHash(explicitHash) +{} + +const std::string& TReadSessionEvent::TDataReceivedEvent::TMessage::GetData() const { + if (DecompressionException) { + std::rethrow_exception(DecompressionException); + } + return IMessage::GetData(); +} + +bool TReadSessionEvent::TDataReceivedEvent::TMessage::HasException() const { + return DecompressionException != nullptr; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetOffset() const { + return Information.Offset; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMessageGroupId() const { + return Information.MessageGroupId; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetSeqNo() const { + return Information.SeqNo; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetCreateTime() const { + return Information.CreateTime; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetWriteTime() const { + return Information.WriteTime; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::TMessage::GetIp() const { + return Information.Ip; +} + +const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMeta() const { + return Information.Meta; +} + +void TReadSessionEvent::TDataReceivedEvent::TMessage::DebugString(TStringBuilder& ret, bool printData) const { + DebugStringImpl(ret, "Message", *this, printData, [this](TStringBuilder& ret) { + DebugStringImpl(this->Information, ret); + }); +} + +TReadSessionEvent::TDataReceivedEvent::TMessage::TMessage(const std::string& data, + std::exception_ptr decompressionException, + const TMessageInformation& information, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash) + : IMessage(data, partitionStream, partitionKey, explicitHash) + , DecompressionException(std::move(decompressionException)) + , Information(information) +{ +} + +void TReadSessionEvent::TDataReceivedEvent::TMessage::Commit() { + static_cast(PartitionStream.Get())->Commit(Information.Offset, Information.Offset + 1); +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetBlocksCount() const { + return Information.size(); +} + +ECodec TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCodec() const { + return Codec; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetOffset(ui64 index) const { + return Information.at(index).Offset; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMessageGroupId(ui64 index) const { + return Information.at(index).MessageGroupId; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetSeqNo(ui64 index) const { + return Information.at(index).SeqNo; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCreateTime(ui64 index) const { + return Information.at(index).CreateTime; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetWriteTime(ui64 index) const { + return Information.at(index).WriteTime; +} + +const std::string& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetIp(ui64 index) const { + return Information.at(index).Ip; +} + +const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMeta(ui64 index) const { + return Information.at(index).Meta; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetUncompressedSize(ui64 index) const { + return Information.at(index).UncompressedSize; +} + +void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::DebugString(TStringBuilder& ret, bool printData) const { + DebugStringImpl( + ret, + "CompressedMessage", + *this, + printData, + [this](TStringBuilder& ret) { + for (auto& info : this->Information) { + DebugStringImpl(info, ret); + } + }, + Codec + ); +} + +TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::TCompressedMessage(ECodec codec, + const std::string& data, + const std::vector& information, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash) + : IMessage(data, partitionStream, partitionKey, explicitHash) + , Codec(codec) + , Information(information) +{} + +void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::Commit() { + static_cast(PartitionStream.Get())->Commit( + Information.front().Offset, + Information.back().Offset + 1 + ); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.cpp new file mode 100644 index 000000000000..efafbfa2d5da --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.cpp @@ -0,0 +1,152 @@ +#include "write_session.h" + +#include + +#include + +#include +#include +#include + +namespace NYdb::inline V3::NPersQueue { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSession + +TWriteSession::TWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : TContextOwner(settings, std::move(client), std::move(connections), std::move(dbDriverState)) { +} + +void TWriteSession::Start(const TDuration& delay) { + TryGetImpl()->Start(delay); +} + +NThreading::TFuture TWriteSession::GetInitSeqNo() { + return TryGetImpl()->GetInitSeqNo(); +} + +std::optional TWriteSession::GetEvent(bool block) { + return TryGetImpl()->EventsQueue->GetEvent(block); +} + +std::vector TWriteSession::GetEvents(bool block, std::optional maxEventsCount) { + return TryGetImpl()->EventsQueue->GetEvents(block, maxEventsCount); +} + +NThreading::TFuture TWriteSession::WaitEvent() { + return TryGetImpl()->EventsQueue->WaitEvent(); +} + +void TWriteSession::WriteEncoded(TContinuationToken&& token, std::string_view data, ECodec codec, ui32 originalSize, + std::optional seqNo, std::optional createTimestamp) { + TryGetImpl()->WriteInternal(std::move(token), data, codec, originalSize, seqNo, createTimestamp); +} + +void TWriteSession::Write(TContinuationToken&& token, std::string_view data, std::optional seqNo, + std::optional createTimestamp) { + TryGetImpl()->WriteInternal(std::move(token), data, {}, 0, seqNo, createTimestamp); +} + +bool TWriteSession::Close(TDuration closeTimeout) { + return TryGetImpl()->Close(closeTimeout); +} + +TWriteSession::~TWriteSession() { + TryGetImpl()->Close(TDuration::Zero()); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TSimpleBlockingWriteSession + +TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState +) { + auto subSettings = settings; + if (settings.EventHandlers_.AcksHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use AcksHandler, resetting."); + subSettings.EventHandlers_.AcksHandler({}); + } + if (settings.EventHandlers_.ReadyToAcceptHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use ReadyToAcceptHandler, resetting."); + subSettings.EventHandlers_.ReadyToAcceptHandler({}); + } + if (settings.EventHandlers_.SessionClosedHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use SessionClosedHandler, resetting."); + subSettings.EventHandlers_.SessionClosedHandler({}); + } + if (settings.EventHandlers_.CommonHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use CommonHandler, resetting."); + subSettings.EventHandlers_.CommonHandler({}); + } + Writer = std::make_shared(subSettings, client, connections, dbDriverState); + Writer->Start(TDuration::Max()); +} + +ui64 TSimpleBlockingWriteSession::GetInitSeqNo() { + return Writer->GetInitSeqNo().GetValueSync(); +} + +bool TSimpleBlockingWriteSession::Write( + std::string_view data, std::optional seqNo, std::optional createTimestamp, const TDuration& blockTimeout +) { + auto continuationToken = WaitForToken(blockTimeout); + if (continuationToken.has_value()) { + Writer->Write(std::move(*continuationToken), std::move(data), seqNo, createTimestamp); + return true; + } + return false; +} + +std::optional TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) { + TInstant startTime = TInstant::Now(); + TDuration remainingTime = timeout; + + std::optional token = std::nullopt; + + while (IsAlive() && remainingTime > TDuration::Zero()) { + Writer->WaitEvent().Wait(remainingTime); + + for (auto event : Writer->GetEvents()) { + if (auto* readyEvent = std::get_if(&event)) { + Y_ABORT_UNLESS(!token.has_value()); + token = std::move(readyEvent->ContinuationToken); + } else if (auto* ackEvent = std::get_if(&event)) { + // discard + } else if (auto* closeSessionEvent = std::get_if(&event)) { + Closed.store(true); + return std::nullopt; + } + } + + if (token.has_value()) { + return token; + } + + remainingTime = timeout - (TInstant::Now() - startTime); + } + + return std::nullopt; +} + +TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() { + return Writer->GetCounters(); +} + + +bool TSimpleBlockingWriteSession::IsAlive() const { + return !Closed.load(); +} + +bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) { + Closed.store(true); + return Writer->Close(std::move(closeTimeout)); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.h new file mode 100644 index 000000000000..7fa9f4c2c618 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session.h @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +namespace NTests { + class TSimpleWriteSessionTestAdapter; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSession + +class TWriteSession : public IWriteSession, + public TContextOwner { +private: + friend class TSimpleBlockingWriteSession; + friend class TPersQueueClient; + friend class NTests::TSimpleWriteSessionTestAdapter; + +public: + TWriteSession(const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + std::optional GetEvent(bool block = false) override; + std::vector GetEvents(bool block = false, + std::optional maxEventsCount = std::nullopt) override; + NThreading::TFuture GetInitSeqNo() override; + + void Write(TContinuationToken&& continuationToken, std::string_view data, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + + void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, ui32 originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + + + NThreading::TFuture WaitEvent() override; + + // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. + bool Close(TDuration closeTimeout = TDuration::Max()) override; + + TWriterCounters::TPtr GetCounters() override {Y_ABORT("Unimplemented"); } //ToDo - unimplemented; + + ~TWriteSession(); // will not call close - destroy everything without acks + +private: + void Start(const TDuration& delay); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TSimpleBlockingWriteSession + +class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { + friend class NTests::TSimpleWriteSessionTestAdapter; + +private: + using TClientMessage = TWriteSessionImpl::TClientMessage; + using TServerMessage = TWriteSessionImpl::TServerMessage; + using IWriteSessionConnectionProcessorFactory = TWriteSessionImpl::IWriteSessionConnectionProcessorFactory; + using IProcessor = TWriteSessionImpl::IProcessor; + +public: + TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, + const TDuration& blockTimeout = TDuration::Max()) override; + + ui64 GetInitSeqNo() override; + + bool Close(TDuration closeTimeout = TDuration::Max()) override; + bool IsAlive() const override; + + TWriterCounters::TPtr GetCounters() override; + +protected: + std::shared_ptr Writer; + +private: + std::optional WaitForToken(const TDuration& timeout); + + std::atomic_bool Closed = false; +}; + + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp new file mode 100644 index 000000000000..022bea0714ed --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp @@ -0,0 +1,1405 @@ +#include "write_session_impl.h" + +#include + +#include + +#include + +#include +#include +#include + + +namespace NYdb::inline V3::NPersQueue { + +const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSessionImpl + +TWriteSessionImpl::TWriteSessionImpl( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : Settings(settings) + , Client(std::move(client)) + , Connections(std::move(connections)) + , DbDriverState(std::move(dbDriverState)) + , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") + , InitSeqNoPromise(NThreading::NewPromise()) + , WakeupInterval( + Settings.BatchFlushInterval_.value_or(TDuration::Zero()) ? + std::min(Settings.BatchFlushInterval_.value_or(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100)) + : + TDuration::MilliSeconds(100) + ) +{ + if (!Settings.RetryPolicy_) { + Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); + } + if (Settings.PreferredCluster_ && !Settings.AllowFallbackToOtherClusters_) { + TargetCluster = *Settings.PreferredCluster_; + NUtils::ToLower(TargetCluster); + } + if (Settings.Counters_.has_value()) { + Counters = *Settings.Counters_; + } else { + Counters = MakeIntrusive(new ::NMonitoring::TDynamicCounters()); + } + +} + +void TWriteSessionImpl::Start(const TDuration& delay) { + Y_ABORT_UNLESS(SelfContext); + + if (!EventsQueue) { +#define WRAP_HANDLER(type, handler, ...) \ + if (auto h = Settings.EventHandlers_.handler##_) { \ + Settings.EventHandlers_.handler([ctx = SelfContext, h = std::move(h)](__VA_ARGS__ type& ev){ \ + if (auto self = ctx->LockShared()) { \ + h(ev); \ + } \ + }); \ + } + WRAP_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler); + WRAP_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHandler); + WRAP_HANDLER(TSessionClosedEvent, SessionClosedHandler, const); + WRAP_HANDLER(TWriteSessionEvent::TEvent, CommonHandler); +#undef WRAP_HANDLER + + EventsQueue = std::make_shared(Settings); + } + + ++ConnectionAttemptsDone; + if (!Started) { + { + std::lock_guard guard(Lock); + HandleWakeUpImpl(); + } + InitWriter(); + } + Started = true; + + DoCdsRequest(delay); +} + +TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStatus& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + THandleResult result; + if (Aborting.load()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session is aborting and will not restart"); + return result; + } + + SessionEstablished = false; + if (!RetryState) { + RetryState = Settings.RetryPolicy_->CreateRetryState(); + } + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); + + if (nextDelay) { + result.StartDelay = *nextDelay; + result.DoRestart = true; + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will restart in " << result.StartDelay); + ResetForRetryImpl(); + } else { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session will not restart after a fatal error"); + result.DoStop = true; + CheckHandleResultImpl(result); + } + return result; +} + +bool IsFederation(const std::string& endpoint) { + std::string_view host = GetHost(endpoint); + return host == "logbroker.yandex.net" || host == "logbroker-prestable.yandex.net"; +} + +void TWriteSessionImpl::DoCdsRequest(TDuration delay) { + bool cdsRequestIsUnnecessary; + { + std::lock_guard guard(Lock); + if (Aborting.load()) { + return; + } + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session: Do CDS request"); + + cdsRequestIsUnnecessary = (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Off || + (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Auto && !IsFederation(DbDriverState->DiscoveryEndpoint))); + + if (!cdsRequestIsUnnecessary) { + auto extractor = [cbContext = SelfContext] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; + if (any) { + any->UnpackTo(&result); + } + TStatus st(std::move(status)); + if (auto self = cbContext->LockShared()) { + self->OnCdsResponse(st, result); + } + }; + + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; + auto* params = req.add_write_sessions(); + params->set_topic(TStringType{Settings.Path_}); + params->set_source_id(TStringType{Settings.MessageGroupId_}); + if (Settings.PartitionGroupId_.has_value()) + params->set_partition_group(Settings.PartitionGroupId_.value()); + if (Settings.PreferredCluster_.has_value()) + params->set_preferred_cluster_name(TStringType{Settings.PreferredCluster_.value()}); + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Do schedule cds request after " << delay.MilliSeconds() << " ms\n"); + auto cdsRequestCall = [req_=std::move(req), extr=std::move(extractor), connections = std::shared_ptr(Connections), dbState=DbDriverState, settings=Settings]() mutable { + LOG_LAZY(dbState->Log, TLOG_INFO, TStringBuilder() << "MessageGroupId [" << settings.MessageGroupId_ << "] Running cds request ms\n"); + connections->RunDeferred( + std::move(req_), + std::move(extr), + &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, + dbState, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); // TODO: make client timeout setting + }; + Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), delay); + return; + } + } + + if (cdsRequestIsUnnecessary) { + DoConnect(delay, DbDriverState->DiscoveryEndpoint); + return; + } +} + +void TWriteSessionImpl::OnCdsResponse( + TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result +) { + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got CDS response: \n" << result.ShortDebugString()); + std::string endpoint, name; + THandleResult handleResult; + if (!status.IsSuccess()) { + { + std::lock_guard guard(Lock); + handleResult = OnErrorImpl({ + status.GetStatus(), + MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues()) + }); + } + ProcessHandleResult(handleResult); + return; + } + + NYdb::NIssue::TIssues issues; + EStatus errorStatus = EStatus::INTERNAL_ERROR; + { + std::lock_guard guard(Lock); + const Ydb::PersQueue::ClusterDiscovery::WriteSessionClusters& wsClusters = result.write_sessions_clusters(0); + bool isFirst = true; + + for (const auto& clusterInfo : wsClusters.clusters()) { + std::string normalizedName = clusterInfo.name(); + NUtils::ToLower(normalizedName); + + if(isFirst) { + isFirst = false; + PreferredClusterByCDS = clusterInfo.name(); + } + + if (!clusterInfo.available()) { + if (!TargetCluster.empty() && TargetCluster == normalizedName) { + errorStatus = EStatus::UNAVAILABLE; + issues.AddIssue(TStringBuilder() << "Selected destination cluster: " << normalizedName + << " is currently disabled"); + break; + } + continue; + } + if (clusterInfo.endpoint().empty()) { + issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster " + << normalizedName); + } else { + name = clusterInfo.name(); + endpoint = ApplyClusterEndpoint(DbDriverState->DiscoveryEndpoint, clusterInfo.endpoint()); + break; + } + } + if (endpoint.empty()) { + errorStatus = EStatus::GENERIC_ERROR; + issues.AddIssue(TStringBuilder() << "Could not get valid endpoint from cluster discovery"); + } + } + if (issues) { + { + std::lock_guard guard(Lock); + handleResult = OnErrorImpl({errorStatus, std::move(issues)}); + } + ProcessHandleResult(handleResult); + return; + } + { + std::lock_guard guard(Lock); + if (InitialCluster.empty()) { + InitialCluster = name; + } + CurrentCluster = name; + } + DoConnect(TDuration::Zero(), endpoint); + +} + +void TWriteSessionImpl::InitWriter() { // No Lock, very initial start - no race yet as well. + CompressionExecutor = Settings.CompressionExecutor_; + IExecutor::TPtr executor; + executor = CreateSyncExecutor(); + executor->Start(); + Executor = std::move(executor); + + Settings.CompressionExecutor_->Start(); + Settings.EventHandlers_.HandlersExecutor_->Start(); + +} +// Client method +NThreading::TFuture TWriteSessionImpl::GetInitSeqNo() { + if (Settings.ValidateSeqNo_) { + if (AutoSeqNoMode.has_value() && *AutoSeqNoMode) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Cannot call GetInitSeqNo in Auto SeqNo mode"); + ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode"); + } + else + AutoSeqNoMode = false; + } + return InitSeqNoPromise.GetFuture(); +} + +std::string DebugString(const TWriteSessionEvent::TEvent& event) { + return std::visit([](const auto& ev) { return ev.DebugString(); }, event); +} + +// Client method +std::optional TWriteSessionImpl::GetEvent(bool block) { + return EventsQueue->GetEvent(block); +} + +// Client method +std::vector TWriteSessionImpl::GetEvents(bool block, std::optional maxEventsCount) { + return EventsQueue->GetEvents(block, maxEventsCount); +} + +ui64 TWriteSessionImpl::GetIdImpl(ui64 seqNo) { + Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); + Y_ABORT_UNLESS(!*AutoSeqNoMode || InitSeqNo.contains(CurrentCluster) && seqNo > InitSeqNo[CurrentCluster]); + return *AutoSeqNoMode ? seqNo - InitSeqNo[CurrentCluster] : seqNo; +} + +ui64 TWriteSessionImpl::GetSeqNoImpl(ui64 id) { + Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); + return *AutoSeqNoMode ? id + InitSeqNo[CurrentCluster] : id; + +} + +ui64 TWriteSessionImpl::GetNextIdImpl(const std::optional& seqNo) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + ui64 id = ++NextId; + + if (!AutoSeqNoMode.has_value()) { + AutoSeqNoMode = !seqNo.has_value(); + } + if (seqNo.has_value()) { + if (*AutoSeqNoMode) { + LOG_LAZY(DbDriverState->Log, + TLOG_ERR, + LogPrefix() << "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" + ); + ThrowFatalError( + "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" + ); + + } else { + id = *seqNo; + } + } else if (!(*AutoSeqNoMode)) { + LOG_LAZY(DbDriverState->Log, + TLOG_ERR, + LogPrefix() << "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" + ); + ThrowFatalError( + "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" + ); + } + return id; +} + +inline void TWriteSessionImpl::CheckHandleResultImpl(THandleResult& result) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true); +} + +void TWriteSessionImpl::ProcessHandleResult(THandleResult& result) { + if (result.DoRestart) { + Start(result.StartDelay); + } else if (result.DoSetSeqNo) { + InitSeqNoPromise.SetException("session closed"); + } +} + +NThreading::TFuture TWriteSessionImpl::WaitEvent() { + return EventsQueue->WaitEvent(); +} + +// Client method. +void TWriteSessionImpl::WriteInternal( + TContinuationToken&&, std::string_view data, std::optional codec, ui32 originalSize, std::optional seqNo, std::optional createTimestamp + ) { + TInstant createdAtValue = createTimestamp.value_or(TInstant::Now()); + bool readyToAccept = false; + size_t bufferSize = data.size(); + { + std::lock_guard guard(Lock); + CurrentBatch.Add(GetNextIdImpl(seqNo), createdAtValue, data, codec, originalSize); + + FlushWriteIfRequiredImpl(); + readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk; + } + if (readyToAccept) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } +} + +// Client method. +void TWriteSessionImpl::WriteEncoded( + TContinuationToken&& token, std::string_view data, ECodec codec, ui32 originalSize, std::optional seqNo, std::optional createTimestamp + ) { + WriteInternal(std::move(token), data, codec, originalSize, seqNo, createTimestamp); +} + +void TWriteSessionImpl::Write( + TContinuationToken&& token, std::string_view data, std::optional seqNo, std::optional createTimestamp + ) { + WriteInternal(std::move(token), data, {}, 0, seqNo, createTimestamp); +} + + +TWriteSessionImpl::THandleResult TWriteSessionImpl::OnErrorImpl(NYdb::TPlainStatus&& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + (*Counters->Errors)++; + auto result = RestartImpl(status); + if (result.DoStop) { + CloseImpl(status.Status, std::move(status.Issues)); + } + return result; +} + +// No lock +void TWriteSessionImpl::DoConnect(const TDuration& delay, const std::string& endpoint) { + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Start write session. Will connect to endpoint: " << endpoint); + + NYdbGrpc::IQueueClientContextPtr prevConnectContext; + NYdbGrpc::IQueueClientContextPtr prevConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr prevConnectDelayContext; + NYdbGrpc::IQueueClientContextPtr connectContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectDelayContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr; + TRpcRequestSettings reqSettings; + std::shared_ptr connectionFactory; + + // Callbacks + std::function connectCallback; + std::function connectTimeoutCallback; + + { + std::lock_guard guard(Lock); + if (Aborting) { + return; + } + ++ConnectionGeneration; + auto subclient = Client->GetClientForEndpoint(endpoint); + auto clientContext = subclient->CreateContext(); + if (!clientContext) { + AbortImpl(); + // Grpc and WriteSession is closing right now. + return; + } + auto prevClientContext = std::exchange(ClientContext, clientContext); + + ServerMessage = std::make_shared(); + + connectionFactory = subclient->CreateWriteSessionConnectionProcessorFactory(); + ConnectionFactory = connectionFactory; + + connectContext = ClientContext->CreateContext(); + if (delay) + connectDelayContext = ClientContext->CreateContext(); + connectTimeoutContext = ClientContext->CreateContext(); + + // Previous operations contexts. + + // Set new context + prevConnectContext = std::exchange(ConnectContext, connectContext); + prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); + prevConnectDelayContext = std::exchange(ConnectDelayContext, connectDelayContext); + Y_ASSERT(ConnectContext); + Y_ASSERT(ConnectTimeoutContext); + + if (Processor) { + Processor->Cancel(); + } + + // Cancel previous operations. + Cancel(prevConnectContext); + if (prevConnectDelayContext) + Cancel(prevConnectDelayContext); + Cancel(prevConnectTimeoutContext); + Cancel(prevClientContext); + Y_ASSERT(connectContext); + Y_ASSERT(connectTimeoutContext); + + reqSettings = TRpcRequestSettings::Make(Settings); + + connectCallback = [cbContext = SelfContext, + connectContext = connectContext](TPlainStatus&& st, typename IProcessor::TPtr&& processor) { + if (auto self = cbContext->LockShared()) { + self->OnConnect(std::move(st), std::move(processor), connectContext); + } + }; + + connectTimeoutCallback = [cbContext = SelfContext, connectTimeoutContext = connectTimeoutContext](bool ok) { + if (ok) { + if (auto self = cbContext->LockShared()) { + self->OnConnectTimeout(connectTimeoutContext); + } + } + }; + } + + connectionFactory->CreateProcessor( + std::move(connectCallback), + reqSettings, + std::move(connectContext), + TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting. + std::move(connectTimeoutContext), + std::move(connectTimeoutCallback), + delay, + std::move(connectDelayContext) + ); +} + +// RPC callback. +void TWriteSessionImpl::OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session: connect timeout"); + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (ConnectTimeoutContext == connectTimeoutContext) { + Cancel(ConnectContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + } else { + return; + } + TStringBuilder description; + description << "Failed to establish connection to server. Attempts done: " << ConnectionAttemptsDone; + handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description)); + if (handleResult.DoStop) { + CloseImpl( + EStatus::TIMEOUT, + description + ); + } + } + ProcessHandleResult(handleResult); +} + +// RPC callback. +void TWriteSessionImpl::OnConnect( + TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NYdbGrpc::IQueueClientContextPtr& connectContext +) { + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (ConnectContext == connectContext) { + Cancel(ConnectTimeoutContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + + if (st.Ok()) { + Processor = std::move(processor); + InitImpl(); + // Still should call ReadFromProcessor(); + } + } else { + return; + } + if (!st.Ok()) { + handleResult = RestartImpl(st); + if (handleResult.DoStop) { + CloseImpl( + st.Status, + MakeIssueWithSubIssues( + TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint + << "\". Attempts done: " << ConnectionAttemptsDone, + st.Issues + ) + ); + } + } + } + if (st.Ok()) + ReadFromProcessor(); // Out of Init + ProcessHandleResult(handleResult); +} + +// Produce init request for session. +void TWriteSessionImpl::InitImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + Ydb::PersQueue::V1::StreamingWriteClientMessage req; + auto* init = req.mutable_init_request(); + init->set_topic(TStringType{Settings.Path_}); + init->set_message_group_id(TStringType{Settings.MessageGroupId_}); + if (Settings.PartitionGroupId_) { + init->set_partition_group_id(*Settings.PartitionGroupId_); + } + init->set_max_supported_format_version(0); + init->set_preferred_cluster(TStringType{PreferredClusterByCDS}); + + for (const auto& attr : Settings.Meta_.Fields) { + (*init->mutable_session_meta())[attr.first] = attr.second; + } + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: send init request: "<< req.ShortDebugString()); + WriteToProcessorImpl(std::move(req)); +} + +// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe +void TWriteSessionImpl::WriteToProcessorImpl(TWriteSessionImpl::TClientMessage&& req) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + Y_ASSERT(Processor); + if (Aborting) { + return; + } + auto callback = [cbContext = SelfContext, + connectionGeneration = ConnectionGeneration](NYdbGrpc::TGrpcStatus&& grpcStatus) { + if (auto self = cbContext->LockShared()) { + self->OnWriteDone(std::move(grpcStatus), connectionGeneration); + } + }; + + Processor->Write(std::move(req), std::move(callback)); +} + +void TWriteSessionImpl::ReadFromProcessor() { + Y_ASSERT(Processor); + IProcessor::TPtr prc; + ui64 generation; + std::function callback; + { + std::lock_guard guard(Lock); + if (Aborting) { + return; + } + prc = Processor; + generation = ConnectionGeneration; + callback = [cbContext = SelfContext, + connectionGeneration = generation, + processor = prc, + serverMessage = ServerMessage] + (NYdbGrpc::TGrpcStatus&& grpcStatus) { + if (auto self = cbContext->LockShared()) { + self->OnReadDone(std::move(grpcStatus), connectionGeneration); + } + }; + } + prc->Read(ServerMessage.get(), std::move(callback)); +} + +void TWriteSessionImpl::OnWriteDone(NYdbGrpc::TGrpcStatus&& status, size_t connectionGeneration) { + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (Aborting) { + return; + } + if(!status.Ok()) { + handleResult = OnErrorImpl(status); + } + } + ProcessHandleResult(handleResult); +} + +void TWriteSessionImpl::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) { + TPlainStatus errorStatus; + TProcessSrvMessageResult processResult; + bool needSetValue = false; + if (!grpcStatus.Ok()) { + errorStatus = TPlainStatus(std::move(grpcStatus)); + } + bool doRead = false; + { + std::lock_guard guard(Lock); + UpdateTimedCountersImpl(); + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (errorStatus.Ok()) { + if (IsErrorMessage(*ServerMessage)) { + errorStatus = MakeErrorFromProto(*ServerMessage); + } else { + processResult = ProcessServerMessageImpl(); + needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.has_value() && (InitSeqNoSetDone = true); + if (errorStatus.Ok() && processResult.Ok) { + doRead = true; + } + } + } + } + if (doRead) + ReadFromProcessor(); + + { + std::lock_guard guard(Lock); + if (!errorStatus.Ok()) { + if (processResult.Ok) { // Otherwise, OnError was already called + processResult.HandleResult = RestartImpl(errorStatus); + } + } + if (processResult.HandleResult.DoStop) { + CloseImpl(std::move(errorStatus)); + } + } + for (auto& event : processResult.Events) { + EventsQueue->PushEvent(std::move(event)); + } + if (needSetValue) { + InitSeqNoPromise.SetValue(*processResult.InitSeqNo); + processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case. + } + ProcessHandleResult(processResult.HandleResult); +} + +TStringBuilder TWriteSessionImpl::LogPrefix() const { + return TStringBuilder() << "MessageGroupId [" << Settings.MessageGroupId_ << "] SessionId [" << SessionId << "] "; +} + +std::string TWriteSessionEvent::TAcksEvent::DebugString() const { + TStringBuilder res; + res << "AcksEvent:"; + for (auto& ack : Acks) { + res << " { seqNo : " << ack.SeqNo << ", State : " << ack.State; + if (ack.Details) { + res << ", offset : " << ack.Details->Offset << ", partitionId : " << ack.Details->PartitionId; + } + res << " }"; + } + if (!Acks.empty() && Acks.back().Stat) { + auto& stat = Acks.back().Stat; + res << " write stat: Write time " << stat->WriteTime << " total time in partition queue " << stat->TotalTimeInPartitionQueue + << " partition quoted time " << stat->PartitionQuotedTime << " topic quoted time " << stat->TopicQuotedTime; + } + return res; +} + +std::string TWriteSessionEvent::TReadyToAcceptEvent::DebugString() const { + return "ReadyToAcceptEvent"; +} + + +TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMessageImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + TProcessSrvMessageResult result; + switch (ServerMessage->server_message_case()) { + case TServerMessage::SERVER_MESSAGE_NOT_SET: { + SessionEstablished = false; + result.HandleResult = OnErrorImpl({ + static_cast(ServerMessage->status()), + {NYdb::NIssue::TIssue{ServerMessage->DebugString()}} + }); + result.Ok = false; + break; + } + case TServerMessage::kInitResponse: { + const auto& initResponse = ServerMessage->init_response(); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session established. Init response: " << initResponse.ShortDebugString()); + SessionId = initResponse.session_id(); + PartitionId = initResponse.partition_id(); + ui64 newLastSeqNo = initResponse.last_sequence_number(); + result.InitSeqNo = newLastSeqNo; + if (!InitSeqNo.contains(CurrentCluster)) { + InitSeqNo[CurrentCluster] = newLastSeqNo >= MinUnsentId ? newLastSeqNo - MinUnsentId + 1 : 0; + } + + SessionEstablished = true; + LastCountersUpdateTs = TInstant::Now(); + SessionStartedTs = TInstant::Now(); + OnErrorResolved(); + + if (!FirstTokenSent) { + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + FirstTokenSent = true; + } + // Kickstart send after session reestablishment + SendImpl(); + break; + } + case TServerMessage::kBatchWriteResponse: { + TWriteSessionEvent::TAcksEvent acksEvent; + const auto& batchWriteResponse = ServerMessage->batch_write_response(); + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Write session got write response: " << batchWriteResponse.ShortDebugString() + ); + TWriteStat::TPtr writeStat = new TWriteStat{}; + const auto& stat = batchWriteResponse.write_statistics(); + writeStat->WriteTime = TDuration::MilliSeconds(stat.persist_duration_ms()); + writeStat->TotalTimeInPartitionQueue = TDuration::MilliSeconds(stat.queued_in_partition_duration_ms()); + writeStat->PartitionQuotedTime = TDuration::MilliSeconds(stat.throttled_on_partition_duration_ms()); + writeStat->TopicQuotedTime = TDuration::MilliSeconds(stat.throttled_on_topic_duration_ms()); + + for (size_t messageIndex = 0, endIndex = batchWriteResponse.sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) { + // TODO: Fill writer statistics + ui64 sequenceNumber = batchWriteResponse.sequence_numbers(messageIndex); + + acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{ + GetIdImpl(sequenceNumber), + batchWriteResponse.already_written(messageIndex) ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN: + TWriteSessionEvent::TWriteAck::EES_WRITTEN, + TWriteSessionEvent::TWriteAck::TWrittenMessageDetails { + static_cast(batchWriteResponse.offsets(messageIndex)), + PartitionId, + }, + writeStat, + }); + + if (CleanupOnAcknowledged(GetIdImpl(sequenceNumber))) { + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } + } + //EventsQueue->PushEvent(std::move(acksEvent)); + result.Events.emplace_back(std::move(acksEvent)); + break; + } + case TServerMessage::kUpdateTokenResponse: { + UpdateTokenInProgress = false; + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: token updated successfully"); + UpdateTokenIfNeededImpl(); + break; + } + } + return result; +} + +bool TWriteSessionImpl::CleanupOnAcknowledged(ui64 id) { + bool result = false; + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: acknoledged message " << id); + UpdateTimedCountersImpl(); + if (SentOriginalMessages.empty() || SentOriginalMessages.front().Id != id){ + std::cerr << "State before restart was:\n" << StateStr << "\n\n"; + DumpState(); + std::cerr << "State on ack with id " << id << " is:\n"; + std::cerr << StateStr << "\n\n"; + Y_ABORT("got unknown ack"); + } + + const auto& sentFront = SentOriginalMessages.front(); + ui64 size = 0; + ui64 compressedSize = 0; + if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == id) { + auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); + result = memoryUsage.NowOk && !memoryUsage.WasOk; + const auto& front = SentPackedMessage.front(); + if (front.Compressed) { + compressedSize = front.Data.size(); + } else { + size = front.Data.size(); + } + + (*Counters->MessagesWritten) += front.MessageCount; + (*Counters->MessagesInflight) -= front.MessageCount; + (*Counters->BytesWritten) += front.OriginalSize; + + SentPackedMessage.pop(); + } else { + size = sentFront.Size; + (*Counters->BytesWritten) += sentFront.Size; + (*Counters->MessagesWritten)++; + (*Counters->MessagesInflight)--; + } + + (*Counters->BytesInflightCompressed) -= compressedSize; + (*Counters->BytesWrittenCompressed) += compressedSize; + (*Counters->BytesInflightUncompressed) -= size; + + Y_ABORT_UNLESS(Counters->BytesInflightCompressed->Val() >= 0); + Y_ABORT_UNLESS(Counters->BytesInflightUncompressed->Val() >= 0); + + Y_ABORT_UNLESS(sentFront.Id == id); + + (*Counters->BytesInflightTotal) = MemoryUsage; + SentOriginalMessages.pop(); + return result; +} + +TMemoryUsageChange TWriteSessionImpl::OnMemoryUsageChangedImpl(i64 diff) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_; + //if (diff < 0) { + // Y_ABORT_UNLESS(MemoryUsage >= static_cast(std::abs(diff))); + //} + MemoryUsage += diff; + bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_; + if (wasOk != nowOk) { + if (wasOk) { + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Estimated memory usage " << MemoryUsage + << "[B] reached maximum (" << Settings.MaxMemoryUsage_ << "[B])" + ); + } + else { + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Estimated memory usage got back to normal " << MemoryUsage << "[B]" + ); + } + } + return {wasOk, nowOk}; +} + +TBuffer CompressBuffer(std::shared_ptr client, std::vector& data, ECodec codec, i32 level) { + TBuffer result; + Y_UNUSED(client); + std::unique_ptr coder = TCodecMap::GetTheCodecMap().GetOrThrow((ui32)codec)->CreateCoder(result, level); + for (auto& buffer : data) { + coder->Write(buffer.data(), buffer.size()); + } + coder->Finish(); + return result; +} + +// May call OnCompressed with sync executor. No external lock. +void TWriteSessionImpl::CompressImpl(TBlock&& block_) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (Aborting) { + return; + } + Y_ABORT_UNLESS(block_.Valid); + + std::shared_ptr blockPtr(std::make_shared()); + blockPtr->Move(block_); + auto lambda = [cbContext = SelfContext, + codec = Settings.Codec_, + level = Settings.CompressionLevel_, + isSyncCompression = !CompressionExecutor->IsAsync(), + blockPtr, + client = Client]() mutable { + Y_ABORT_UNLESS(!blockPtr->Compressed); + + auto compressedData = CompressBuffer( + std::move(client), blockPtr->OriginalDataRefs, codec, level + ); + Y_ABORT_UNLESS(!compressedData.Empty()); + blockPtr->Data = std::move(compressedData); + blockPtr->Compressed = true; + blockPtr->CodecID = GetCodecId(codec); + if (auto self = cbContext->LockShared()) { + self->OnCompressed(std::move(*blockPtr), isSyncCompression); + } + }; + + CompressionExecutor->Post(std::move(lambda)); +} + +void TWriteSessionImpl::OnCompressed(TBlock&& block, bool isSyncCompression) { + TMemoryUsageChange memoryUsage; + if (!isSyncCompression) { + std::lock_guard guard(Lock); + memoryUsage = OnCompressedImpl(std::move(block)); + } else { + memoryUsage = OnCompressedImpl(std::move(block)); + } + if (memoryUsage.NowOk && !memoryUsage.WasOk) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } +} + +TMemoryUsageChange TWriteSessionImpl::OnCompressedImpl(TBlock&& block) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTimedCountersImpl(); + Y_ABORT_UNLESS(block.Valid); + auto memoryUsage = OnMemoryUsageChangedImpl(static_cast(block.Data.size()) - block.OriginalMemoryUsage); + (*Counters->BytesInflightUncompressed) -= block.OriginalSize; + (*Counters->BytesInflightCompressed) += block.Data.size(); + + PackedMessagesToSend.emplace(std::move(block)); + SendImpl(); + return memoryUsage; +} + +void TWriteSessionImpl::ResetForRetryImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + DumpState(); + + SessionEstablished = false; + const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size(); + const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size(); + while (!SentPackedMessage.empty()) { + PackedMessagesToSend.emplace(std::move(SentPackedMessage.front())); + SentPackedMessage.pop(); + } + ui64 minId = PackedMessagesToSend.empty() ? NextId + 1 : PackedMessagesToSend.top().Offset; + std::queue freshOriginalMessagesToSend; + OriginalMessagesToSend.swap(freshOriginalMessagesToSend); + while (!SentOriginalMessages.empty()) { + OriginalMessagesToSend.emplace(std::move(SentOriginalMessages.front())); + SentOriginalMessages.pop(); + } + while (!freshOriginalMessagesToSend.empty()) { + OriginalMessagesToSend.emplace(std::move(freshOriginalMessagesToSend.front())); + freshOriginalMessagesToSend.pop(); + } + if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().Id < minId) + minId = OriginalMessagesToSend.front().Id; + MinUnsentId = minId; + Y_ABORT_UNLESS(PackedMessagesToSend.size() == totalPackedMessages); + Y_ABORT_UNLESS(OriginalMessagesToSend.size() == totalOriginalMessages); +} + +// Called from client Write() methods +void TWriteSessionImpl::FlushWriteIfRequiredImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) { + MessagesAcquired += static_cast(CurrentBatch.Acquire()); + if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.value_or(TDuration::Zero()) + || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.value_or(0) + || CurrentBatch.CurrentSize >= MaxBlockSize + || CurrentBatch.Messages.size() >= MaxBlockMessageCount + || CurrentBatch.HasCodec() + ) { + WriteBatchImpl(); + return; + } + } +} + +size_t TWriteSessionImpl::WriteBatchImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Write " << CurrentBatch.Messages.size() << " messages with Id from " + << CurrentBatch.Messages.begin()->Id << " to " << CurrentBatch.Messages.back().Id + ); + + Y_ABORT_UNLESS(CurrentBatch.Messages.size() <= MaxBlockMessageCount); + + const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); + if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { + MessagesAcquired += static_cast(CurrentBatch.Acquire()); + } + + size_t size = 0; + for (size_t i = 0; i != CurrentBatch.Messages.size();) { + TBlock block{}; + for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) { + auto id = CurrentBatch.Messages[i].Id; + auto createTs = CurrentBatch.Messages[i].CreatedAt; + + if (!block.MessageCount) { + block.Offset = id; + } + + block.MessageCount += 1; + const auto& datum = CurrentBatch.Messages[i].DataRef; + block.OriginalSize += datum.size(); + block.OriginalMemoryUsage = CurrentBatch.Data.size(); + block.OriginalDataRefs.emplace_back(datum); + if (CurrentBatch.Messages[i].Codec.has_value()) { + Y_ABORT_UNLESS(CurrentBatch.Messages.size() == 1); + block.CodecID = GetCodecId(*CurrentBatch.Messages[i].Codec); + block.OriginalSize = CurrentBatch.Messages[i].OriginalSize; + block.Compressed = false; + } + size += datum.size(); + UpdateTimedCountersImpl(); + (*Counters->BytesInflightUncompressed) += datum.size(); + (*Counters->MessagesInflight)++; + OriginalMessagesToSend.emplace(id, createTs, datum.size()); + } + block.Data = std::move(CurrentBatch.Data); + if (skipCompression) { + PackedMessagesToSend.emplace(std::move(block)); + } else { + CompressImpl(std::move(block)); + } + } + CurrentBatch.Reset(); + if (skipCompression) { + SendImpl(); + } + return size; +} + +size_t GetMaxGrpcMessageSize() { + return 120_MB; +} + +bool TWriteSessionImpl::IsReadyToSendNextImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!SessionEstablished) { + return false; + } + if (Aborting) + return false; + if (PackedMessagesToSend.empty()) { + return false; + } + Y_ABORT_UNLESS(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages"); + if (OriginalMessagesToSend.front().Id > PackedMessagesToSend.top().Offset) { + + std::cerr << " State before restart was:\n" << StateStr << "\n\n"; + DumpState(); + std::cerr << " State after restart is:\n" << StateStr << "\n\n"; + Y_ABORT("Lost original message(s)"); + } + + return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().Id; +} + +void TWriteSessionImpl::DumpState() { + TStringBuilder s; + s << "STATE:\n"; + + auto omts = OriginalMessagesToSend; + s << "OriginalMessagesToSend(" << omts.size() << "):"; + ui32 i = 20; + while(!omts.empty() && i-- > 0) { + s << " " << omts.front().Id; + omts.pop(); + } + if (!omts.empty()) s << " ..."; + s << "\n"; + + s << "SentOriginalMessages(" << SentOriginalMessages.size() << "):"; + omts = SentOriginalMessages; + i = 20; + while(!omts.empty() && i-- > 0) { + s << " " << omts.front().Id; + omts.pop(); + } + if (omts.size() > 20) { + s << " ..."; + for (ui32 skip = omts.size() <= 20 ? 0 : (omts.size() - 20); skip > 0; --skip) { + omts.pop(); + } + } + while(!omts.empty()) { + s << " " << omts.front().Id; + omts.pop(); + } + s << "\n"; + s << "PackedMessagesToSend(" << PackedMessagesToSend.size() << "):"; + i = 20; + std::vector tmpPackedMessagesToSend; + while (!PackedMessagesToSend.empty() && i-- > 0) { + s << " (" << PackedMessagesToSend.top().Offset << ", " << PackedMessagesToSend.top().MessageCount << ")"; + TBlock block; + block.Move(PackedMessagesToSend.top()); + PackedMessagesToSend.pop(); + tmpPackedMessagesToSend.emplace_back(std::move(block)); + } + if (PackedMessagesToSend.size() > 0) s << " ..."; + s << "\n"; + for (auto it = tmpPackedMessagesToSend.begin(); it != tmpPackedMessagesToSend.end(); ++it) { + PackedMessagesToSend.push(std::move(*it)); + } + tmpPackedMessagesToSend.clear(); + + auto spm = std::move(SentPackedMessage); + s << "SentPackedMessages(" << spm.size() << "):"; + while(!spm.empty()) { + s << " (" << spm.front().Offset << ", " << spm.front().MessageCount << ")"; + SentPackedMessage.push(std::move(spm.front())); + spm.pop(); + } + s << "\n"; + + StateStr = s; +} + + +void TWriteSessionImpl::UpdateTokenIfNeededImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: try to update token"); + + if (!DbDriverState->CredentialsProvider || UpdateTokenInProgress || !SessionEstablished) + return; + TClientMessage clientMessage; + auto* updateRequest = clientMessage.mutable_update_token_request(); + auto token = DbDriverState->CredentialsProvider->GetAuthInfo(); + if (token == PrevToken) + return; + UpdateTokenInProgress = true; + updateRequest->set_token(TStringType{token}); + PrevToken = token; + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: updating token"); + + Processor->Write(std::move(clientMessage)); +} + +void TWriteSessionImpl::SendImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB + while(IsReadyToSendNextImpl()) { + TClientMessage clientMessage; + auto* writeRequest = clientMessage.mutable_write_request(); + auto sentAtMs = TInstant::Now().MilliSeconds(); + + // Sent blocks while we can without messages reordering + while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) { + const auto& block = PackedMessagesToSend.top(); + Y_ABORT_UNLESS(block.Valid); + for (size_t i = 0; i != block.MessageCount; ++i) { + Y_ABORT_UNLESS(!OriginalMessagesToSend.empty()); + + auto& message = OriginalMessagesToSend.front(); + + writeRequest->add_sent_at_ms(sentAtMs); + writeRequest->add_sequence_numbers(GetSeqNoImpl(message.Id)); + writeRequest->add_message_sizes(message.Size); + writeRequest->add_created_at_ms(message.CreatedAt.MilliSeconds()); + + SentOriginalMessages.emplace(std::move(message)); + OriginalMessagesToSend.pop(); + } + + writeRequest->add_blocks_offsets(block.Offset); + writeRequest->add_blocks_message_counts(block.MessageCount); + writeRequest->add_blocks_part_numbers(block.PartNumber); + writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize); + writeRequest->add_blocks_headers(TStringType{block.CodecID}); + if (block.Compressed) + writeRequest->add_blocks_data(block.Data.data(), block.Data.size()); + else { + for (auto& buffer: block.OriginalDataRefs) { + writeRequest->add_blocks_data(buffer.data(), buffer.size()); + } + } + + TBlock moveBlock; + moveBlock.Move(block); + SentPackedMessage.emplace(std::move(moveBlock)); + PackedMessagesToSend.pop(); + } + UpdateTokenIfNeededImpl(); + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Send " << writeRequest->sequence_numbers_size() << " message(s) (" + << OriginalMessagesToSend.size() << " left), first sequence number is " + << writeRequest->sequence_numbers(0) + ); + Processor->Write(std::move(clientMessage)); + } +} + +// Client method, no Lock +bool TWriteSessionImpl::Close(TDuration closeTimeout) { + if (Aborting.load()) + return false; + LOG_LAZY(DbDriverState->Log, + TLOG_INFO, + LogPrefix() << "Write session: close. Timeout = " << closeTimeout.MilliSeconds() << " ms" + ); + auto startTime = TInstant::Now(); + auto remaining = closeTimeout; + bool ready = false; + bool needSetSeqNoValue = false; + while (remaining > TDuration::Zero()) { + { + std::lock_guard guard(Lock); + if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) { + ready = true; + } + if (Aborting.load()) + break; + } + if (ready) { + break; + } + remaining = closeTimeout - (TInstant::Now() - startTime); + Sleep(Min(TDuration::MilliSeconds(100), remaining)); + } + { + std::lock_guard guard(Lock); + ready = (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) && !Aborting.load(); + } + { + std::lock_guard guard(Lock); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); + needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + if (needSetSeqNoValue) { + InitSeqNoPromise.SetException("session closed"); + } + if (ready) { + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session: gracefully shut down, all writes complete"); + } else { + LOG_LAZY(DbDriverState->Log, + TLOG_WARNING, + LogPrefix() << "Write session: could not confirm all writes in time" + << " or session aborted, perform hard shutdown" + ); + } + return ready; +} + +void TWriteSessionImpl::HandleWakeUpImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + FlushWriteIfRequiredImpl(); + if (Aborting.load()) { + return; + } + auto callback = [cbContext = SelfContext] (bool ok) + { + if (!ok) { + return; + } + + if (auto self = cbContext->LockShared()) { + std::lock_guard guard(self->Lock); + self->HandleWakeUpImpl(); + } + }; + + if (TInstant::Now() - LastTokenUpdate > UPDATE_TOKEN_PERIOD) { + LastTokenUpdate = TInstant::Now(); + UpdateTokenIfNeededImpl(); + } + + const auto flushAfter = CurrentBatch.StartedAt == TInstant::Zero() + ? WakeupInterval + : WakeupInterval - Min(Now() - CurrentBatch.StartedAt, WakeupInterval); + Connections->ScheduleCallback(flushAfter, std::move(callback)); +} + +void TWriteSessionImpl::UpdateTimedCountersImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto now = TInstant::Now(); + auto delta = (now - LastCountersUpdateTs).MilliSeconds(); + double percent = 100.0 / Settings.MaxMemoryUsage_; + + Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta); + Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta); + Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta); + + *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds(); + LastCountersUpdateTs = now; + if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) { + LastCountersLogTs = TInstant::Now(); + +#define LOG_COUNTER(counter) \ + << " " Y_STRINGIZE(counter) ": " \ + << Counters->counter->Val() \ + /**/ + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() + << "Counters: {" + LOG_COUNTER(Errors) + LOG_COUNTER(CurrentSessionLifetimeMs) + LOG_COUNTER(BytesWritten) + LOG_COUNTER(MessagesWritten) + LOG_COUNTER(BytesWrittenCompressed) + LOG_COUNTER(BytesInflightUncompressed) + LOG_COUNTER(BytesInflightCompressed) + LOG_COUNTER(BytesInflightTotal) + LOG_COUNTER(MessagesInflight) + << " }" + ); + +#undef LOG_COUNTER + } +} + +void TWriteSessionImpl::AbortImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Aborting.load()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: aborting"); + Aborting.store(1); + Cancel(ConnectContext); + Cancel(ConnectTimeoutContext); + Cancel(ConnectDelayContext); + if (Processor) + Processor->Cancel(); + + Cancel(ClientContext); + ClientContext.reset(); // removes context from contexts set from underlying gRPC-client. + } +} + +void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); + EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues))); + AbortImpl(); +} + +void TWriteSessionImpl::CloseImpl(EStatus statusCode, const std::string& message) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + CloseImpl(statusCode, std::move(issues)); +} + +void TWriteSessionImpl::CloseImpl(TPlainStatus&& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); + EventsQueue->Close(TSessionClosedEvent(std::move(status))); + AbortImpl(); +} + +TWriteSessionImpl::~TWriteSessionImpl() { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: destroy"); + bool needClose = false; + { + std::lock_guard guard(Lock); + if (!Aborting.load()) { + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); + + needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + } + if (needClose) { + InitSeqNoPromise.SetException("session closed"); + } +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h new file mode 100644 index 000000000000..483570c944cf --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h @@ -0,0 +1,445 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSessionEventsQueue + +class TWriteSessionEventsQueue: public TBaseSessionEventsQueue { + using TParent = TBaseSessionEventsQueue; + +public: + TWriteSessionEventsQueue(const TWriteSessionSettings& settings) + : TParent(settings) + {} + + void PushEvent(TEventInfo eventInfo) { + if (Closed || ApplyHandler(eventInfo)) { + return; + } + + TWaiter waiter; + { + std::lock_guard guard(Mutex); + Events.emplace(std::move(eventInfo)); + waiter = PopWaiterImpl(); + } + waiter.Signal(); // Does nothing if waiter is empty. + } + + std::optional GetEvent(bool block = false) { + std::optional eventInfo; + { + std::lock_guard guard(Mutex); + if (block) { + WaitEventsImpl(); + } + if (HasEventsImpl()) { + eventInfo = GetEventImpl(); + } else { + return std::nullopt; + } + } + eventInfo->OnUserRetrievedEvent(); + return std::move(eventInfo->Event); + } + + std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt) { + std::vector eventInfos; + { + std::lock_guard guard(Mutex); + if (block) { + WaitEventsImpl(); + } + eventInfos.reserve(Min(Events.size() + CloseEvent.has_value(), maxEventsCount.value_or(std::numeric_limits::max()))); + while (!Events.empty()) { + eventInfos.emplace_back(GetEventImpl()); + if (maxEventsCount.has_value() && eventInfos.size() >= *maxEventsCount) { + break; + } + } + if (CloseEvent && Events.empty() && (!maxEventsCount.has_value() || eventInfos.size() < *maxEventsCount)) { + eventInfos.push_back({*CloseEvent}); + } + } + + std::vector result; + result.reserve(eventInfos.size()); + for (TEventInfo& eventInfo : eventInfos) { + eventInfo.OnUserRetrievedEvent(); + result.emplace_back(std::move(eventInfo.Event)); + } + return result; + } + + void Close(const TSessionClosedEvent& event) { + TWaiter waiter; + { + std::lock_guard guard(Mutex); + CloseEvent = event; + Closed = true; + waiter = TWaiter(Waiter.ExtractPromise(), this); + } + + TEventInfo info(event); + ApplyHandler(info); + + waiter.Signal(); + } + +private: + struct THandlersVisitor : public TParent::TBaseHandlersVisitor { + using TParent::TBaseHandlersVisitor::TBaseHandlersVisitor; + +#define DECLARE_HANDLER(type, handler, answer) \ + bool operator()(type&) { \ + if (this->PushHandler( \ + std::move(TParent::TBaseHandlersVisitor::Event), \ + this->Settings.EventHandlers_.handler, \ + this->Settings.EventHandlers_.CommonHandler_)) { \ + return answer; \ + } \ + return false; \ + } \ + /**/ + + DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true); + DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHandler_, true); + DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied + +#undef DECLARE_HANDLER + + bool Visit() { + return std::visit(*this, Event); + } + }; + + bool ApplyHandler(TEventInfo& eventInfo) { + THandlersVisitor visitor(Settings, eventInfo.GetEvent()); + return visitor.Visit(); + } + + TEventInfo GetEventImpl() { // Assumes that we're under lock and that the event queue has events. + Y_ASSERT(HasEventsImpl()); + if (!Events.empty()) { + TEventInfo event = std::move(Events.front()); + Events.pop(); + RenewWaiterImpl(); + return event; + } + Y_ASSERT(CloseEvent); + return {*CloseEvent}; + } +}; + +struct TMemoryUsageChange { + bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update + bool NowOk; //!< Same, only after update +}; + +namespace NTests { + class TSimpleWriteSessionTestAdapter; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSessionImpl + +class TWriteSessionImpl : public TContinuationTokenIssuer, + public TEnableSelfContext { +private: + friend class TWriteSession; + friend class TSimpleBlockingWriteSession; + friend class NTests::TSimpleWriteSessionTestAdapter; + +private: + using TClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage; + using TServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage; + using IWriteSessionConnectionProcessorFactory = + TPersQueueClient::TImpl::IWriteSessionConnectionProcessorFactory; + using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor; + + struct TMessage { + ui64 Id; + TInstant CreatedAt; + std::string_view DataRef; + std::optional Codec; + ui32 OriginalSize; // only for coded messages + TMessage(ui64 id, const TInstant& createdAt, std::string_view data, std::optional codec = {}, ui32 originalSize = 0) + : Id(id) + , CreatedAt(createdAt) + , DataRef(data) + , Codec(codec) + , OriginalSize(originalSize) + {} + }; + + struct TMessageBatch { + TBuffer Data; + std::vector Messages; + ui64 CurrentSize = 0; + TInstant StartedAt = TInstant::Zero(); + bool Acquired = false; + bool FlushRequested = false; + void Add(ui64 id, const TInstant& createdAt, std::string_view data, std::optional codec, ui32 originalSize) { + if (StartedAt == TInstant::Zero()) + StartedAt = TInstant::Now(); + CurrentSize += codec ? originalSize : data.size(); + Messages.emplace_back(id, createdAt, data, codec, originalSize); + Acquired = false; + } + + bool HasCodec() const { + return Messages.empty() ? false : Messages.front().Codec.has_value(); + } + + bool Acquire() { + if (Acquired || Messages.empty()) + return false; + auto currSize = Data.size(); + Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size()); + Messages.back().DataRef = std::string_view(Data.data() + currSize, Data.size() - currSize); + Acquired = true; + return true; + } + + bool Empty() const noexcept { + return CurrentSize == 0 && Messages.empty(); + } + + void Reset() { + StartedAt = TInstant::Zero(); + Messages.clear(); + Data.Clear(); + Acquired = false; + CurrentSize = 0; + FlushRequested = false; + } + }; + + struct TBlock { + size_t Offset = 0; //!< First message sequence number in the block + size_t MessageCount = 0; + size_t PartNumber = 0; + size_t OriginalSize = 0; + size_t OriginalMemoryUsage = 0; + std::string CodecID = GetCodecId(ECodec::RAW); + mutable std::vector OriginalDataRefs; + mutable TBuffer Data; + bool Compressed = false; + mutable bool Valid = true; + + TBlock& operator=(TBlock&&) = default; + TBlock(TBlock&&) = default; + TBlock() = default; + + //For taking ownership by copying from const object, f.e. lambda -> std::function, priority_queue + void Move(const TBlock& rhs) { + Offset = rhs.Offset; + MessageCount = rhs.MessageCount; + PartNumber = rhs.PartNumber; + OriginalSize = rhs.OriginalSize; + OriginalMemoryUsage = rhs.OriginalMemoryUsage; + CodecID = rhs.CodecID; + OriginalDataRefs.swap(rhs.OriginalDataRefs); + Data.Swap(rhs.Data); + Compressed = rhs.Compressed; + + rhs.Data.Clear(); + rhs.OriginalDataRefs.clear(); + } + }; + + struct TOriginalMessage { + ui64 Id; + TInstant CreatedAt; + size_t Size; + TOriginalMessage(const ui64 id, const TInstant createdAt, const size_t size) + : Id(id) + , CreatedAt(createdAt) + , Size(size) + {} + }; + + //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue + struct Greater { + bool operator() (const TBlock& lhs, const TBlock& rhs) { + return lhs.Offset > rhs.Offset; + } + }; + + struct THandleResult { + bool DoRestart = false; + TDuration StartDelay = TDuration::Zero(); + bool DoStop = false; + bool DoSetSeqNo = false; + }; + struct TProcessSrvMessageResult { + THandleResult HandleResult; + std::optional InitSeqNo; + std::vector Events; + bool Ok = true; + }; + + THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action + +public: + TWriteSessionImpl(const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + std::optional GetEvent(bool block = false); + std::vector GetEvents(bool block = false, + std::optional maxEventsCount = std::nullopt); + NThreading::TFuture GetInitSeqNo(); + + void Write(TContinuationToken&& continuationToken, std::string_view data, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); + + void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, ui32 originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); + + + NThreading::TFuture WaitEvent(); + + // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. + bool Close(TDuration closeTimeout = TDuration::Max()); + + TWriterCounters::TPtr GetCounters() {Y_ABORT("Unimplemented"); } //ToDo - unimplemented; + + const TWriteSessionSettings& GetSettings() const { + return Settings; + } + + ~TWriteSessionImpl(); // will not call close - destroy everything without acks + +private: + + TStringBuilder LogPrefix() const; + + void UpdateTokenIfNeededImpl(); + + void WriteInternal(TContinuationToken&& continuationToken, std::string_view data, std::optional codec, ui32 originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); + + void FlushWriteIfRequiredImpl(); + size_t WriteBatchImpl(); + void Start(const TDuration& delay); + void InitWriter(); + + void DoCdsRequest(TDuration delay = TDuration::Zero()); + void OnCdsResponse(TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result); + void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor, + const NYdbGrpc::IQueueClientContextPtr& connectContext); + void OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext); + void ResetForRetryImpl(); + THandleResult RestartImpl(const TPlainStatus& status); + void DoConnect(const TDuration& delay, const std::string& endpoint); + void InitImpl(); + void ReadFromProcessor(); // Assumes that we're under lock. + void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock. + void OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration); + void OnWriteDone(NYdbGrpc::TGrpcStatus&& status, size_t connectionGeneration); + TProcessSrvMessageResult ProcessServerMessageImpl(); + TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff); + TBuffer CompressBufferImpl(std::vector& data, ECodec codec, i32 level); + void CompressImpl(TBlock&& block); + void OnCompressed(TBlock&& block, bool isSyncCompression=false); + TMemoryUsageChange OnCompressedImpl(TBlock&& block); + + //std::string GetDebugIdentity() const; + Ydb::PersQueue::V1::StreamingWriteClientMessage GetInitClientMessage(); + bool CleanupOnAcknowledged(ui64 id); + bool IsReadyToSendNextImpl(); + void DumpState(); + ui64 GetNextIdImpl(const std::optional& seqNo); + ui64 GetSeqNoImpl(ui64 id); + ui64 GetIdImpl(ui64 seqNo); + void SendImpl(); + void AbortImpl(); + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); + void CloseImpl(EStatus statusCode, const std::string& message); + void CloseImpl(TPlainStatus&& status); + + void OnErrorResolved() { + RetryState = nullptr; + } + void CheckHandleResultImpl(THandleResult& result); + void ProcessHandleResult(THandleResult& result); + void HandleWakeUpImpl(); + void UpdateTimedCountersImpl(); + +private: + TWriteSessionSettings Settings; + std::shared_ptr Client; + std::shared_ptr Connections; + std::string TargetCluster; + std::string InitialCluster; + std::string CurrentCluster; + std::string PreferredClusterByCDS; + std::shared_ptr ConnectionFactory; + TDbDriverStatePtr DbDriverState; + std::string PrevToken; + bool UpdateTokenInProgress = false; + TInstant LastTokenUpdate = TInstant::Zero(); + std::shared_ptr EventsQueue; + NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. + NYdbGrpc::IQueueClientContextPtr ConnectContext; + NYdbGrpc::IQueueClientContextPtr ConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr ConnectDelayContext; + size_t ConnectionGeneration = 0; + size_t ConnectionAttemptsDone = 0; + TAdaptiveLock Lock; + IProcessor::TPtr Processor; + IRetryPolicy::IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). + std::shared_ptr ServerMessage; // Server message to write server response to. + + std::string SessionId; + IExecutor::TPtr Executor; + IExecutor::TPtr CompressionExecutor; + size_t MemoryUsage = 0; //!< Estimated amount of memory used + bool FirstTokenSent = false; + + TMessageBatch CurrentBatch; + + std::queue OriginalMessagesToSend; + std::priority_queue, Greater> PackedMessagesToSend; + //! Messages that are sent but yet not acknowledged + std::queue SentOriginalMessages; + std::queue SentPackedMessage; + + const size_t MaxBlockSize = std::numeric_limits::max(); + const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility + bool Connected = false; + bool Started = false; + std::atomic Aborting = 0; + bool SessionEstablished = false; + ui32 PartitionId = 0; + ui64 NextId = 0; + ui64 MinUnsentId = 1; + std::map InitSeqNo; + std::optional AutoSeqNoMode; + bool ValidateSeqNoMode = false; + + NThreading::TPromise InitSeqNoPromise; + bool InitSeqNoSetDone = false; + TInstant SessionStartedTs; + TInstant LastCountersUpdateTs = TInstant::Zero(); + TInstant LastCountersLogTs; + TWriterCounters::TPtr Counters; + TDuration WakeupInterval; + + std::string StateStr; + +protected: + ui64 MessagesAcquired = 0; +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/ya.make new file mode 100644 index 000000000000..204122207d04 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/ya.make @@ -0,0 +1,38 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + aliases.h + common.h + common.cpp + persqueue_impl.h + persqueue_impl.cpp + persqueue.cpp + read_session.h + read_session.cpp + read_session_messages.cpp + write_session_impl.h + write_session_impl.cpp + write_session.h + write_session.cpp +) + +PEERDIR( + library/cpp/monlib/dynamic_counters + library/cpp/monlib/metrics + library/cpp/string_utils/url + library/cpp/containers/disjoint_interval_tree + ydb/public/sdk/cpp/src/library/grpc/client + ydb/public/sdk/cpp/src/library/persqueue/obfuscate + ydb/public/api/grpc/draft + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/topic/common + ydb/public/sdk/cpp/src/client/topic/impl + +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/aliases.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/aliases.h new file mode 100644 index 000000000000..0cd8c0c83549 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/aliases.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace NYdb::inline V3::NPersQueue { + +// codecs +using NTopic::ECodec; +using NTopic::ICodec; +using NTopic::TCodecMap; +using NTopic::TGzipCodec; +using NTopic::TZstdCodec; +using NTopic::TUnsupportedCodec; + +// counters +using NTopic::TCounterPtr; +using NTopic::TReaderCounters; +using NTopic::TWriterCounters; +using NTopic::MakeCountersNotNull; +using NTopic::HasNullCounters; + +// errors +// using NTopic::GetRetryErrorClass; +// using NTopic::GetRetryErrorClassV2; + +// common events +using NTopic::TWriteSessionMeta; +using NTopic::TMessageMeta; +using NTopic::TSessionClosedEvent; +using NTopic::TSessionClosedHandler; +// TODO reuse TPrintable + +// executor +using NTopic::IExecutor; +using NTopic::CreateThreadPoolExecutorAdapter; +using NTopic::CreateThreadPoolExecutor; +using NTopic::CreateSyncExecutor; + +// retry policy +using NTopic::IRetryPolicy; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/client.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/client.h new file mode 100644 index 000000000000..71bfd09507d5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/client.h @@ -0,0 +1,70 @@ +#pragma once + +#include "control_plane.h" +#include "read_session.h" +#include "write_session.h" + +#include +#include + +namespace NYdb::inline V3 { + class TProtoAccessor; +} + +namespace NYdb::inline V3::NPersQueue { + +struct TPersQueueClientSettings : public TCommonClientSettingsBase { + using TSelf = TPersQueueClientSettings; + + //! Default executor for compression tasks. + FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultCompressionExecutor, CreateThreadPoolExecutor(2)); + + //! Default executor for callbacks. + FLUENT_SETTING_DEFAULT(IExecutor::TPtr, DefaultHandlersExecutor, CreateThreadPoolExecutor(1)); + + //! Manages cluster discovery mode. + FLUENT_SETTING_DEFAULT(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); +}; + +// PersQueue client. +class TPersQueueClient { +public: + class TImpl; + + TPersQueueClient(const TDriver& driver, const TPersQueueClientSettings& settings = TPersQueueClientSettings()); + + void ProvideCodec(ECodec codecId, std::unique_ptr&& codecImpl); + + // Create a new topic. + TAsyncStatus CreateTopic(const std::string& path, const TCreateTopicSettings& = {}); + + // Update a topic. + TAsyncStatus AlterTopic(const std::string& path, const TAlterTopicSettings& = {}); + + // Delete a topic. + TAsyncStatus DropTopic(const std::string& path, const TDropTopicSettings& = {}); + + // Add topic read rule + TAsyncStatus AddReadRule(const std::string& path, const TAddReadRuleSettings& = {}); + + // Remove topic read rule + TAsyncStatus RemoveReadRule(const std::string& path, const TRemoveReadRuleSettings& = {}); + + // Describe settings of topic. + TAsyncDescribeTopicResult DescribeTopic(const std::string& path, const TDescribeTopicSettings& = {}); + + //! Create read session. + std::shared_ptr CreateReadSession(const TReadSessionSettings& settings); + + //! Create write session. + std::shared_ptr CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TWriteSessionSettings& settings); + +protected: + void OverrideCodec(ECodec codecId, std::unique_ptr&& codecImpl); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/control_plane.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/control_plane.h new file mode 100644 index 000000000000..dc50e11e4d5e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/control_plane.h @@ -0,0 +1,325 @@ +#pragma once + +#include "aliases.h" + +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + class TProtoAccessor; +} + +namespace NYdb::inline V3::NPersQueue { + +enum class EFormat { + BASE = 1, +}; + +struct TCredentials { + enum class EMode { + NOT_SET = 1, + OAUTH_TOKEN = 2, + JWT_PARAMS = 3, + IAM = 4, + }; + + TCredentials() = default; + TCredentials(const Ydb::PersQueue::V1::Credentials& credentials); + EMode GetMode() const; + std::string GetOauthToken() const; + std::string GetJwtParams() const; + + std::string GetIamServiceAccountKey() const; + std::string GetIamEndpoint() const; + +private: + EMode Mode_; + Ydb::PersQueue::V1::Credentials Credentials_; +}; + + +// Result for describe resource request. +struct TDescribeTopicResult : public TStatus { + friend class NYdb::V3::TProtoAccessor; + + struct TTopicSettings { + TTopicSettings(const Ydb::PersQueue::V1::TopicSettings&); + + #define GETTER(TYPE, NAME) TYPE NAME() const { return NAME##_; } + + struct TReadRule { + TReadRule(const Ydb::PersQueue::V1::TopicSettings::ReadRule&); + + GETTER(std::string, ConsumerName); + GETTER(bool, Important); + GETTER(TInstant, StartingMessageTimestamp); + GETTER(EFormat, SupportedFormat); + const std::vector& SupportedCodecs() const { + return SupportedCodecs_; + } + GETTER(ui32, Version); + GETTER(std::string, ServiceType); + + private: + std::string ConsumerName_; + bool Important_; + TInstant StartingMessageTimestamp_; + EFormat SupportedFormat_; + std::vector SupportedCodecs_; + ui32 Version_; + std::string ServiceType_; + }; + + struct TRemoteMirrorRule { + TRemoteMirrorRule(const Ydb::PersQueue::V1::TopicSettings::RemoteMirrorRule&); + GETTER(std::string, Endpoint); + GETTER(std::string, TopicPath); + GETTER(std::string, ConsumerName); + GETTER(TInstant, StartingMessageTimestamp); + GETTER(TCredentials, Credentials); + GETTER(std::string, Database); + + private: + std::string Endpoint_; + std::string TopicPath_; + std::string ConsumerName_; + TInstant StartingMessageTimestamp_; + TCredentials Credentials_; + std::string Database_; + }; + + GETTER(ui32, PartitionsCount); + GETTER(TDuration, RetentionPeriod); + GETTER(EFormat, SupportedFormat); + const std::vector& SupportedCodecs() const { + return SupportedCodecs_; + } + GETTER(ui64, MaxPartitionStorageSize); + GETTER(ui64, MaxPartitionWriteSpeed); + GETTER(ui64, MaxPartitionWriteBurst); + GETTER(bool, ClientWriteDisabled); + + // attributes + GETTER(bool, AllowUnauthenticatedWrite); + GETTER(bool, AllowUnauthenticatedRead); + GETTER(std::optional, PartitionsPerTablet); + GETTER(std::optional, AbcId); + GETTER(std::optional, AbcSlug); + GETTER(std::optional, FederationAccount); + + const std::vector& ReadRules() const { + return ReadRules_; + } + GETTER(std::optional, RemoteMirrorRule); + + GETTER(std::optional, MaxPartitionsCount); + GETTER(std::optional, StabilizationWindow); + GETTER(std::optional, UpUtilizationPercent); + GETTER(std::optional, DownUtilizationPercent); + GETTER(std::optional, AutoPartitioningStrategy); + + +#undef GETTER + + private: + ui32 PartitionsCount_; + TDuration RetentionPeriod_; + EFormat SupportedFormat_; + std::vector SupportedCodecs_; + ui64 MaxPartitionStorageSize_; + ui64 MaxPartitionWriteSpeed_; + ui64 MaxPartitionWriteBurst_; + bool ClientWriteDisabled_; + std::vector ReadRules_; + std::optional RemoteMirrorRule_; + // attributes + bool AllowUnauthenticatedRead_; + bool AllowUnauthenticatedWrite_; + std::optional PartitionsPerTablet_; + std::optional AbcId_; + std::optional AbcSlug_; + std::string FederationAccount_; + + std::optional MaxPartitionsCount_; + std::optional StabilizationWindow_; + std::optional UpUtilizationPercent_; + std::optional DownUtilizationPercent_; + std::optional AutoPartitioningStrategy_; + }; + + TDescribeTopicResult(TStatus status, const Ydb::PersQueue::V1::DescribeTopicResult& result); + + const TTopicSettings& TopicSettings() const { + return TopicSettings_; + } + +private: + TTopicSettings TopicSettings_; + [[nodiscard]] const Ydb::PersQueue::V1::DescribeTopicResult& GetProto() const { + return Proto_; + } + const Ydb::PersQueue::V1::DescribeTopicResult Proto_; +}; + +using TAsyncDescribeTopicResult = NThreading::TFuture; + + + +const std::vector& GetDefaultCodecs(); + +struct TReadRuleSettings { + TReadRuleSettings() {} + using TSelf = TReadRuleSettings; + FLUENT_SETTING(std::string, ConsumerName); + FLUENT_SETTING_DEFAULT(bool, Important, false); + FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); + FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) + FLUENT_SETTING_DEFAULT(std::vector, SupportedCodecs, GetDefaultCodecs()); + + FLUENT_SETTING_DEFAULT(ui32, Version, 0); + FLUENT_SETTING(std::string, ServiceType); + + TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) { + ConsumerName_ = settings.ConsumerName(); + Important_ = settings.Important(); + StartingMessageTimestamp_ = settings.StartingMessageTimestamp(); + SupportedFormat_ = settings.SupportedFormat(); + SupportedCodecs_.clear(); + for (const auto& codec : settings.SupportedCodecs()) { + SupportedCodecs_.push_back(codec); + } + Version_ = settings.Version(); + ServiceType_ = settings.ServiceType(); + return *this; + } + +}; + +// Settings for topic. +template +struct TTopicSettings : public TOperationRequestSettings { + friend class TPersQueueClient; + + struct TRemoteMirrorRuleSettings { + TRemoteMirrorRuleSettings() {} + using TSelf = TRemoteMirrorRuleSettings; + FLUENT_SETTING(std::string, Endpoint); + FLUENT_SETTING(std::string, TopicPath); + FLUENT_SETTING(std::string, ConsumerName); + FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); + FLUENT_SETTING(TCredentials, Credentials); + FLUENT_SETTING(std::string, Database); + + TRemoteMirrorRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TRemoteMirrorRule& settings) { + Endpoint_ = settings.Endpoint(); + TopicPath_ = settings.TopicPath(); + ConsumerName_ = settings.ConsumerName(); + StartingMessageTimestamp_ = settings.StartingMessageTimestamp(); + Credentials_ = settings.Credentials(); + Database_ = settings.Database(); + return *this; + } + + }; + + using TSelf = TDerived; + + FLUENT_SETTING_DEFAULT(ui32, PartitionsCount, 1); + FLUENT_SETTING_DEFAULT(TDuration, RetentionPeriod, TDuration::Hours(18)); + FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) + FLUENT_SETTING_DEFAULT(std::vector, SupportedCodecs, GetDefaultCodecs()); + + FLUENT_SETTING_DEFAULT(ui64, MaxPartitionStorageSize, 0); + FLUENT_SETTING_DEFAULT(ui64, MaxPartitionWriteSpeed, 2_MB); + FLUENT_SETTING_DEFAULT(ui64, MaxPartitionWriteBurst, 2_MB); + + FLUENT_SETTING_DEFAULT(bool, ClientWriteDisabled, false); + FLUENT_SETTING_DEFAULT(bool, AllowUnauthenticatedWrite, false); + FLUENT_SETTING_DEFAULT(bool, AllowUnauthenticatedRead, false); + + FLUENT_SETTING_OPTIONAL(ui32, PartitionsPerTablet); + + FLUENT_SETTING_OPTIONAL(ui32, AbcId); + FLUENT_SETTING_OPTIONAL(std::string, AbcSlug); + FLUENT_SETTING_OPTIONAL(std::string, FederationAccount); + + //TODO: FLUENT_SETTING_VECTOR + FLUENT_SETTING_DEFAULT(std::vector, ReadRules, {}); + FLUENT_SETTING_OPTIONAL(TRemoteMirrorRuleSettings, RemoteMirrorRule); + + TSelf& SetSettings(const TDescribeTopicResult::TTopicSettings& settings) { + + PartitionsCount_ = settings.PartitionsCount(); + RetentionPeriod_ = settings.RetentionPeriod(); + SupportedFormat_ = settings.SupportedFormat(); + SupportedCodecs_.clear(); + for (const auto& codec : settings.SupportedCodecs()) { + SupportedCodecs_.push_back(codec); + } + MaxPartitionStorageSize_ = settings.MaxPartitionStorageSize(); + MaxPartitionWriteSpeed_ = settings.MaxPartitionWriteSpeed(); + MaxPartitionWriteBurst_ = settings.MaxPartitionWriteBurst(); + ClientWriteDisabled_ = settings.ClientWriteDisabled(); + AllowUnauthenticatedRead_ = settings.AllowUnauthenticatedRead(); + AllowUnauthenticatedWrite_ = settings.AllowUnauthenticatedWrite(); + PartitionsPerTablet_ = settings.PartitionsPerTablet(); + AbcId_ = settings.AbcId(); + AbcSlug_ = settings.AbcSlug(); + FederationAccount_ = settings.FederationAccount(); + ReadRules_.clear(); + for (const auto& readRule : settings.ReadRules()) { + ReadRules_.push_back({}); + ReadRules_.back().SetSettings(readRule); + } + if (settings.RemoteMirrorRule()) { + RemoteMirrorRule_ = TRemoteMirrorRuleSettings().SetSettings(settings.RemoteMirrorRule().value()); + } + + MaxPartitionsCount_ = settings.MaxPartitionsCount(); + StabilizationWindow_ = settings.StabilizationWindow(); + UpUtilizationPercent_ = settings.UpUtilizationPercent(); + DownUtilizationPercent_ = settings.DownUtilizationPercent(); + AutoPartitioningStrategy_ = settings.AutoPartitioningStrategy(); + + return static_cast(*this); + } + +private: + std::optional MaxPartitionsCount_; + std::optional StabilizationWindow_; + std::optional UpUtilizationPercent_; + std::optional DownUtilizationPercent_; + std::optional AutoPartitioningStrategy_; +}; + + +// Settings for create resource request. +struct TCreateTopicSettings : public TTopicSettings { +}; + +// Settings for alter resource request. +struct TAlterTopicSettings : public TTopicSettings { +}; + +// Settings for drop resource request. +struct TDropTopicSettings : public TOperationRequestSettings {}; + +// Settings for describe resource request. +struct TDescribeTopicSettings : public TOperationRequestSettings {}; + +// Settings for add read rule request +struct TAddReadRuleSettings : public TTopicSettings { + FLUENT_SETTING(TReadRuleSettings, ReadRule); +}; + +// Settings for remove read rule request +struct TRemoveReadRuleSettings : public TOperationRequestSettings { + FLUENT_SETTING(std::string, ConsumerName); +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_events.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_events.h new file mode 100644 index 000000000000..7152be325139 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_events.h @@ -0,0 +1,474 @@ +#pragma once + +#include "aliases.h" + +#include + +namespace NYdb::inline V3::NPersQueue { + +//! Partition stream. +struct TPartitionStream : public TThrRefBase { + using TPtr = TIntrusivePtr; + + +public: + + //! Temporary stop receiving data from this partition stream. + // virtual void StopReading() = 0; // Not implemented yet. + + //! Resume receiving data from this partition stream after StopReading() call. + // virtual void ResumeReading() = 0; // Not implemented yet. + + //! Request partition stream status. + //! Result will come to TPartitionStreamStatusEvent. + virtual void RequestStatus() = 0; + + //! + //! Properties. + //! + + //! Unique identifier of partition stream inside session. + //! It is unique inside one read session. + ui64 GetPartitionStreamId() const { + return PartitionStreamId; + } + + //! Topic path. + const std::string& GetTopicPath() const { + return TopicPath; + } + + //! Cluster name. + const std::string& GetCluster() const { + return Cluster; + } + + //! Partition group id. + ui64 GetPartitionGroupId() const { + return PartitionGroupId; + } + + //! Partition id. + ui64 GetPartitionId() const { + return PartitionId; + } + +protected: + ui64 PartitionStreamId; + std::string TopicPath; + std::string Cluster; + ui64 PartitionGroupId; + ui64 PartitionId; +}; + + +//! Events for read session. +struct TReadSessionEvent { + + //! Event with new data. + //! Contains batch of messages from single partition stream. + struct TDataReceivedEvent { + + struct TMessageInformation { + TMessageInformation(ui64 offset, + std::string messageGroupId, + ui64 seqNo, + TInstant createTime, + TInstant writeTime, + std::string ip, + TWriteSessionMeta::TPtr meta, + ui64 uncompressedSize); + ui64 Offset; + std::string MessageGroupId; + ui64 SeqNo; + TInstant CreateTime; + TInstant WriteTime; + std::string Ip; + TWriteSessionMeta::TPtr Meta; + ui64 UncompressedSize; + }; + + class IMessage { + public: + virtual const std::string& GetData() const; + + //! Partition stream. Same as in batch. + const TPartitionStream::TPtr& GetPartitionStream() const; + + const std::string& GetPartitionKey() const; + + const std::string GetExplicitHash() const; + + virtual void Commit() = 0; + + std::string DebugString(bool printData = false) const; + virtual void DebugString(TStringBuilder& ret, bool printData = false) const = 0; + + IMessage(const std::string& data, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash); + + virtual ~IMessage() = default; + protected: + std::string Data; + + TPartitionStream::TPtr PartitionStream; + std::string PartitionKey; + std::string ExplicitHash; + }; + + //! Single message. + struct TMessage : public IMessage { + //! User data. + //! Throws decompressor exception if decompression failed. + const std::string& GetData() const override; + + bool HasException() const; + + //! Message offset. + ui64 GetOffset() const; + + //! Message group id. + const std::string& GetMessageGroupId() const; + + //! Sequence number. + ui64 GetSeqNo() const; + + //! Message creation timestamp. + TInstant GetCreateTime() const; + + //! Message write timestamp. + TInstant GetWriteTime() const; + + //! Ip address of message source host. + const std::string& GetIp() const; + + //! Metainfo. + const TWriteSessionMeta::TPtr& GetMeta() const; + + TMessage(const std::string& data, + std::exception_ptr decompressionException, + const TMessageInformation& information, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash); + + //! Commits single message. + void Commit() override; + + using IMessage::DebugString; + void DebugString(TStringBuilder& ret, bool printData = false) const override; + + private: + std::exception_ptr DecompressionException; + TMessageInformation Information; + }; + + struct TCompressedMessage : public IMessage { + //! Messages count in compressed data + ui64 GetBlocksCount() const; + + //! Message codec + ECodec GetCodec() const; + + //! Message offset. + ui64 GetOffset(ui64 index) const; + + //! Message group id. + const std::string& GetMessageGroupId(ui64 index) const; + + //! Sequence number. + ui64 GetSeqNo(ui64 index) const; + + //! Message creation timestamp. + TInstant GetCreateTime(ui64 index) const; + + //! Message write timestamp. + TInstant GetWriteTime(ui64 index) const; + + //! Ip address of message source host. + const std::string& GetIp(ui64 index) const; + + //! Metainfo. + const TWriteSessionMeta::TPtr& GetMeta(ui64 index) const; + + //! Uncompressed block size. + ui64 GetUncompressedSize(ui64 index) const; + + virtual ~TCompressedMessage() {} + TCompressedMessage(ECodec codec, + const std::string& data, + const std::vector& information, + TPartitionStream::TPtr partitionStream, + const std::string& partitionKey, + const std::string& explicitHash); + + //! Commits all offsets in compressed message. + void Commit() override; + + using IMessage::DebugString; + void DebugString(TStringBuilder& ret, bool printData = false) const override; + + private: + ECodec Codec; + std::vector Information; + }; + + //! Partition stream. + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + bool IsCompressedMessages() const { + return !CompressedMessages.empty(); + } + + size_t GetMessagesCount() const { + return Messages.size() + CompressedMessages.size(); + } + + //! Get messages. + std::vector& GetMessages() { + CheckMessagesFilled(false); + return Messages; + } + + const std::vector& GetMessages() const { + CheckMessagesFilled(false); + return Messages; + } + + //! Get compressed messages. + std::vector& GetCompressedMessages() { + CheckMessagesFilled(true); + return CompressedMessages; + } + + const std::vector& GetCompressedMessages() const { + CheckMessagesFilled(true); + return CompressedMessages; + } + + //! Commits all messages in batch. + void Commit(); + + std::string DebugString(bool printData = false) const; + + TDataReceivedEvent(std::vector messages, + std::vector compressedMessages, + TPartitionStream::TPtr partitionStream); + + private: + void CheckMessagesFilled(bool compressed) const { + Y_ABORT_UNLESS(!Messages.empty() || !CompressedMessages.empty()); + if (compressed && CompressedMessages.empty()) { + ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session"; + } + if (!compressed && Messages.empty()) { + ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session"; + } + } + + private: + std::vector Messages; + std::vector CompressedMessages; + TPartitionStream::TPtr PartitionStream; + std::vector> OffsetRanges; + }; + + //! Acknowledgement for commit request. + struct TCommitAcknowledgementEvent { + //! Partition stream. + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + //! Committed offset. + //! This means that from now the first available + //! message offset in current partition + //! for current consumer is this offset. + //! All messages before are committed and futher never be available. + ui64 GetCommittedOffset() const { + return CommittedOffset; + } + + std::string DebugString() const; + + TCommitAcknowledgementEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset); + + private: + TPartitionStream::TPtr PartitionStream; + ui64 CommittedOffset; + }; + + //! Server request for creating partition stream. + struct TCreatePartitionStreamEvent { + TCreatePartitionStreamEvent(TPartitionStream::TPtr, ui64 committedOffset, ui64 endOffset); + + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + //! Current committed offset in partition stream. + ui64 GetCommittedOffset() const { + return CommittedOffset; + } + + //! Offset of first not existing message in partition stream. + ui64 GetEndOffset() const { + return EndOffset; + } + + //! Confirm partition stream creation. + //! This signals that user is ready to receive data from this partition stream. + //! If maybe is empty then no rewinding + void Confirm(std::optional readOffset = std::nullopt, std::optional commitOffset = std::nullopt); + + std::string DebugString() const; + + private: + TPartitionStream::TPtr PartitionStream; + ui64 CommittedOffset; + ui64 EndOffset; + }; + + //! Server request for destroying partition stream. + //! Server can destroy partition stream gracefully + //! for rebalancing among all topic clients. + struct TDestroyPartitionStreamEvent { + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + //! Last offset of the partition stream that was committed. + ui64 GetCommittedOffset() const { + return CommittedOffset; + } + + //! Confirm partition stream destruction. + //! Confirm has no effect if TPartitionStreamClosedEvent for same partition stream with is received. + void Confirm(); + + std::string DebugString() const; + + TDestroyPartitionStreamEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset); + + private: + TPartitionStream::TPtr PartitionStream; + ui64 CommittedOffset; + }; + + //! Status for partition stream requested via TPartitionStream::RequestStatus() + struct TPartitionStreamStatusEvent { + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + //! Committed offset. + ui64 GetCommittedOffset() const { + return CommittedOffset; + } + + //! Offset of next message (that is not yet read by session). + ui64 GetReadOffset() const { + return ReadOffset; + } + + //! Offset of first not existing message in partition. + ui64 GetEndOffset() const { + return EndOffset; + } + + //! Write watermark. + //! The last written timestamp of message in this partition stream. + TInstant GetWriteWatermark() const { + return WriteWatermark; + } + + std::string DebugString() const; + + TPartitionStreamStatusEvent(TPartitionStream::TPtr partitionStream, ui64 committedOffset, ui64 readOffset, ui64 endOffset, TInstant writeWatermark); + + private: + TPartitionStream::TPtr PartitionStream; + ui64 CommittedOffset = 0; + ui64 ReadOffset = 0; + ui64 EndOffset = 0; + TInstant WriteWatermark; + }; + + //! Event that signals user about + //! partition stream death. + //! This could be after graceful destruction of + //! partition stream or when connection with partition was lost. + struct TPartitionStreamClosedEvent { + enum class EReason { + DestroyConfirmedByUser, + Lost, + ConnectionLost, + }; + + const TPartitionStream::TPtr& GetPartitionStream() const { + return PartitionStream; + } + + EReason GetReason() const { + return Reason; + } + + std::string DebugString() const; + + TPartitionStreamClosedEvent(TPartitionStream::TPtr partitionStream, EReason reason); + + private: + TPartitionStream::TPtr PartitionStream; + EReason Reason; + }; + + using TEvent = std::variant; +}; + +//! Set of offsets to commit. +//! Class that could store offsets in order to commit them later. +//! This class is not thread safe. +class TDeferredCommit { +public: + //! Add message to set. + void Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message); + + //! Add all messages from dataReceivedEvent to set. + void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); + + //! Add offsets range to set. + void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset); + + //! Add offset to set. + void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset); + + //! Commit all added offsets. + void Commit(); + + TDeferredCommit(); + TDeferredCommit(const TDeferredCommit&) = delete; + TDeferredCommit(TDeferredCommit&&); + TDeferredCommit& operator=(const TDeferredCommit&) = delete; + TDeferredCommit& operator=(TDeferredCommit&&); + + ~TDeferredCommit(); + +private: + class TImpl; + std::unique_ptr Impl; +}; + +//! Event debug string. +std::string DebugString(const TReadSessionEvent::TEvent& event); + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_session.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_session.h new file mode 100644 index 000000000000..9d80184840f4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/read_session.h @@ -0,0 +1,262 @@ +#pragma once + +#include "aliases.h" +#include "read_events.h" + +#include +#include + +#include + +#include + + +namespace NYdb::inline V3::NPersQueue { + +//! Read settings for single topic. +struct TTopicReadSettings { + using TSelf = TTopicReadSettings; + + TTopicReadSettings() = default; + TTopicReadSettings(const TTopicReadSettings&) = default; + TTopicReadSettings(TTopicReadSettings&&) = default; + TTopicReadSettings(const std::string& path) { + Path(path); + } + + TTopicReadSettings& operator=(const TTopicReadSettings&) = default; + TTopicReadSettings& operator=(TTopicReadSettings&&) = default; + + //! Path of topic to read. + FLUENT_SETTING(std::string, Path); + + //! Start reading from this timestamp. + FLUENT_SETTING_OPTIONAL(TInstant, StartingMessageTimestamp); + + //! Partition groups to read. + //! 1-based. + FLUENT_SETTING_VECTOR(ui64, PartitionGroupIds); +}; + +//! Settings for read session. +struct TReadSessionSettings : public TRequestSettings { + using TSelf = TReadSessionSettings; + + struct TEventHandlers { + using TSelf = TEventHandlers; + + //! Set simple handler with data processing and also + //! set other handlers with default behaviour. + //! They automatically commit data after processing + //! and confirm partition stream events. + //! + //! Sets the following handlers: + //! DataReceivedHandler: sets DataReceivedHandler to handler that calls dataHandler and (if commitDataAfterProcessing is set) then calls Commit(). + //! CommitAcknowledgementHandler to handler that does nothing. + //! CreatePartitionStreamHandler to handler that confirms event. + //! DestroyPartitionStreamHandler to handler that confirms event. + //! PartitionStreamStatusHandler to handler that does nothing. + //! PartitionStreamClosedHandler to handler that does nothing. + //! + //! dataHandler: handler of data event. + //! commitDataAfterProcessing: automatically commit data after calling of dataHandler. + //! gracefulReleaseAfterCommit: wait for commit acknowledgements for all inflight data before confirming partition stream destroy. + TSelf& SimpleDataHandlers(std::function dataHandler, bool commitDataAfterProcessing = false, bool gracefulReleaseAfterCommit = true); + + //! Data size limit for the DataReceivedHandler handler. + //! The data size may exceed this limit. + FLUENT_SETTING_DEFAULT(size_t, MaxMessagesBytes, Max()); + + //! Function to handle data events. + //! If this handler is set, data events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, DataReceivedHandler); + + //! Function to handle commit ack events. + //! If this handler is set, commit ack events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, CommitAcknowledgementHandler); + + //! Function to handle create partition stream events. + //! If this handler is set, create partition stream events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, CreatePartitionStreamHandler); + + //! Function to handle destroy partition stream events. + //! If this handler is set, destroy partition stream events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, DestroyPartitionStreamHandler); + + //! Function to handle partition stream status events. + //! If this handler is set, partition stream status events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, PartitionStreamStatusHandler); + + //! Function to handle partition stream closed events. + //! If this handler is set, partition stream closed events will be handled by handler, + //! otherwise sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(std::function, PartitionStreamClosedHandler); + + //! Function to handle session closed events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TReadSession::GetEvent(). + //! Default value is empty function (not set). + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + + //! Function to handle all event types. + //! If event with current type has no handler for this type of event, + //! this handler (if specified) will be used. + //! If this handler is not specified, event can be received with TReadSession::GetEvent() method. + FLUENT_SETTING(std::function, CommonHandler); + + //! Executor for handlers. + //! If not set, default single threaded executor will be used. + FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + }; + + //! Consumer. + FLUENT_SETTING(std::string, ConsumerName); + + //! Topics. + FLUENT_SETTING_VECTOR(TTopicReadSettings, Topics); + + //! Default variant. + //! Read topic instance specified in "Topics" from all clusters. + TSelf& ReadAll() { + Clusters_.clear(); + return ReadOnlyOriginal(true); + } + + //! Read original topic instances specified in "Topics" from several clusters. + TSelf& ReadOriginal(std::vector clusters) { + Clusters_ = std::move(clusters); + return ReadOnlyOriginal(true); + } + + //! Read mirrored topics specified in "Topics" from one cluster. + TSelf& ReadMirrored(const std::string& cluster) { + Clusters_ = { cluster }; + return ReadOnlyOriginal(false); + } + + //! Disable Clusters discovery. ReadMirrored/ReadOriginal/ReadAll will not have any effect + //! if this option is true. + FLUENT_SETTING_DEFAULT(bool, DisableClusterDiscovery, false); + + //! Maximum memory usage for read session. + FLUENT_SETTING_DEFAULT(size_t, MaxMemoryUsageBytes, 100_MB); + + //! Max message time lag. All messages older that now - MaxTimeLag will be ignored. + FLUENT_SETTING_OPTIONAL(TDuration, MaxTimeLag); + + //! Start reading from this timestamp. + FLUENT_SETTING_OPTIONAL(TInstant, StartingMessageTimestamp); + + //! Policy for reconnections. + //! IRetryPolicy::GetDefaultPolicy() if null (not set). + FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + + //! Event handlers. + //! See description in TEventHandlers class. + FLUENT_SETTING(TEventHandlers, EventHandlers); + + //! Decompress messages + FLUENT_SETTING_DEFAULT(bool, Decompress, true); + + //! Executor for decompression tasks. + //! If not set, default executor will be used. + FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor); + + //! Counters. + //! If counters are not provided explicitly, + //! they will be created inside session (without link with parent counters). + FLUENT_SETTING(TReaderCounters::TPtr, Counters); + + //! Read only original topic instance, don't read mirrored. + //! + //! It's better to control this setting via ReadAll()/ReadMirrored()/ReadOriginal() helpers. + FLUENT_SETTING_DEFAULT(bool, ReadOnlyOriginal, true); + + //! Read topics from specified clusters. + //! + //! It's better to control this setting via ReadAll()/ReadMirrored()/ReadOriginal() helpers. + //! + //! 1. If ReadOnlyOriginal is true and Clusters are empty read will be done from all topic instances from all clusters. + //! Use ReadAll() function for this variant. + //! 2. If ReadOnlyOriginal is true and Clusters are not empty read will be done from specified clusters. + //! Use ReadOriginal() function for this variant. + //! 3. If ReadOnlyOriginal is false and one cluster is specified read will be done from all topic instances (mirrored and original) in one cluster. + //! Use ReadMirrored() function for this variant. + FLUENT_SETTING_VECTOR(std::string, Clusters); + + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + + //! Experimental option + FLUENT_SETTING_OPTIONAL(bool, RangesMode); + + //! Log. + FLUENT_SETTING_OPTIONAL(TLog, Log); +}; + +class IReadSession { +public: + //! Main reader loop. + //! Wait for next reader event. + virtual NThreading::TFuture WaitEvent() = 0; + + //! Main reader loop. + //! Get read session events. + //! Blocks until event occurs if "block" is set. + //! + //! maxEventsCount: maximum events count in batch. + //! maxByteSize: total size limit of data messages in batch. + //! block: block until event occurs. + //! + //! If maxEventsCount is not specified, + //! read session chooses event batch size automatically. + virtual std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt, size_t maxByteSize = std::numeric_limits::max()) = 0; + + //! Get single event. + virtual std::optional GetEvent(bool block = false, size_t maxByteSize = std::numeric_limits::max()) = 0; + + //! Add topic to session, in other words, start reading new topic. + // virtual void AddTopic(const TTopicReadSettings& topicReadSettings) = 0; // Not implemented yet. + + //! Remove topic from session. + // virtual void RemoveTopic(const std::string& path) = 0; // Not implemented yet. + + //! Remove partition groups of topic from session. + // virtual void RemoveTopic(const std::string& path, const std::vector& partitionGruops) = 0; // Not implemented yet. + + //! Stop reading data and process only control events. + //! You might need this function if a receiving side + //! is not ready to process data. + virtual void StopReadingData() = 0; + + //! Resume reading data. + virtual void ResumeReadingData() = 0; + + //! Close read session. + //! Waits for all commit acknowledgments to arrive. + //! Force close after timeout. + //! This method is blocking. + //! When session is closed, + //! TSessionClosedEvent arrives. + virtual bool Close(TDuration timeout = TDuration::Max()) = 0; + + //! Reader counters with different stats (see TReaderConuters). + virtual TReaderCounters::TPtr GetCounters() const = 0; + + //! Get unique identifier of read session. + virtual std::string GetSessionId() const = 0; + + virtual ~IReadSession() = default; +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_events.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_events.h new file mode 100644 index 000000000000..2f5cee513e9f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_events.h @@ -0,0 +1,84 @@ +#pragma once + +#include "aliases.h" + +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +struct TWriteStat : public TThrRefBase { + TDuration WriteTime; + TDuration TotalTimeInPartitionQueue; + TDuration PartitionQuotedTime; + TDuration TopicQuotedTime; + using TPtr = TIntrusivePtr; +}; + +using NTopic::TContinuationToken; +using NTopic::TContinuationTokenIssuer; + +//! Events for write session. +struct TWriteSessionEvent { + + //! Event with acknowledge for written messages. + struct TWriteAck { + //! Write result. + enum EEventState { + EES_WRITTEN, //! Successfully written. + EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication. + EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer. + }; + //! Details of successfully written message. + struct TWrittenMessageDetails { + ui64 Offset; + ui64 PartitionId; + }; + //! Same SeqNo as provided on write. + ui64 SeqNo; + EEventState State; + //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED. + std::optional Details; + //! Write stats from server. See TWriteStat. nullptr for DISCARDED event. + TWriteStat::TPtr Stat; + + }; + + struct TAcksEvent { + //! Acks could be batched from several WriteBatch/Write requests. + //! Acks for messages from one WriteBatch request could be emitted as several TAcksEvents - + //! they are provided to client as soon as possible. + std::vector Acks; + + std::string DebugString() const; + + }; + + //! Indicates that a writer is ready to accept new message(s). + //! Continuation token should be kept and then used in write methods. + struct TReadyToAcceptEvent { + mutable TContinuationToken ContinuationToken; + + TReadyToAcceptEvent() = delete; + TReadyToAcceptEvent(TContinuationToken&& t) : ContinuationToken(std::move(t)) { + } + TReadyToAcceptEvent(TReadyToAcceptEvent&&) = default; + TReadyToAcceptEvent(const TReadyToAcceptEvent& other) : ContinuationToken(std::move(other.ContinuationToken)) { + } + TReadyToAcceptEvent& operator=(TReadyToAcceptEvent&&) = default; + TReadyToAcceptEvent& operator=(const TReadyToAcceptEvent& other) { + ContinuationToken = std::move(other.ContinuationToken); + return *this; + } + + std::string DebugString() const; + }; + + using TEvent = std::variant; +}; + +//! Event debug string. +std::string DebugString(const TWriteSessionEvent::TEvent& event); + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_session.h b/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_session.h new file mode 100644 index 000000000000..98852c46488d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/write_session.h @@ -0,0 +1,205 @@ +#pragma once + +#include "aliases.h" +#include "write_events.h" + +#include +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + +enum class EClusterDiscoveryMode { + Auto = 0, // enables cluster discovery only for hostname "logbroker.yandex.net" and "logbroker-prestable.yandex.net" + On, + Off +}; + +//! Settings for write session. +struct TWriteSessionSettings : public TRequestSettings { + using TSelf = TWriteSessionSettings; + + TWriteSessionSettings() = default; + TWriteSessionSettings(const TWriteSessionSettings&) = default; + TWriteSessionSettings(TWriteSessionSettings&&) = default; + TWriteSessionSettings(const std::string& path, const std::string& messageGroupId) { + Path(path); + MessageGroupId(messageGroupId); + } + + TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default; + TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; + + //! Path of topic to write. + FLUENT_SETTING(std::string, Path); + + //! MessageGroupId (aka SourceId) to use. + FLUENT_SETTING(std::string, MessageGroupId); + + //! Write to an exact partition group. Generally server assigns partition group automatically. + //! Using this option is not recommended unless you know for sure why you need it. + FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId); + + //! Preferred LB cluster. Used for multi-cluster installation. + //! If specified cluster is unavailable, session will write to other cluster. + FLUENT_SETTING_OPTIONAL(std::string, PreferredCluster); + + //! Write to other clusters if there are problems with connection + //! to the first one. + FLUENT_SETTING_DEFAULT(bool, AllowFallbackToOtherClusters, true); + + //! codec and level to use for data compression prior to write. + FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); + FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4); + + //! Writer will not accept new messages if memory usage exceeds this limit. + //! Memory usage consists of raw data pending compression and compressed messages being sent. + FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20_MB); + + //! Maximum messages accepted by writer but not written (with confirmation from server). + //! Writer will not accept new messages after reaching the limit. + FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000); + + //! Retry policy enables automatic retries for non-fatal errors. + //! IRetryPolicy::GetDefaultPolicy() if null (not set). + FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + + //! User metadata that may be attached to write session. + TWriteSessionSettings& AppendSessionMeta(const std::string& key, const std::string& value) { + Meta_.Fields[key] = value; + return *this; + }; + + NYdb::NPersQueue::TWriteSessionMeta Meta_; + + //! Writer will accumulate messages until reaching up to BatchFlushSize bytes + //! but for no longer than BatchFlushInterval. + //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch. + //! Greatly increases performance for small messages. + //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients + //! sending small messages at high rate). + FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); + FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes); + + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); + + FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); + + //! Executor for compression tasks. + //! If not set, default executor will be used. + FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); + + struct TEventHandlers { + using TSelf = TEventHandlers; + using TWriteAckHandler = std::function; + using TReadyToAcceptHandler = std::function; + + //! Function to handle Acks events. + //! If this handler is set, write ack events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TWriteAckHandler, AcksHandler); + + //! Function to handle ReadyToAccept event. + //! If this handler is set, write these events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHandler); + + //! Function to handle close session events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + + //! Function to handle all event types. + //! If event with current type has no handler for this type of event, + //! this handler (if specified) will be used. + //! If this handler is not specified, event can be received with TWriteSession::GetEvent() method. + FLUENT_SETTING(std::function, CommonHandler); + + //! Executor for handlers. + //! If not set, default single threaded executor will be used. + FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); + + [[deprecated("Typo in name. Use ReadyToAcceptHandler instead.")]] + TSelf& ReadyToAcceptHander(const TReadyToAcceptHandler& value) { + return ReadyToAcceptHandler(value); + } + }; + + //! Event handlers. + FLUENT_SETTING(TEventHandlers, EventHandlers); + + //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception. + FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true); + + //! Manages cluster discovery mode. + FLUENT_SETTING_DEFAULT(EClusterDiscoveryMode, ClusterDiscoveryMode, EClusterDiscoveryMode::Auto); +}; + +//! Simple write session. Does not need event handlers. Does not provide Events, ContinuationTokens, write Acks. +class ISimpleBlockingWriteSession : public TThrRefBase { +public: + //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded; + //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout. + //! no Ack is provided. + virtual bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, + const TDuration& blockTimeout = TDuration::Max()) = 0; + + //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init. + virtual ui64 GetInitSeqNo() = 0; + + //! Complete all active writes, wait for ack from server and close. + //! closeTimeout - max time to wait. Empty Maybe means infinity. + //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. + + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Returns true if write session is alive and acitve. False if session was closed. + virtual bool IsAlive() const = 0; + + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close immediately and destroy, don't wait for anything. + virtual ~ISimpleBlockingWriteSession() = default; +}; + +//! Generic write session with all capabilities. +class IWriteSession { +public: + //! Future that is set when next event is available. + virtual NThreading::TFuture WaitEvent() = 0; + + //! Wait and return next event. Use WaitEvent() for non-blocking wait. + virtual std::optional GetEvent(bool block = false) = 0; + + //! Get several events in one call. + //! If blocking = false, instantly returns up to maxEventsCount available events. + //! If blocking = true, blocks till maxEventsCount events are available. + //! If maxEventsCount is unset, write session decides the count to return itself. + virtual std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt) = 0; + + //! Future that is set when initial SeqNo is available. + virtual NThreading::TFuture GetInitSeqNo() = 0; + + //! Write single message. + //! continuationToken - a token earlier provided to client with ReadyToAccept event. + virtual void Write(TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) = 0; + + //! Write single message that is already coded by codec. Codec from settings does not apply to this message. + //! continuationToken - a token earlier provided to client with ReadyToAccept event. + //! originalSize - size of unpacked message + virtual void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, ui32 originalSize, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) = 0; + + + //! Wait for all writes to complete (no more that closeTimeout()), than close. Empty maybe - means infinite timeout. + //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Writer counters with different stats (see TWriterConuters). + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close() with timeout = 0 and destroy everything instantly. + virtual ~IWriteSession() = default; +}; + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/include/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/include/ya.make new file mode 100644 index 000000000000..ee49322fea6a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/include/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/src/client/persqueue_public/include/control_plane.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/src/client/persqueue_public/include/read_events.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/src/client/persqueue_public/include/write_events.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/src/client/persqueue_public/include/write_session.h) + +SRCS( + aliases.h + client.h + control_plane.h + read_events.h + read_session.h + write_events.h + write_session.h +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/grpc/draft + ydb/public/api/protos +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/persqueue.h b/ydb/public/sdk/cpp/src/client/persqueue_public/persqueue.h new file mode 100644 index 000000000000..c4673171ba6b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/persqueue.h @@ -0,0 +1,3 @@ +#pragma once + +#include "include/client.h" diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/basic_usage_ut.cpp similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/basic_usage_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/basic_usage_ut.cpp index 50f7aab8f56d..06bfefb801f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/basic_usage_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -71,7 +71,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { std::visit(TOverloaded { [&](TReadSessionEvent::TDataReceivedEvent& event) { for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); + std::string sourceId = message.GetMessageGroupId(); ui32 seqNo = message.GetSeqNo(); UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); ++readMessageCount; @@ -190,16 +190,16 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto readSession = client.CreateReadSession(readSettings); auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& createPartitionStream = std::get(*event); createPartitionStream.Confirm(); UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvent(true, 0), TContractViolation); - UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, Nothing(), 0), TContractViolation); + UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, std::nullopt, 0), TContractViolation); event = readSession->GetEvent(true, 1); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& dataReceived = std::get(*event); dataReceived.Commit(); @@ -380,7 +380,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { std::visit(TOverloaded { [&](TReadSessionEvent::TDataReceivedEvent& event) { for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); + std::string sourceId = message.GetMessageGroupId(); ui32 seqNo = message.GetSeqNo(); UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); ++readMessageCount; @@ -447,7 +447,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { public: TBrokenCredentialsProvider() {} virtual ~TBrokenCredentialsProvider() {} - TStringType GetAuthInfo() const { + std::string GetAuthInfo() const { ythrow yexception() << "exception during creation"; return ""; } @@ -463,7 +463,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { return std::make_shared(); } - virtual TStringType GetClientIdentity() const { + virtual std::string GetClientIdentity() const { return "abacaba"; } }; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/common_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/common_ut.cpp similarity index 93% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/common_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/common_ut.cpp index 79f42b4076d1..07115d62e058 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/common_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/common_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compress_executor_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/compress_executor_ut.cpp similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compress_executor_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/compress_executor_ut.cpp index 5aa46301f9c0..baed9bd81aac 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compress_executor_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/compress_executor_ut.cpp @@ -1,4 +1,4 @@ -#include +#include namespace NYdb::NPersQueue::NTests { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compression_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/compression_ut.cpp similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compression_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/compression_ut.cpp index 7688a58158ad..6119a4b8a9cc 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/compression_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/compression_ut.cpp @@ -1,4 +1,4 @@ -#include +#include namespace NYdb::NPersQueue::NTests { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/read_session_ut.cpp similarity index 96% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/read_session_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/read_session_ut.cpp index f504043b8e3e..8ab0db971236 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/read_session_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/read_session_ut.cpp @@ -1,11 +1,11 @@ -#include +#include #define INCLUDE_YDB_INTERNAL_H -#include +#include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include #include @@ -330,7 +330,7 @@ struct TMockReadSessionProcessor : public TMockProcessorFactory* metadata, TReadCallback callback) override { + void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { Y_UNUSED(metadata); Y_UNUSED(callback); UNIT_ASSERT_C(false, "This method is not expected to be called"); @@ -578,7 +578,7 @@ class TSynchronousExecutor : public ::IExecutor { } }; -extern TLogFormatter NYdb::GetPrefixLogFormatter(const TString& prefix); // Defined in ydb.cpp. +extern TLogFormatter NYdb::GetPrefixLogFormatter(const std::string& prefix); // Defined in ydb.cpp. TReadSessionImplTestSetup::TReadSessionImplTestSetup() { Settings @@ -662,7 +662,7 @@ void TReadSessionImplTestSetup::SuccessfulInit(bool hasInitRequest) { TPartitionStream::TPtr TReadSessionImplTestSetup::CreatePartitionStream(const TString& topic, const TString& cluster, ui64 partition, ui64 assignId) { MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().CreatePartitionStream(topic, cluster, partition, assignId)); // Callback will be called. - TMaybe event = EventsQueue->GetEvent(true); + std::optional event = EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); auto& createEvent = std::get(*event); @@ -673,7 +673,7 @@ TPartitionStream::TPtr TReadSessionImplTestSetup::CreatePartitionStream(const TS } void TReadSessionImplTestSetup::AssertNoEvents() { - TMaybe event = GetEventsQueue()->GetEvent(false); + std::optional event = GetEventsQueue()->GetEvent(false); UNIT_ASSERT(!event); } @@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { TDeferredCommit dc; // Event 1: create partition stream. { - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); std::get(*event).Confirm(); @@ -699,7 +699,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { } // Event 2: data. { - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); @@ -715,7 +715,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { setup.WriteToTopic({"message3"}); // Event 3: data. { - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); @@ -736,14 +736,14 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { // Event 4: commit ack. if (commit) { // (commit && close) branch check is broken with current TReadSession::Close quick fix - TMaybe event = session->GetEvent(!close); // Event is expected to be already in queue if closed. + std::optional event = session->GetEvent(!close); // Event is expected to be already in queue if closed. UNIT_ASSERT(event); Cerr << "commit ack or close event " << DebugString(*event) << Endl; UNIT_ASSERT(std::holds_alternative(*event) || std::holds_alternative(*event)); } if (close && !commit) { - TMaybe event = session->GetEvent(false); + std::optional event = session->GetEvent(false); UNIT_ASSERT(event); Cerr << "close event " << DebugString(*event) << Endl; UNIT_ASSERT(std::holds_alternative(*event)); @@ -775,7 +775,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { std::shared_ptr session = \ setup.GetPersQueueClient().CreateReadSession(settings); \ session->WaitEvent().Wait(); \ - TMaybe event = \ + std::optional event = \ session->GetEvent(true); \ UNIT_ASSERT(event); \ Cerr << DebugString(*event) << Endl; \ @@ -812,7 +812,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { // Set policy with max retries == 3. settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(TDuration::MilliSeconds(10), TDuration::MilliSeconds(10), TDuration::MilliSeconds(100), 3)); std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); Cerr << DebugString(*event) << Endl; UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); @@ -827,7 +827,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { // Success. { std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); Cerr << DebugString(*event) << Endl; UNIT_ASSERT_NOT_EVENT_TYPE(*event, TSessionClosedEvent); @@ -837,7 +837,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { { settings.AppendClusters("unknown_cluster"); std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); Cerr << DebugString(*event) << Endl; UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); @@ -847,7 +847,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { { settings.ReadOriginal({"unknown_cluster"}); std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); Cerr << DebugString(*event) << Endl; UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); @@ -861,7 +861,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { // Event 1: create partition stream. { - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); std::get(*event).Confirm(); @@ -869,8 +869,8 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { } // Event 2: receive data. - auto GetDataEvent = [&](const TString& content) -> TMaybe { - TMaybe event = session->GetEvent(true); + auto GetDataEvent = [&](const TString& content) -> std::optional { + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); @@ -880,7 +880,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { return event; }; - TMaybe dataEvents[2]; + std::optional dataEvents[2]; dataEvents[0] = GetDataEvent("message1"); @@ -899,7 +899,7 @@ Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { // Commit and check that other events will come. for (int i = 0; i < 2; ++i) { std::get(*dataEvents[i]).Commit(); - TMaybe event = session->GetEvent(true); + std::optional event = session->GetEvent(true); UNIT_ASSERT(event); Y_ASSERT(std::holds_alternative(*event)); Cerr << DebugString(*event) << Endl; @@ -1105,7 +1105,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().CreatePartitionStream()); // Callback will be called. { - TVector events = setup.EventsQueue->GetEvents(true); + std::vector events = setup.EventsQueue->GetEvents(true); UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TCreatePartitionStreamEvent); auto& event = std::get(events[0]); @@ -1146,7 +1146,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { // Check destroy event. if (!forceful) { - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDestroyPartitionStreamEvent); auto& destroyEvent = std::get(*event); @@ -1165,7 +1165,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { // Check closed event. { - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamClosedEvent); auto& closedEvent = std::get(*event); @@ -1200,7 +1200,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { } for (ui64 i = 1; i <= 2; ) { - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); auto& dataEvent = std::get(*event); @@ -1226,7 +1226,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { // Exception was passed during decompression. { - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); auto& dataEvent = std::get(*event); @@ -1254,7 +1254,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { .Batch("src_id") .CompressMessage(1, data, codec)); - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); auto& dataEvent = std::get(*event); @@ -1360,7 +1360,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { ui64 prevSeqNo = 41; for (size_t i = 0; i < batches; ++i) { Cerr << "Getting new event" << Endl; - TMaybe event = setup.EventsQueue->GetEvent(true, batchLimit); + std::optional event = setup.EventsQueue->GetEvent(true, batchLimit); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); Cerr << DebugString(*event) << Endl; @@ -1500,7 +1500,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { } { - TVector events = setup.EventsQueue->GetEvents(true); + std::vector events = setup.EventsQueue->GetEvents(true); UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(events[0]); @@ -1509,7 +1509,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { } { - TVector events = setup.EventsQueue->GetEvents(true); + std::vector events = setup.EventsQueue->GetEvents(true); UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(events[0]); @@ -1540,7 +1540,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { stream->RequestStatus(); - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamStatusEvent); auto& statusEvent = std::get(*event); @@ -1591,7 +1591,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { })); for (int i = 0; i < 2; ) { - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); @@ -1616,7 +1616,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { .Batch("src_id") .CompressMessage(1, "message1")); - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); @@ -1693,7 +1693,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() .ForcefulReleasePartitionStream()); - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamClosedEvent); calledPromise.GetFuture().Wait(); @@ -1724,7 +1724,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { .Batch("src_id") .CompressMessage(1, "message1")); - TMaybe event = setup.EventsQueue->GetEvent(true); + std::optional event = setup.EventsQueue->GetEvent(true); UNIT_ASSERT(event); UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/retry_policy_ut.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/retry_policy_ut.cpp similarity index 99% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/retry_policy_ut.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/retry_policy_ut.cpp index 81b8801f0506..9739f6fb63dc 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/retry_policy_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/retry_policy_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -316,7 +316,7 @@ Y_UNIT_TEST_SUITE(RetryPolicy) { Cerr << "===Data event\n"; auto& clusterName = event.GetPartitionStream()->GetCluster(); for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); + auto sourceId = TString{message.GetMessageGroupId()}; ui32 seqNo = message.GetSeqNo(); if (sourceId == sourceId1) { UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp similarity index 94% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp index 75fd7ae041b3..ee87596df8c1 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp @@ -1,6 +1,6 @@ #include "data_plane_helpers.h" -#include -#include +#include +#include namespace NKikimr::NPersQueueTests { @@ -53,7 +53,7 @@ namespace NKikimr::NPersQueueTests { std::optional partitionGroup, std::optional codec, std::optional reconnectOnFailure, - THashMap sessionMeta, + std::unordered_map sessionMeta, const TString& userAgent ) { auto settings = TWriteSessionSettings().Path(topic).MessageGroupId(sourceId); @@ -105,7 +105,7 @@ namespace NKikimr::NPersQueueTests { auto future = reader->WaitEvent(); future.Wait(timeout); - TMaybe event = reader->GetEvent(false, 1); + std::optional event = reader->GetEvent(false, 1); if (!event) return {}; if (auto dataEvent = std::get_if(&*event)) { @@ -125,8 +125,7 @@ namespace NKikimr::NPersQueueTests { while (true) { auto future = reader->WaitEvent(); future.Wait(timeout); - - TMaybe event = reader->GetEvent(false, 1); + std::optional event = reader->GetEvent(false, 1); if (!event) return {}; if (auto e = std::get_if(&*event)) { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.h b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h similarity index 87% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.h rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h index 3b39a0d94946..a62a0de014d6 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/data_plane_helpers.h +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h @@ -1,9 +1,11 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include + +#include namespace NKikimr::NPersQueueTests { @@ -35,7 +37,7 @@ namespace NKikimr::NPersQueueTests { std::optional partitionGroup = {}, std::optional codec = {}, std::optional reconnectOnFailure = {}, - THashMap sessionMeta = {}, + std::unordered_map sessionMeta = {}, const TString& userAgent = {} ); @@ -43,7 +45,7 @@ namespace NKikimr::NPersQueueTests { NYdb::TDriver& driver, const NYdb::NPersQueue::TReadSessionSettings& settings, std::shared_ptr creds = nullptr - ); + ); std::shared_ptr CreateReader( NYdb::TDriver& driver, diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/sdk_test_setup.h b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/sdk_test_setup.h rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h index b1c9bf2ab426..3827827504e6 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/sdk_test_setup.h +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h @@ -1,6 +1,6 @@ #pragma once #include "test_server.h" -#include +#include #include #include @@ -88,7 +88,7 @@ class SDKTestSetup { } } - static TString GetTestTopic() { + static std::string GetTestTopic() { return "test-topic"; } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_server.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_server.cpp similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_server.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_server.cpp diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_server.h b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_server.h similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_server.h rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_server.h index bfc6aaa39481..80abcba47fc2 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_server.h +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_server.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_utils.h b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_utils.h similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/test_utils.h rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/test_utils.h diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ut_utils.cpp similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.cpp rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ut_utils.cpp diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.h b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ut_utils.h similarity index 96% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.h rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ut_utils.h index aa7fb790f9a2..a68664bdcd5f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.h +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ut_utils.h @@ -1,11 +1,11 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include -#include +#include using namespace NKikimr; using namespace NKikimr::NPersQueueTests; @@ -45,7 +45,7 @@ class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort); cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); Driver = MakeHolder(cfg); } return *Driver; @@ -168,7 +168,7 @@ struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop { } Y_ABORT_UNLESS(continueToken); - TMaybe seqNo = Nothing(); + std::optional seqNo = std::nullopt; if (!AutoSeqNo) { seqNo = acknowledgeableMessage.SequenceNumber; Log << TLOG_INFO << "[" << sourceId << "] Write messages with sequence numbers " @@ -377,7 +377,7 @@ class TYdbPqTestExecutor : public IAsyncExecutor { Stop = true; Thread.Join(); } - void PostImpl(TVector&& fs) override { + void PostImpl(std::vector&& fs) override { for (auto& f : fs) { TasksQueue.Enqueue(std::move(f)); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ya.make similarity index 54% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ya.make rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ya.make index caa9e092fb00..de1690acb9d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ya.make +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils/ya.make @@ -1,5 +1,7 @@ LIBRARY() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( data_plane_helpers.h data_plane_helpers.cpp @@ -16,11 +18,11 @@ PEERDIR( library/cpp/testing/unittest library/cpp/threading/chunk_queue ydb/core/testlib/default - ydb/library/persqueue/topic_parser_public - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/with_offset_ranges_mode_ut/ya.make similarity index 62% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut/ya.make rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/with_offset_ranges_mode_ut/ya.make index 0c5378f8b8bf..4ddaf666a0fa 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut/ya.make +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/with_offset_ranges_mode_ut/ya.make @@ -1,5 +1,7 @@ UNITTEST() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) SIZE(LARGE) TAG(ya:fat) @@ -13,8 +15,8 @@ PEERDIR( library/cpp/testing/gmock_in_unittest ydb/public/lib/json_value ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ) YQL_LAST_ABI_VERSION() @@ -22,8 +24,8 @@ YQL_LAST_ABI_VERSION() ENV(PQ_OFFSET_RANGES_MODE="1") SRCDIR( - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut - ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public/ut + ydb/public/sdk/cpp/src/client/persqueue_public ) SRCS( diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ya.make similarity index 54% rename from ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ya.make rename to ydb/public/sdk/cpp/src/client/persqueue_public/ut/ya.make index 870d50b0fada..b8f9f22eb759 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ya.make +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ut/ya.make @@ -1,4 +1,6 @@ -UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_persqueue_public) +UNITTEST_FOR(ydb/public/sdk/cpp/src/client/persqueue_public) + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) SIZE(LARGE) @@ -14,12 +16,12 @@ PEERDIR( ydb/core/testlib/default ydb/public/lib/json_value ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_persqueue_public/impl - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_topic/codecs - ydb/public/sdk/cpp/client/ydb_topic/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public/impl + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/topic/impl ) YQL_LAST_ABI_VERSION() diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/ya.make b/ydb/public/sdk/cpp/src/client/persqueue_public/ya.make new file mode 100644 index 000000000000..55cf1d667459 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/ya.make @@ -0,0 +1,29 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + persqueue.h +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/persqueue_public/impl + ydb/public/sdk/cpp/src/client/persqueue_public/include + + ydb/public/sdk/cpp/src/client/topic/common + ydb/public/sdk/cpp/src/client/topic/impl + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic + + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/driver + + ydb/public/api/grpc + ydb/public/api/grpc/draft + ydb/public/api/protos +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/sdk/cpp/src/client/proto/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/proto/CMakeLists.txt new file mode 100644 index 000000000000..cd2b8590e6b4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/proto/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-ydb_proto) + +target_link_libraries(client-ydb_proto PUBLIC + yutil + api-grpc + api-grpc-draft + api-protos + library-operation_id + client-ydb_params + client-ydb_value +) + +target_sources(client-ydb_proto PRIVATE + accessor.cpp +) + +_ydb_sdk_make_client_component(Proto client-ydb_proto) diff --git a/ydb/public/sdk/cpp/src/client/proto/accessor.cpp b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp new file mode 100644 index 000000000000..9adbba5c0e6d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp @@ -0,0 +1,138 @@ +#include + +#include + +namespace NYdb::inline V3 { + +const Ydb::Type& TProtoAccessor::GetProto(const TType& type) { + return type.GetProto(); +} + +const Ydb::Value& TProtoAccessor::GetProto(const TValue& value) { + return value.GetProto(); +} + +// exports & imports +template +typename TProtoSettings::Scheme TProtoAccessor::GetProto(ES3Scheme value) { + switch (value) { + case ES3Scheme::HTTP: + return TProtoSettings::HTTP; + case ES3Scheme::HTTPS: + return TProtoSettings::HTTPS; + } +} + +const ::google::protobuf::Map& TProtoAccessor::GetProtoMap(const TParams& params) { + return params.GetProtoMap(); +} + +::google::protobuf::Map* TProtoAccessor::GetProtoMapPtr(TParams& params) { + return params.GetProtoMapPtr(); +} + +template Ydb::Export::ExportToS3Settings::Scheme TProtoAccessor::GetProto(ES3Scheme value); +template Ydb::Import::ImportFromS3Settings::Scheme TProtoAccessor::GetProto(ES3Scheme value); + +template +ES3Scheme TProtoAccessor::FromProto(typename TProtoSettings::Scheme value) { + switch (value) { + case TProtoSettings::HTTP: + return ES3Scheme::HTTP; + default: + return ES3Scheme::HTTPS; + } +} + +template ES3Scheme TProtoAccessor::FromProto(Ydb::Export::ExportToS3Settings::Scheme value); +template ES3Scheme TProtoAccessor::FromProto(Ydb::Import::ImportFromS3Settings::Scheme value); + +Ydb::Export::ExportToS3Settings::StorageClass TProtoAccessor::GetProto(NExport::TExportToS3Settings::EStorageClass value) { + switch (value) { + case NExport::TExportToS3Settings::EStorageClass::STANDARD: + return Ydb::Export::ExportToS3Settings::STANDARD; + case NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY: + return Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY; + case NExport::TExportToS3Settings::EStorageClass::STANDARD_IA: + return Ydb::Export::ExportToS3Settings::STANDARD_IA; + case NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA: + return Ydb::Export::ExportToS3Settings::ONEZONE_IA; + case NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING: + return Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING; + case NExport::TExportToS3Settings::EStorageClass::GLACIER: + return Ydb::Export::ExportToS3Settings::GLACIER; + case NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE: + return Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE; + case NExport::TExportToS3Settings::EStorageClass::OUTPOSTS: + return Ydb::Export::ExportToS3Settings::OUTPOSTS; + default: + return Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED; + } +} + +NExport::TExportToS3Settings::EStorageClass TProtoAccessor::FromProto(Ydb::Export::ExportToS3Settings::StorageClass value) { + switch (value) { + case Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED: + return NExport::TExportToS3Settings::EStorageClass::NOT_SET; + case Ydb::Export::ExportToS3Settings::STANDARD: + return NExport::TExportToS3Settings::EStorageClass::STANDARD; + case Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY: + return NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY; + case Ydb::Export::ExportToS3Settings::STANDARD_IA: + return NExport::TExportToS3Settings::EStorageClass::STANDARD_IA; + case Ydb::Export::ExportToS3Settings::ONEZONE_IA: + return NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA; + case Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING: + return NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING; + case Ydb::Export::ExportToS3Settings::GLACIER: + return NExport::TExportToS3Settings::EStorageClass::GLACIER; + case Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE: + return NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE; + case Ydb::Export::ExportToS3Settings::OUTPOSTS: + return NExport::TExportToS3Settings::EStorageClass::OUTPOSTS; + default: + return NExport::TExportToS3Settings::EStorageClass::UNKNOWN; + } +} + +NExport::EExportProgress TProtoAccessor::FromProto(Ydb::Export::ExportProgress::Progress value) { + switch (value) { + case Ydb::Export::ExportProgress::PROGRESS_UNSPECIFIED: + return NExport::EExportProgress::Unspecified; + case Ydb::Export::ExportProgress::PROGRESS_PREPARING: + return NExport::EExportProgress::Preparing; + case Ydb::Export::ExportProgress::PROGRESS_TRANSFER_DATA: + return NExport::EExportProgress::TransferData; + case Ydb::Export::ExportProgress::PROGRESS_DONE: + return NExport::EExportProgress::Done; + case Ydb::Export::ExportProgress::PROGRESS_CANCELLATION: + return NExport::EExportProgress::Cancellation; + case Ydb::Export::ExportProgress::PROGRESS_CANCELLED: + return NExport::EExportProgress::Cancelled; + default: + return NExport::EExportProgress::Unknown; + } +} + +NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress::Progress value) { + switch (value) { + case Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED: + return NImport::EImportProgress::Unspecified; + case Ydb::Import::ImportProgress::PROGRESS_PREPARING: + return NImport::EImportProgress::Preparing; + case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: + return NImport::EImportProgress::TransferData; + case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: + return NImport::EImportProgress::BuildIndexes; + case Ydb::Import::ImportProgress::PROGRESS_DONE: + return NImport::EImportProgress::Done; + case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: + return NImport::EImportProgress::Cancellation; + case Ydb::Import::ImportProgress::PROGRESS_CANCELLED: + return NImport::EImportProgress::Cancelled; + default: + return NImport::EImportProgress::Unknown; + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/proto/ya.make b/ydb/public/sdk/cpp/src/client/proto/ya.make new file mode 100644 index 000000000000..229886bed040 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/proto/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + accessor.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/grpc/draft + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/value +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/query/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/query/CMakeLists.txt new file mode 100644 index 000000000000..f35da5edb253 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/CMakeLists.txt @@ -0,0 +1,27 @@ +add_subdirectory(impl) + +_ydb_sdk_add_library(client-ydb_query) + +target_link_libraries(client-ydb_query PUBLIC + yutil + impl-ydb_internal-make_request + impl-ydb_internal-kqp_session_common + impl-ydb_internal-session_pool + impl-ydb_internal-retry + client-ydb_common_client + client-ydb_driver + client-ydb_query-impl + client-ydb_result + client-ydb_types-operation + api-protos + api-grpc +) + +target_sources(client-ydb_query PRIVATE + client.cpp + query.cpp + stats.cpp + tx.cpp +) + +_ydb_sdk_make_client_component(Query client-ydb_query) diff --git a/ydb/public/sdk/cpp/src/client/query/client.cpp b/ydb/public/sdk/cpp/src/client/query/client.cpp new file mode 100644 index 000000000000..761274f43499 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/client.cpp @@ -0,0 +1,732 @@ +#include "impl/client_session.h" + +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#include +#include +#include +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NQuery { + +using TRetryContextResultAsync = NRetry::Async::TRetryContext; +using TRetryContextAsync = NRetry::Async::TRetryContext; + +NYdb::NRetry::TRetryOperationSettings GetRetrySettings(TDuration timeout, bool isIndempotent) { + return NYdb::NRetry::TRetryOperationSettings() + .Idempotent(isIndempotent) + .GetSessionClientTimeout(timeout) + .MaxTimeout(timeout); +} + +TCreateSessionSettings::TCreateSessionSettings() { + ClientTimeout_ = TDuration::Seconds(5); +}; + +static void SetTxSettings(const TTxSettings& txSettings, Ydb::Query::TransactionSettings* proto) +{ + switch (txSettings.GetMode()) { + case TTxSettings::TS_SERIALIZABLE_RW: + proto->mutable_serializable_read_write(); + break; + case TTxSettings::TS_ONLINE_RO: + proto->mutable_online_read_only()->set_allow_inconsistent_reads( + txSettings.OnlineSettings_.AllowInconsistentReads_); + break; + case TTxSettings::TS_STALE_RO: + proto->mutable_stale_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RO: + proto->mutable_snapshot_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RW: + proto->mutable_snapshot_read_write(); + break; + default: + throw TContractViolation("Unexpected transaction mode."); + } +} + +class TQueryClient::TImpl: public TClientImplCommon, public ISessionClient { + friend class ::NYdb::NQuery::TSession; +public: + TImpl(std::shared_ptr&& connections, const TClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + , Settings_(settings) + , SessionPool_(Settings_.SessionPoolSettings_.MaxActiveSessions_) + { + } + + ~TImpl() { + // TODO: Drain sessions. + } + + TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session = {}) + { + return TExecQueryImpl::StreamExecuteQuery( + Connections_, DbDriverState_, query, txControl, params, settings, session); + } + + TAsyncExecuteQueryResult ExecuteQuery(const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, + const std::optional& session = {}) + { + return TExecQueryImpl::ExecuteQuery( + Connections_, DbDriverState_, query, txControl, params, settings, session); + } + + NThreading::TFuture ExecuteScript(const std::string& script, const std::optional& params, const TExecuteScriptSettings& settings) { + using namespace Ydb::Query; + auto request = MakeOperationRequest(settings); + request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); + request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); + request.set_pool_id(TStringType{settings.ResourcePool_}); + request.mutable_script_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); + request.mutable_script_content()->set_text(TStringType{script}); + SetDuration(settings.ResultsTtl_, *request.mutable_results_ttl()); + + if (params) { + *request.mutable_parameters() = params->GetProtoMap(); + } + + auto promise = NThreading::NewPromise(); + + auto responseCb = [promise] + (Ydb::Operations::Operation* response, TPlainStatus status) mutable { + try { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus executeScriptStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + promise.SetValue(TScriptExecutionOperation(TStatus(std::move(executeScriptStatus)), std::move(*response))); + } else { + promise.SetValue(TScriptExecutionOperation(TStatus(std::move(status)))); + } + } catch (...) { + promise.SetException(std::current_exception()); + } + }; + + Connections_->Run( + std::move(request), + responseCb, + &V1::QueryService::Stub::AsyncExecuteScript, + DbDriverState_, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncFetchScriptResultsResult FetchScriptResults(const NKikimr::NOperationId::TOperationId& operationId, int64_t resultSetIndex, const TFetchScriptResultsSettings& settings) { + auto request = MakeRequest(); + request.set_operation_id(TStringType{operationId.ToString()}); + request.set_result_set_index(resultSetIndex); + return FetchScriptResultsImpl(std::move(request), settings); + } + + TAsyncStatus RollbackTransaction(const std::string& txId, const NYdb::NQuery::TRollbackTxSettings& settings, const TSession& session) { + using namespace Ydb::Query; + auto request = MakeRequest(); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); + + auto promise = NThreading::NewPromise(); + + auto responseCb = [promise, session] + (Ydb::Query::RollbackTransactionResponse* response, TPlainStatus status) mutable { + try { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus rollbackTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + + promise.SetValue(std::move(rollbackTxStatus)); + } else { + promise.SetValue(TStatus(std::move(status))); + } + } catch (...) { + promise.SetException(std::current_exception()); + } + }; + + Connections_->Run( + std::move(request), + responseCb, + &V1::QueryService::Stub::AsyncRollbackTransaction, + DbDriverState_, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey())); + + return promise.GetFuture(); + } + + TAsyncCommitTransactionResult CommitTransaction(const std::string& txId, const NYdb::NQuery::TCommitTxSettings& settings, const TSession& session) { + using namespace Ydb::Query; + auto request = MakeRequest(); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); + + auto promise = NThreading::NewPromise(); + + auto responseCb = [promise, session] + (Ydb::Query::CommitTransactionResponse* response, TPlainStatus status) mutable { + try { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus commitTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + + TCommitTransactionResult commitTxResult(std::move(commitTxStatus)); + promise.SetValue(std::move(commitTxResult)); + } else { + promise.SetValue(TCommitTransactionResult(TStatus(std::move(status)))); + } + } catch (...) { + promise.SetException(std::current_exception()); + } + }; + + Connections_->Run( + std::move(request), + responseCb, + &V1::QueryService::Stub::AsyncCommitTransaction, + DbDriverState_, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey())); + + return promise.GetFuture(); + } + + TAsyncBeginTransactionResult BeginTransaction(const TTxSettings& txSettings, + const TBeginTxSettings& settings, const TSession& session) + { + using namespace Ydb::Query; + auto request = MakeRequest(); + request.set_session_id(TStringType{session.GetId()}); + SetTxSettings(txSettings, request.mutable_tx_settings()); + + auto promise = NThreading::NewPromise(); + + auto responseCb = [promise, session] + (Ydb::Query::BeginTransactionResponse* response, TPlainStatus status) mutable { + try { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus beginTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + + TBeginTransactionResult beginTxResult(std::move(beginTxStatus), + TTransaction(session, response->tx_meta().id())); + promise.SetValue(std::move(beginTxResult)); + } else { + promise.SetValue(TBeginTransactionResult( + TStatus(std::move(status)), TTransaction(session, ""))); + } + } catch (...) { + promise.SetException(std::current_exception()); + } + }; + + Connections_->Run( + std::move(request), + responseCb, + &V1::QueryService::Stub::AsyncBeginTransaction, + DbDriverState_, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey())); + + return promise.GetFuture(); + } + + TAsyncFetchScriptResultsResult FetchScriptResultsImpl(Ydb::Query::FetchScriptResultsRequest&& request, const TFetchScriptResultsSettings& settings) { + using namespace Ydb::Query; + if (!settings.FetchToken_.empty()) { + request.set_fetch_token(TStringType{settings.FetchToken_}); + } + request.set_rows_limit(settings.RowsLimit_); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (FetchScriptResultsResponse* response, TPlainStatus status) mutable { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus st(static_cast(response->status()), std::move(opIssues)); + + if (st.IsSuccess()) { + promise.SetValue( + TFetchScriptResultsResult( + std::move(st), + TResultSet(std::move(*response->mutable_result_set())), + response->result_set_index(), + response->next_fetch_token() + ) + ); + } else { + promise.SetValue(TFetchScriptResultsResult(std::move(st))); + } + } else { + TStatus st(std::move(status)); + promise.SetValue(TFetchScriptResultsResult(std::move(st))); + } + }; + + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = TDuration::Seconds(60); + + Connections_->Run( + std::move(request), + extractor, + &V1::QueryService::Stub::AsyncFetchScriptResults, + DbDriverState_, + rpcSettings); + + return promise.GetFuture(); + } + + void DeleteSession(TKqpSessionCommon* sessionImpl) override { + //TODO: Remove this copy-paste + + // Closing not owned by session pool session should not fire getting new session + if (sessionImpl->IsOwnedBySessionPool()) { + if (SessionPool_.CheckAndFeedWaiterNewSession(sessionImpl->NeedUpdateActiveCounter())) { + // We requested new session for waiter which already incremented + // active session counter and old session will be deleted + // - skip update active counter in this case + sessionImpl->SetNeedUpdateActiveCounter(false); + } + } + + if (sessionImpl->NeedUpdateActiveCounter()) { + SessionPool_.DecrementActiveCounter(); + } + + delete sessionImpl; + } + + bool ReturnSession(TKqpSessionCommon* sessionImpl) override { + Y_ABORT_UNLESS(sessionImpl->GetState() == TSession::TImpl::S_ACTIVE || + sessionImpl->GetState() == TSession::TImpl::S_IDLE); + + //TODO: Remove this copy-paste from table client + bool needUpdateCounter = sessionImpl->NeedUpdateActiveCounter(); + // Also removes NeedUpdateActiveCounter flag + sessionImpl->MarkIdle(); + sessionImpl->SetTimeInterval(TDuration::Zero()); + if (!SessionPool_.ReturnSession(sessionImpl, needUpdateCounter)) { + sessionImpl->SetNeedUpdateActiveCounter(needUpdateCounter); + return false; + } + return true; + } + + void DoAttachSession(Ydb::Query::CreateSessionResponse* resp, + NThreading::TPromise promise, const std::string& endpoint, + std::shared_ptr client) + { + using TStreamProcessorPtr = TSession::TImpl::TStreamProcessorPtr; + Ydb::Query::AttachSessionRequest request; + const auto sessionId = resp->session_id(); + request.set_session_id(sessionId); + + auto args = std::make_shared(promise, sessionId, endpoint, client, client); + + // Do not pass client timeout here. Session must be alive + TRpcRequestSettings rpcSettings; + rpcSettings.PreferredEndpoint = TEndpointKey(endpoint, GetNodeIdFromSession(sessionId)); + + Connections_->StartReadStream< + Ydb::Query::V1::QueryService, + Ydb::Query::AttachSessionRequest, + Ydb::Query::SessionState> + ( + std::move(request), + [args] (TPlainStatus status, TStreamProcessorPtr processor) mutable { + if (processor) { + TSession::TImpl::MakeImplAsync(processor, args); + } else { + TStatus st(std::move(status)); + args->Promise.SetValue(TCreateSessionResult(std::move(st), TSession())); + } + }, + &Ydb::Query::V1::QueryService::Stub::AsyncAttachSession, + DbDriverState_, + rpcSettings); + } + + TAsyncCreateSessionResult CreateAttachedSession(TDuration timeout) { + using namespace Ydb::Query; + + Ydb::Query::CreateSessionRequest request; + + auto promise = NThreading::NewPromise(); + + auto self = shared_from_this(); + + auto extractor = [promise, self] (Ydb::Query::CreateSessionResponse* resp, TPlainStatus status) mutable { + if (resp) { + if (resp->status() != Ydb::StatusIds::SUCCESS) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(resp->issues(), opIssues); + TStatus st(static_cast(resp->status()), std::move(opIssues)); + promise.SetValue(TCreateSessionResult(std::move(st), TSession())); + } else { + self->DoAttachSession(resp, promise, status.Endpoint, self); + } + } else { + TStatus st(std::move(status)); + promise.SetValue(TCreateSessionResult(std::move(st), TSession())); + } + }; + + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = timeout; + + Connections_->Run( + std::move(request), + extractor, + &V1::QueryService::Stub::AsyncCreateSession, + DbDriverState_, + rpcSettings); + + return promise.GetFuture(); + } + + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings) { + using namespace NSessionPool; + + class TQueryClientGetSessionCtx : public IGetSessionCtx { + public: + TQueryClientGetSessionCtx(std::shared_ptr client, TDuration timeout) + : Promise(NThreading::NewPromise()) + , Client(client) + , ClientTimeout(timeout) + {} + + TAsyncCreateSessionResult GetFuture() { + return Promise.GetFuture(); + } + + void ReplyError(TStatus status) override { + TSession session; + ScheduleReply(TCreateSessionResult(std::move(status), std::move(session))); + } + + void ReplySessionToUser(TKqpSessionCommon* session) override { + TCreateSessionResult val( + TStatus(TPlainStatus()), + TSession( + Client, + static_cast(session) + ) + ); + + ScheduleReply(std::move(val)); + } + + void ReplyNewSession() override { + Client->CreateAttachedSession(ClientTimeout).Subscribe( + [promise{std::move(Promise)}](TAsyncCreateSessionResult future) mutable + { + promise.SetValue(future.ExtractValue()); + }); + } + + private: + void ScheduleReply(TCreateSessionResult val) { + Promise.SetValue(std::move(val)); + } + NThreading::TPromise Promise; + std::shared_ptr Client; + TDuration ClientTimeout; + }; + + auto ctx = std::make_unique(shared_from_this(), settings.ClientTimeout_); + auto future = ctx->GetFuture(); + SessionPool_.GetSession(std::move(ctx)); + return future; + } + + int64_t GetActiveSessionCount() const { + return SessionPool_.GetActiveSessions(); + } + + int64_t GetActiveSessionsLimit() const { + return SessionPool_.GetActiveSessionsLimit(); + } + + int64_t GetCurrentPoolSize() const { + return SessionPool_.GetCurrentPoolSize(); + } + + void StartPeriodicSessionPoolTask() { + // Session pool guarantees than client is alive during call callbacks + auto deletePredicate = [this](TKqpSessionCommon* s, size_t sessionsCount) { + + const auto& sessionPoolSettings = Settings_.SessionPoolSettings_; + const auto spentTime = s->GetTimeToTouchFast() - s->GetTimeInPastFast(); + + if (spentTime >= sessionPoolSettings.CloseIdleThreshold_) { + if (sessionsCount > sessionPoolSettings.MinPoolSize_) { + return true; + } + } + + return false; + }; + + std::weak_ptr weak = shared_from_this(); + Connections_->AddPeriodicTask( + SessionPool_.CreatePeriodicTask( + weak, + NSessionPool::TSessionPool::TKeepAliveCmd(), // no keep-alive cmd for query service + std::move(deletePredicate) + ), NSessionPool::PERIODIC_ACTION_INTERVAL); + } + + void CollectRetryStatAsync(EStatus status) { + Y_UNUSED(status); + } + + void CollectRetryStatSync(EStatus status) { + Y_UNUSED(status); + } + +private: + TClientSettings Settings_; + NSessionPool::TSessionPool SessionPool_; +}; + +TQueryClient::TQueryClient(const TDriver& driver, const TClientSettings& settings) + : Impl_(new TQueryClient::TImpl(CreateInternalInterface(driver), settings)) +{ + Impl_->StartPeriodicSessionPoolTask(); +} + +TAsyncExecuteQueryResult TQueryClient::ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings) +{ + return Impl_->ExecuteQuery(query, txControl, {}, settings); +} + +TAsyncExecuteQueryResult TQueryClient::ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings) +{ + return Impl_->ExecuteQuery(query, txControl, params, settings); +} + +TAsyncExecuteQueryIterator TQueryClient::StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings) +{ + return Impl_->StreamExecuteQuery(query, txControl, {}, settings); +} + +TAsyncExecuteQueryIterator TQueryClient::StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings) +{ + return Impl_->StreamExecuteQuery(query, txControl, params, settings); +} + +NThreading::TFuture TQueryClient::ExecuteScript(const std::string& script, + const TExecuteScriptSettings& settings) +{ + return Impl_->ExecuteScript(script, {}, settings); +} + +NThreading::TFuture TQueryClient::ExecuteScript(const std::string& script, + const TParams& params, const TExecuteScriptSettings& settings) +{ + return Impl_->ExecuteScript(script, params, settings); +} + +TAsyncFetchScriptResultsResult TQueryClient::FetchScriptResults(const NKikimr::NOperationId::TOperationId& operationId, int64_t resultSetIndex, + const TFetchScriptResultsSettings& settings) +{ + return Impl_->FetchScriptResults(operationId, resultSetIndex, settings); +} + +TAsyncCreateSessionResult TQueryClient::GetSession(const TCreateSessionSettings& settings) +{ + return Impl_->GetSession(settings); +} + +int64_t TQueryClient::GetActiveSessionCount() const { + return Impl_->GetActiveSessionCount(); +} + +int64_t TQueryClient::GetActiveSessionsLimit() const { + return Impl_->GetActiveSessionsLimit(); +} + +int64_t TQueryClient::GetCurrentPoolSize() const { + return Impl_->GetCurrentPoolSize(); +} + +TAsyncExecuteQueryResult TQueryClient::RetryQuery(TQueryResultFunc&& queryFunc, TRetryOperationSettings settings) +{ + TRetryContextResultAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +TAsyncStatus TQueryClient::RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings) { + TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +TAsyncStatus TQueryClient::RetryQuery(TQueryWithoutSessionFunc&& queryFunc, TRetryOperationSettings settings) { + TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithoutSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +TStatus TQueryClient::RetryQuerySync(const TQuerySyncFunc& queryFunc, TRetryOperationSettings settings) { + NRetry::Sync::TRetryWithSession ctx(*this, queryFunc, settings); + return ctx.Execute(); +} + +TStatus TQueryClient::RetryQuerySync(const TQueryWithoutSessionSyncFunc& queryFunc, TRetryOperationSettings settings) { + NRetry::Sync::TRetryWithoutSession ctx(*this, queryFunc, settings); + return ctx.Execute(); +} + +TAsyncExecuteQueryResult TQueryClient::RetryQuery(const std::string& query, const TTxControl& txControl, + TDuration timeout, bool isIndempotent) +{ + auto settings = GetRetrySettings(timeout, isIndempotent); + auto queryFunc = [&query, &txControl](TSession session, TDuration duration) -> TAsyncExecuteQueryResult { + return session.ExecuteQuery(query, txControl, TExecuteQuerySettings().ClientTimeout(duration)); + }; + TRetryContextResultAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TCreateSessionResult::TCreateSessionResult(TStatus&& status, TSession&& session) + : TStatus(std::move(status)) + , Session_(std::move(session)) +{} + +TSession TCreateSessionResult::GetSession() const { + CheckStatusOk("TCreateSessionResult::GetSession"); + return Session_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSession::TSession() +{} + +TSession::TSession(std::shared_ptr client, TSession::TImpl* session) + : Client_(client) + , SessionImpl_(session, TKqpSessionCommon::GetSmartDeleter(client)) +{} + +const std::string& TSession::GetId() const { + return SessionImpl_->GetId(); +} + +TAsyncExecuteQueryResult TSession::ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings) +{ + return NSessionPool::InjectSessionStatusInterception( + SessionImpl_, + Client_->ExecuteQuery(query, txControl, {}, settings, *this), + true, + Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); +} + +TAsyncExecuteQueryResult TSession::ExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings) +{ + return NSessionPool::InjectSessionStatusInterception( + SessionImpl_, + Client_->ExecuteQuery(query, txControl, params, settings, *this), + true, + Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); +} + +TAsyncExecuteQueryIterator TSession::StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TExecuteQuerySettings& settings) +{ + return NSessionPool::InjectSessionStatusInterception( + SessionImpl_, + Client_->StreamExecuteQuery(query, txControl, {}, settings, *this), + true, + Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); +} + +TAsyncExecuteQueryIterator TSession::StreamExecuteQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecuteQuerySettings& settings) +{ + return NSessionPool::InjectSessionStatusInterception( + SessionImpl_, + Client_->StreamExecuteQuery(query, txControl, params, settings, *this), + true, + Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); +} + +TAsyncBeginTransactionResult TSession::BeginTransaction(const TTxSettings& txSettings, + const TBeginTxSettings& settings) +{ + return NSessionPool::InjectSessionStatusInterception( + SessionImpl_, + Client_->BeginTransaction(txSettings, settings, *this), + true, + Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); +} + +TTransaction::TTransaction(const TSession& session, const std::string& txId) + : Session_(session) + , TxId_(txId) +{} + +TAsyncCommitTransactionResult TTransaction::Commit(const NYdb::NQuery::TCommitTxSettings& settings) { + return Session_.Client_->CommitTransaction(TxId_, settings, Session_); +} + +TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) { + return Session_.Client_->RollbackTransaction(TxId_, settings, Session_); +} + +TBeginTransactionResult::TBeginTransactionResult(TStatus&& status, TTransaction transaction) + : TStatus(std::move(status)) + , Transaction_(transaction) +{} + +const TTransaction& TBeginTransactionResult::GetTransaction() const { + CheckStatusOk("TBeginTransactionResult::GetTransaction"); + return Transaction_; +} + +const std::vector& TExecuteQueryResult::GetResultSets() const { + return ResultSets_; +} + +TResultSet TExecuteQueryResult::GetResultSet(size_t resultIndex) const { + if (resultIndex >= ResultSets_.size()) { + RaiseError(std::string("Requested index out of range\n")); + } + + return ResultSets_[resultIndex]; +} + +TResultSetParser TExecuteQueryResult::GetResultSetParser(size_t resultIndex) const { + return TResultSetParser(GetResultSet(resultIndex)); +} + +} // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/src/client/query/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/query/impl/CMakeLists.txt new file mode 100644 index 000000000000..52076f6d7004 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/CMakeLists.txt @@ -0,0 +1,16 @@ +_ydb_sdk_add_library(client-ydb_query-impl) + +target_link_libraries(client-ydb_query-impl PUBLIC + yutil + api-grpc + api-protos + client-ydb_common_client-impl + client-ydb_proto +) + +target_sources(client-ydb_query-impl PRIVATE + exec_query.cpp + client_session.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_query-impl) diff --git a/ydb/public/sdk/cpp/src/client/query/impl/client_session.cpp b/ydb/public/sdk/cpp/src/client/query/impl/client_session.cpp new file mode 100644 index 000000000000..8b1c89a8fa87 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/client_session.cpp @@ -0,0 +1,148 @@ +#include "client_session.h" + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +namespace NYdb::inline V3::NQuery { + +// Custom lock primitive to protect session from destroying +// during async read execution. +// The problem is TSession::TImpl holds grpc stream processor by IntrusivePtr +// and this processor alredy refcounted by internal code. +// That mean during TSession::TImpl dtor no gurantee to grpc procerrot will be destroyed. +// StreamProcessor_->Cancel() doesn't help it just start async cancelation but we have no way +// to wait cancelation has done. +// So we need some way to protect access to row session impl pointer +// from async reader (processor callback). We can't use shared/weak ptr here because TSessionImpl +// stores as uniq ptr inside session pool and as shared ptr in the TSession +// when user got session (see GetSmartDeleter related code). + +// Why just not std::mutex? - Requirement do not destroy a mutex while it is locked +// makes it difficult to use here. Moreover we need to allow recursive lock. + +// Why recursive lock? - In happy path we destroy session from CloseFromServer call, +// so the session dtor called from thread which already got the lock. + +// TODO: Proably we can add sync version of Cancel method in to grpc stream procesor to make sure +// no more callback will be called. + +class TSafeTSessionImplHolder { + TSession::TImpl* Ptr; + std::atomic_uint32_t Semaphore; + std::atomic OwnerThread; +public: + TSafeTSessionImplHolder(TSession::TImpl* p) + : Ptr(p) + , Semaphore(0) + {} + + TSession::TImpl* TrySharedOwning() noexcept { + auto old = Semaphore.fetch_add(1); + if (old == 0) { + OwnerThread.store(std::this_thread::get_id()); + return Ptr; + } else { + return nullptr; + } + } + + void Release() noexcept { + OwnerThread.store(std::thread::id()); + Semaphore.store(0); + } + + void WaitAndLock() noexcept { + if (OwnerThread.load() == std::this_thread::get_id()) { + return; + } + + uint32_t cur = 0; + uint32_t newVal = 1; + while (!Semaphore.compare_exchange_weak(cur, newVal, + std::memory_order_release, std::memory_order_relaxed)) { + std::this_thread::yield(); + cur = 0; + } + } +}; + +void TSession::TImpl::StartAsyncRead(TStreamProcessorPtr ptr, std::weak_ptr client, + std::shared_ptr holder) +{ + auto resp = std::make_shared(); + ptr->Read(resp.get(), [resp, ptr, client, holder](NYdbGrpc::TGrpcStatus grpcStatus) mutable { + switch (grpcStatus.GRpcStatusCode) { + case grpc::StatusCode::OK: + StartAsyncRead(ptr, client, holder); + break; + case grpc::StatusCode::OUT_OF_RANGE: { + auto impl = holder->TrySharedOwning(); + if (impl) { + impl->CloseFromServer(client); + holder->Release(); + } + break; + } + } + }); +} + +TSession::TImpl::TImpl(TStreamProcessorPtr ptr, const std::string& sessionId, const std::string& endpoint, std::weak_ptr client) + : TKqpSessionCommon(sessionId, endpoint, true) + , StreamProcessor_(ptr) + , SessionHolder(std::make_shared(this)) +{ + MarkActive(); + SetNeedUpdateActiveCounter(true); + StartAsyncRead(StreamProcessor_, client, SessionHolder); +} + +TSession::TImpl::~TImpl() +{ + StreamProcessor_->Cancel(); + SessionHolder->WaitAndLock(); +} + +void TSession::TImpl::MakeImplAsync(TStreamProcessorPtr ptr, + std::shared_ptr args) +{ + auto resp = std::make_shared(); + ptr->Read(resp.get(), [args, resp, ptr](NYdbGrpc::TGrpcStatus grpcStatus) mutable { + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) { + TStatus st(TPlainStatus(grpcStatus, args->Endpoint)); + args->Promise.SetValue(TCreateSessionResult(std::move(st), TSession())); + + } else { + if (resp->status() == Ydb::StatusIds::SUCCESS) { + NYdb::TStatus st(TPlainStatus(grpcStatus, args->Endpoint)); + TSession::TImpl::NewSmartShared(ptr, std::move(args), st); + + } else { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(resp->issues(), opIssues); + TStatus st(static_cast(resp->status()), std::move(opIssues)); + args->Promise.SetValue(TCreateSessionResult(std::move(st), TSession())); + } + } + }); +} + +void TSession::TImpl::NewSmartShared(TStreamProcessorPtr ptr, + std::shared_ptr args, NYdb::TStatus st) +{ + args->Promise.SetValue( + TCreateSessionResult( + std::move(st), + TSession( + args->Client, + new TSession::TImpl(ptr, args->SessionId, args->Endpoint, args->SessionClient) + ) + ) + ); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/query/impl/client_session.h b/ydb/public/sdk/cpp/src/client/query/impl/client_session.h new file mode 100644 index 000000000000..eb2db954fb8e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/client_session.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include + +namespace NYdb::inline V3::NQuery { + +class TSafeTSessionImplHolder; + +class TSession::TImpl : public TKqpSessionCommon { +public: + struct TAttachSessionArgs { + TAttachSessionArgs(NThreading::TPromise promise, + std::string sessionId, + std::string endpoint, + std::shared_ptr client, + std::weak_ptr sessionClient) + : Promise(promise) + , SessionId(sessionId) + , Endpoint(endpoint) + , Client(client) + , SessionClient(sessionClient) + { } + NThreading::TPromise Promise; + std::string SessionId; + std::string Endpoint; + std::shared_ptr Client; + std::weak_ptr SessionClient; + }; + + using TResponse = Ydb::Query::SessionState; + using TStreamProcessorPtr = NYdbGrpc::IStreamRequestReadProcessor::TPtr; + TImpl(TStreamProcessorPtr ptr, const std::string& id, const std::string& endpoint, std::weak_ptr client); + ~TImpl(); + + static void MakeImplAsync(TStreamProcessorPtr processor, std::shared_ptr args); + +private: + static void NewSmartShared(TStreamProcessorPtr ptr, std::shared_ptr args, NYdb::TStatus status); + + static void StartAsyncRead(TStreamProcessorPtr ptr, std::weak_ptr client, std::shared_ptr session); + +private: + TStreamProcessorPtr StreamProcessor_; + std::shared_ptr SessionHolder; +}; + +} diff --git a/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp new file mode 100644 index 000000000000..5e47a60545f6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp @@ -0,0 +1,338 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "exec_query.h" +#include "client_session.h" + +#include +#include +#include +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include + +namespace NYdb::inline V3::NQuery { + +using namespace NThreading; + +static void SetTxSettings(const TTxSettings& txSettings, Ydb::Query::TransactionSettings* proto) { + switch (txSettings.GetMode()) { + case TTxSettings::TS_SERIALIZABLE_RW: + proto->mutable_serializable_read_write(); + break; + case TTxSettings::TS_ONLINE_RO: + proto->mutable_online_read_only()->set_allow_inconsistent_reads( + txSettings.OnlineSettings_.AllowInconsistentReads_); + break; + case TTxSettings::TS_STALE_RO: + proto->mutable_stale_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RO: + proto->mutable_snapshot_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RW: + proto->mutable_snapshot_read_write(); + break; + default: + throw TContractViolation("Unexpected transaction mode."); + } +} + +class TExecuteQueryIterator::TReaderImpl { +public: + using TSelf = TExecuteQueryIterator::TReaderImpl; + using TResponse = Ydb::Query::ExecuteQueryResponsePart; + using TStreamProcessorPtr = NYdbGrpc::IStreamRequestReadProcessor::TPtr; + using TReadCallback = NYdbGrpc::IStreamRequestReadProcessor::TReadCallback; + using TGRpcStatus = NYdbGrpc::TGrpcStatus; + using TBatchReadResult = std::pair; + + TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint, const std::optional& session) + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) + , Session_(session) + {} + + ~TReaderImpl() { + StreamProcessor_->Cancel(); + } + + bool IsFinished() const { + return Finished_; + } + + TAsyncExecuteQueryPart DoReadNext(std::shared_ptr self) { + auto promise = NThreading::NewPromise(); + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + if (!grpcStatus.Ok()) { + self->Finished_ = true; + promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_)), {}, {}}); + } else { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast(self->Response_.status()); + TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; + TStatus status{std::move(plainStatus)}; + + std::optional stats; + std::optional tx; + if (self->Response_.has_exec_stats()) { + stats = TExecStats(std::move(*self->Response_.mutable_exec_stats())); + } + + if (self->Response_.has_tx_meta() && !self->Response_.tx_meta().id().empty() && self->Session_.has_value()) { + tx = TTransaction(self->Session_.value(), self->Response_.tx_meta().id()); + } + + if (self->Response_.has_result_set()) { + promise.SetValue({ + std::move(status), + TResultSet(std::move(*self->Response_.mutable_result_set())), + self->Response_.result_set_index(), + std::move(stats), + std::move(tx) + }); + } else { + promise.SetValue({std::move(status), std::move(stats), std::move(tx)}); + } + } + }; + + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); + } + + TAsyncExecuteQueryPart ReadNext(std::shared_ptr self) { + if (!Session_) + return DoReadNext(std::move(self)); + + return NSessionPool::InjectSessionStatusInterception( + Session_->SessionImpl_, + DoReadNext(std::move(self)), + false, // no need to ping stream session + TDuration::Zero()); + } + +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + std::string Endpoint_; + std::optional Session_; +}; + +TAsyncExecuteQueryPart TExecuteQueryIterator::ReadNext() { + if (ReaderImpl_->IsFinished()) { + RaiseError("Attempt to perform read on invalid or finished stream"); + } + + return ReaderImpl_->ReadNext(ReaderImpl_); +} + +using TExecuteQueryProcessorPtr = TExecuteQueryIterator::TReaderImpl::TStreamProcessorPtr; + +struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { + using TPtr = TIntrusivePtr; + + TExecuteQueryBuffer(TExecuteQueryIterator&& iterator) + : Promise_(NewPromise()) + , Iterator_(std::move(iterator)) {} + + TPromise Promise_; + TExecuteQueryIterator Iterator_; + std::vector Issues_; + std::vector ResultSets_; + std::optional Stats_; + std::optional Tx_; + + void Next() { + TPtr self(this); + + Iterator_.ReadNext().Subscribe([self](TAsyncExecuteQueryPart partFuture) mutable { + auto part = partFuture.ExtractValue(); + + if (const auto& st = part.GetStats()) { + self->Stats_ = st; + } + + if (!part.IsSuccess()) { + std::optional stats; + std::swap(self->Stats_, stats); + + if (part.EOS()) { + std::vector issues; + std::vector resultProtos; + std::optional tx; + + std::swap(self->Issues_, issues); + std::swap(self->ResultSets_, resultProtos); + std::swap(self->Tx_, tx); + + std::vector resultSets; + for (auto& proto : resultProtos) { + resultSets.emplace_back(std::move(proto)); + } + + self->Promise_.SetValue(TExecuteQueryResult( + TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues(std::move(issues))), + std::move(resultSets), + std::move(stats), + std::move(tx) + )); + } else { + self->Promise_.SetValue(TExecuteQueryResult(std::move(part), {}, std::move(stats), {})); + } + + return; + } + + self->Issues_.insert(self->Issues_.end(), part.GetIssues().begin(), part.GetIssues().end()); + + if (part.HasResultSet()) { + auto inRs = part.ExtractResultSet(); + auto& inRsProto = TProtoAccessor::GetProto(inRs); + + // TODO: Use result sets metadata + if (self->ResultSets_.size() <= part.GetResultSetIndex()) { + self->ResultSets_.resize(part.GetResultSetIndex() + 1); + } + + auto& resultSet = self->ResultSets_[part.GetResultSetIndex()]; + if (resultSet.columns().empty()) { + resultSet.mutable_columns()->CopyFrom(inRsProto.columns()); + } + + resultSet.mutable_rows()->Reserve(resultSet.mutable_rows()->size() + inRsProto.rows_size()); + for (const auto& row : inRsProto.rows()) { + *resultSet.mutable_rows()->Add() = row; + } + } + + if (const auto& tx = part.GetTransaction()) { + self->Tx_ = tx; + } + + self->Next(); + }); + } +}; + +TFuture> StreamExecuteQueryImpl( + const std::shared_ptr& connections, const TDbDriverStatePtr& driverState, + const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, + const TExecuteQuerySettings& settings, const std::optional& session) +{ + auto request = MakeRequest(); + request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); + request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); + request.set_pool_id(TStringType{settings.ResourcePool_}); + request.mutable_query_content()->set_text(TStringType{query}); + request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); + if (session.has_value()) { + request.set_session_id(TStringType{session->GetId()}); + } else if ((txControl.TxSettings_.has_value() && !txControl.CommitTx_) || txControl.TxId_.has_value()) { + throw TContractViolation("Interactive tx must use explisit session"); + } + + if (settings.ConcurrentResultSets_) { + request.set_concurrent_result_sets(*settings.ConcurrentResultSets_); + } + + if (settings.OutputChunkMaxSize_) { + request.set_response_part_limit_bytes(*settings.OutputChunkMaxSize_); + } + + if (txControl.HasTx()) { + auto requestTxControl = request.mutable_tx_control(); + requestTxControl->set_commit_tx(txControl.CommitTx_); + if (txControl.TxId_) { + requestTxControl->set_tx_id(TStringType{txControl.TxId_.value()}); + } else { + Y_ASSERT(txControl.TxSettings_); + SetTxSettings(*txControl.TxSettings_, requestTxControl->mutable_begin_tx()); + } + } else { + Y_ASSERT(!txControl.CommitTx_); + } + + if (params) { + *request.mutable_parameters() = *params; + } + + auto promise = NewPromise>(); + + auto rpcSettings = TRpcRequestSettings::Make(settings); + if (session.has_value()) { + rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(session->GetId())); + } + + connections->StartReadStream< + Ydb::Query::V1::QueryService, + Ydb::Query::ExecuteQueryRequest, + Ydb::Query::ExecuteQueryResponsePart> + ( + std::move(request), + [promise] (TPlainStatus status, TExecuteQueryProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Query::V1::QueryService::Stub::AsyncExecuteQuery, + driverState, + rpcSettings + ); + + return promise.GetFuture(); +} + +TAsyncExecuteQueryIterator TExecQueryImpl::StreamExecuteQuery(const std::shared_ptr& connections, + const TDbDriverStatePtr& driverState, const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session) +{ + auto promise = NewPromise(); + + auto iteratorCallback = [promise, session](TFuture> future) mutable { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TExecuteQueryIterator( + pair.second + ? std::make_shared(pair.second, pair.first.Endpoint, session) + : nullptr, + std::move(pair.first)) + ); + }; + + auto paramsProto = params + ? ¶ms->GetProtoMap() + : nullptr; + + StreamExecuteQueryImpl(connections, driverState, query, txControl, paramsProto, settings, session) + .Subscribe(iteratorCallback); + return promise.GetFuture(); +} + +TAsyncExecuteQueryResult TExecQueryImpl::ExecuteQuery(const std::shared_ptr& connections, + const TDbDriverStatePtr& driverState, const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session) +{ + auto syncSettings = settings; + syncSettings.ConcurrentResultSets(true); + + return StreamExecuteQuery(connections, driverState, query, txControl, params, syncSettings, session) + .Apply([](TAsyncExecuteQueryIterator itFuture){ + auto it = itFuture.ExtractValue(); + + if (!it.IsSuccess()) { + return MakeFuture(std::move(it)); + } + + auto buffer = MakeIntrusive(std::move(it)); + buffer->Next(); + + return buffer->Promise_.GetFuture(); + }); +} + +} // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/src/client/query/impl/exec_query.h b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.h new file mode 100644 index 000000000000..b20246e84beb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace NYdb::inline V3::NQuery { + +class TExecQueryImpl { +public: + static TAsyncExecuteQueryIterator StreamExecuteQuery(const std::shared_ptr& connections, + const TDbDriverStatePtr& driverState, const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session); + + static TAsyncExecuteQueryResult ExecuteQuery(const std::shared_ptr& connections, + const TDbDriverStatePtr& driverState, const std::string& query, const TTxControl& txControl, + const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session); +}; + +} // namespace NYdb::NQuery::NImpl diff --git a/ydb/public/sdk/cpp/src/client/query/impl/ya.make b/ydb/public/sdk/cpp/src/client/query/impl/ya.make new file mode 100644 index 000000000000..777328dbd00d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/impl/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + exec_query.cpp + exec_query.h + client_session.cpp +) + +PEERDIR( + ydb/public/api/grpc/draft + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/proto +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/query/query.cpp b/ydb/public/sdk/cpp/src/client/query/query.cpp new file mode 100644 index 000000000000..def91a0369d9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/query.cpp @@ -0,0 +1,69 @@ +#include + +#include + +namespace NYdb::inline V3::NQuery { + +std::optional ParseStatsMode(std::string_view statsMode) { + if (statsMode == "unspecified") { + return EStatsMode::Unspecified; + } else if (statsMode == "none") { + return EStatsMode::None; + } else if (statsMode == "basic") { + return EStatsMode::Basic; + } else if (statsMode == "full") { + return EStatsMode::Full; + } else if (statsMode == "profile") { + return EStatsMode::Profile; + } + + return {}; +} + +std::string_view StatsModeToString(const EStatsMode statsMode) { + switch (statsMode) { + case EStatsMode::Unspecified: + return "unspecified"; + case EStatsMode::None: + return "none"; + case EStatsMode::Basic: + return "basic"; + case EStatsMode::Full: + return "full"; + case EStatsMode::Profile: + return "profile"; + } +} + +TScriptExecutionOperation::TScriptExecutionOperation(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + Ydb::Query::ExecuteScriptMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + Metadata_.ExecutionId = metadata.execution_id(); + Metadata_.ExecMode = static_cast(metadata.exec_mode()); + Metadata_.ExecStatus = static_cast(metadata.exec_status()); + Metadata_.ExecStats = TExecStats(std::move(*metadata.mutable_exec_stats())); + + Metadata_.ResultSetsMeta.reserve(metadata.result_sets_meta_size()); + for (const auto& resultSetMeta : metadata.result_sets_meta()) { + std::vector columns; + columns.reserve(resultSetMeta.columns_size()); + for (const auto& column : resultSetMeta.columns()) { + columns.emplace_back(column.name(), column.type()); + } + Metadata_.ResultSetsMeta.emplace_back(std::move(columns)); + } + + if (metadata.has_script_content()) { + Metadata_.ScriptContent.Syntax = static_cast(metadata.script_content().syntax()); + Metadata_.ScriptContent.Text = metadata.script_content().text(); + } +} + +TCommitTransactionResult::TCommitTransactionResult(TStatus&& status) + : TStatus(std::move(status)) +{} + +} // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/src/client/query/stats.cpp b/ydb/public/sdk/cpp/src/client/query/stats.cpp new file mode 100644 index 000000000000..e56d5533f627 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/stats.cpp @@ -0,0 +1,73 @@ +#include + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3::NQuery { + +class TExecStats::TImpl { +public: + Ydb::TableStats::QueryStats Proto; +}; + +TExecStats::TExecStats(const Ydb::TableStats::QueryStats& proto) { + Impl_ = std::make_shared(); + Impl_->Proto = proto; +} + +TExecStats::TExecStats(Ydb::TableStats::QueryStats&& proto) { + Impl_ = std::make_shared(); + Impl_->Proto = std::move(proto); +} + +std::string TExecStats::ToString(bool withPlan) const { + auto proto = Impl_->Proto; + + if (!withPlan) { + proto.clear_query_plan(); + proto.clear_query_ast(); + } + + TStringType res; + ::google::protobuf::TextFormat::PrintToString(proto, &res); + return res; +} + +std::optional TExecStats::GetPlan() const { + auto proto = Impl_->Proto; + + if (proto.query_plan().empty()) { + return {}; + } + + return proto.query_plan(); +} + +std::optional TExecStats::GetAst() const { + auto proto = Impl_->Proto; + + if (proto.query_ast().empty()) { + return {}; + } + + return proto.query_ast(); +} + +TDuration TExecStats::GetTotalDuration() const { + return TDuration::MicroSeconds(Impl_->Proto.total_duration_us()); +} + +TDuration TExecStats::GetTotalCpuTime() const { + return TDuration::MicroSeconds(Impl_->Proto.total_cpu_time_us()); +} + +const Ydb::TableStats::QueryStats& TExecStats::GetProto() const { + return Impl_->Proto; +} + +} // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/src/client/query/tx.cpp b/ydb/public/sdk/cpp/src/client/query/tx.cpp new file mode 100644 index 000000000000..1db116f0d90d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/tx.cpp @@ -0,0 +1 @@ +#include diff --git a/ydb/public/sdk/cpp/src/client/query/ya.make b/ydb/public/sdk/cpp/src/client/query/ya.make new file mode 100644 index 000000000000..e46846114304 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/query/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + client.cpp + query.cpp + stats.cpp + tx.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common + ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool + ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry + ydb/public/sdk/cpp/src/client/common_client + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/query/impl + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/rate_limiter/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/rate_limiter/CMakeLists.txt new file mode 100644 index 000000000000..a5599378b80c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/rate_limiter/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(client-ydb_rate_limiter) + +target_link_libraries(client-ydb_rate_limiter PUBLIC + yutil + api-grpc + client-ydb_common_client-impl + client-ydb_driver +) + +target_sources(client-ydb_rate_limiter PRIVATE + rate_limiter.cpp +) + +_ydb_sdk_make_client_component(RateLimiter client-ydb_rate_limiter) diff --git a/ydb/public/sdk/cpp/src/client/rate_limiter/rate_limiter.cpp b/ydb/public/sdk/cpp/src/client/rate_limiter/rate_limiter.cpp new file mode 100644 index 000000000000..46dba55ba0e1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/rate_limiter/rate_limiter.cpp @@ -0,0 +1,215 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +namespace NYdb::inline V3::NRateLimiter { + +TListResourcesResult::TListResourcesResult(TStatus status, std::vector paths) + : TStatus(std::move(status)) + , ResourcePaths_(std::move(paths)) +{ +} + +TDescribeResourceResult::TDescribeResourceResult(TStatus status, const Ydb::RateLimiter::DescribeResourceResult& result) + : TStatus(std::move(status)) + , ResourcePath_(result.resource().resource_path()) + , HierarchicalDrrProps_(result.resource().hierarchical_drr()) +{ +} + +TDescribeResourceResult::THierarchicalDrrProps::THierarchicalDrrProps(const Ydb::RateLimiter::HierarchicalDrrSettings& settings) { + if (settings.max_units_per_second()) { + MaxUnitsPerSecond_ = settings.max_units_per_second(); + } + + if (settings.max_burst_size_coefficient()) { + MaxBurstSizeCoefficient_ = settings.max_burst_size_coefficient(); + } + + if (settings.prefetch_coefficient()) { + PrefetchCoefficient_ = settings.prefetch_coefficient(); + } + + if (settings.prefetch_watermark()) { + PrefetchWatermark_ = settings.prefetch_watermark(); + } +} + +class TRateLimiterClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + template + static TRequest MakePropsCreateOrAlterRequest(const std::string& coordinationNodePath, const std::string& resourcePath, const TSettings& settings) { + TRequest request = MakeOperationRequest(settings); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + + Ydb::RateLimiter::Resource& resource = *request.mutable_resource(); + resource.set_resource_path(TStringType{resourcePath}); + + Ydb::RateLimiter::HierarchicalDrrSettings& hdrr = *resource.mutable_hierarchical_drr(); + if (settings.MaxUnitsPerSecond_) { + hdrr.set_max_units_per_second(*settings.MaxUnitsPerSecond_); + } + if (settings.MaxBurstSizeCoefficient_) { + hdrr.set_max_burst_size_coefficient(*settings.MaxBurstSizeCoefficient_); + } + if (settings.PrefetchCoefficient_) { + hdrr.set_prefetch_coefficient(*settings.PrefetchCoefficient_); + } + if (settings.PrefetchWatermark_) { + hdrr.set_prefetch_watermark(*settings.PrefetchWatermark_); + } + + return request; + } + + TAsyncStatus CreateResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TCreateResourceSettings& settings) { + auto request = MakePropsCreateOrAlterRequest(coordinationNodePath, resourcePath, settings); + + return RunSimple( + std::move(request), + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncCreateResource, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AlterResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAlterResourceSettings& settings) { + auto request = MakePropsCreateOrAlterRequest(coordinationNodePath, resourcePath, settings); + + return RunSimple( + std::move(request), + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncAlterResource, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus DropResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDropResourceSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); + + return RunSimple( + std::move(request), + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncDropResource, + TRpcRequestSettings::Make(settings)); + } + + TAsyncListResourcesResult ListResources(const std::string& coordinationNodePath, const std::string& resourcePath, const TListResourcesSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); + request.set_recursive(settings.Recursive_); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector list; + if (any) { + Ydb::RateLimiter::ListResourcesResult result; + any->UnpackTo(&result); + list.reserve(result.resource_paths_size()); + for (const std::string& path : result.resource_paths()) { + list.push_back(path); + } + } + + TListResourcesResult val(TStatus(std::move(status)), std::move(list)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncListResources, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncDescribeResourceResult DescribeResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDescribeResourceSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::RateLimiter::DescribeResourceResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeResourceResult val(TStatus(std::move(status)), result); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncDescribeResource, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncStatus AcquireResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAcquireResourceSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); + + if (settings.IsUsedAmount_) { + request.set_used(settings.Amount_.value()); + } else { + request.set_required(settings.Amount_.value()); + } + + return RunSimple( + std::move(request), + &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncAcquireResource, + TRpcRequestSettings::Make(settings)); + } +}; + +TRateLimiterClient::TRateLimiterClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncStatus TRateLimiterClient::CreateResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TCreateResourceSettings& settings) { + return Impl_->CreateResource(coordinationNodePath, resourcePath, settings); +} + +TAsyncStatus TRateLimiterClient::AlterResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAlterResourceSettings& settings) { + return Impl_->AlterResource(coordinationNodePath, resourcePath, settings); +} + +TAsyncStatus TRateLimiterClient::DropResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDropResourceSettings& settings) { + return Impl_->DropResource(coordinationNodePath, resourcePath, settings); +} + +TAsyncListResourcesResult TRateLimiterClient::ListResources(const std::string& coordinationNodePath, const std::string& resourcePath, const TListResourcesSettings& settings) { + return Impl_->ListResources(coordinationNodePath, resourcePath, settings); +} + +TAsyncDescribeResourceResult TRateLimiterClient::DescribeResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TDescribeResourceSettings& settings) { + return Impl_->DescribeResource(coordinationNodePath, resourcePath, settings); +} + +TAsyncStatus TRateLimiterClient::AcquireResource(const std::string& coordinationNodePath, const std::string& resourcePath, const TAcquireResourceSettings& settings) { + return Impl_->AcquireResource(coordinationNodePath, resourcePath, settings); +} + +} // namespace NYdb::NRateLimiter diff --git a/ydb/public/sdk/cpp/src/client/rate_limiter/ya.make b/ydb/public/sdk/cpp/src/client/rate_limiter/ya.make new file mode 100644 index 000000000000..6b5a90872c2d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/rate_limiter/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + rate_limiter.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/resources/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/resources/CMakeLists.txt new file mode 100644 index 000000000000..3074324f62f6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/CMakeLists.txt @@ -0,0 +1,29 @@ +_ydb_sdk_add_library(client-resources) +target_link_libraries(client-resources PUBLIC + yutil + resource +) +target_sources(client-resources PRIVATE + ydb_resources.cpp + ydb_ca.cpp +) + +add_global_library_for(client-resources.global client-resources) +target_link_libraries(client-resources.global PUBLIC + yutil + resource +) +target_sources(client-resources.global PRIVATE + ${YDB_SDK_BINARY_DIR}/src/client/resources/6ed212bf45019efe2a5e72b6d5ed50fb.cpp +) +resources(client-resources.global + ${YDB_SDK_BINARY_DIR}/src/client/resources/6ed212bf45019efe2a5e72b6d5ed50fb.cpp + INPUTS + ${YDB_SDK_SOURCE_DIR}/src/client/resources/ydb_sdk_version.txt + ${YDB_SDK_SOURCE_DIR}/src/client/resources/ydb_root_ca.pem + KEYS + ydb_sdk_version_v3.txt + ydb_root_ca_v3.pem +) + +_ydb_sdk_make_client_component(Resources client-resources client-resources.global) diff --git a/ydb/public/sdk/cpp/src/client/resources/ya.make b/ydb/public/sdk/cpp/src/client/resources/ya.make new file mode 100644 index 000000000000..b593c84bca07 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + ydb_resources.cpp + ydb_ca.cpp +) + +RESOURCE( + ydb/public/sdk/cpp/src/client/resources/ydb_sdk_version.txt ydb_sdk_version_v3.txt + ydb/public/sdk/cpp/src/client/resources/ydb_root_ca.pem ydb_root_ca_v3.pem +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/resources/ydb_ca.cpp b/ydb/public/sdk/cpp/src/client/resources/ydb_ca.cpp new file mode 100644 index 000000000000..96ff0618bbdc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/ydb_ca.cpp @@ -0,0 +1,11 @@ +#include + +#include + +namespace NYdb::inline V3 { + +std::string GetRootCertificate() { + return NResource::Find("ydb_root_ca_v3.pem"); +} + +} // namespace NYdb \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/resources/ydb_resources.cpp b/ydb/public/sdk/cpp/src/client/resources/ydb_resources.cpp new file mode 100644 index 000000000000..22a7bdd39a5c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/ydb_resources.cpp @@ -0,0 +1,35 @@ +#include + +#include + +namespace NYdb::inline V3 { + +const char* YDB_AUTH_TICKET_HEADER = "x-ydb-auth-ticket"; +const char* YDB_DATABASE_HEADER = "x-ydb-database"; +const char* YDB_TRACE_ID_HEADER = "x-ydb-trace-id"; +const char* OTEL_TRACE_HEADER = "traceparent"; // https://w3c.github.io/trace-context/#header-name +const char* YDB_SDK_BUILD_INFO_HEADER = "x-ydb-sdk-build-info"; +const char* YDB_APPLICATION_NAME = "x-ydb-application-name"; +const char* YDB_CLIENT_PID = "x-ydb-client-pid"; +const char* YDB_REQUEST_TYPE_HEADER = "x-ydb-request-type"; +const char* YDB_CONSUMED_UNITS_HEADER = "x-ydb-consumed-units"; + +// The x-ydb-server-hints header. +// This header can be sent in the trailing metadata with response. +// The only possible value is "session-close". In this case client should gracefully close the session, where the request was executed. +const char* YDB_SERVER_HINTS = "x-ydb-server-hints"; + +// The message that server can send in trailing metadata to client in order to gracefully shutdown the session. +const char* YDB_SESSION_CLOSE = "session-close"; +// Feature compabilities. +// The client should send a feature capability-header in order to enable a feature on the server side. +// Send ("x-ydb-client-capabilities", "session-balancer") pair in metadata with a СreateSession request to enable server side session balancing feature. +const char* YDB_CLIENT_CAPABILITIES = "x-ydb-client-capabilities"; +const char* YDB_CLIENT_CAPABILITY_SESSION_BALANCER = "session-balancer"; + + +std::string GetSdkSemver() { + return NResource::Find("ydb_sdk_version_v3.txt"); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/resources/ydb_root_ca.pem b/ydb/public/sdk/cpp/src/client/resources/ydb_root_ca.pem new file mode 100644 index 000000000000..b424dfe50a54 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/ydb_root_ca.pem @@ -0,0 +1,4738 @@ +-----BEGIN CERTIFICATE----- +MIIE3TCCAsWgAwIBAgIKPxb5sAAAAAAAFzANBgkqhkiG9w0BAQ0FADAfMR0wGwYD +VQQDExRZYW5kZXhJbnRlcm5hbFJvb3RDQTAeFw0xNzA2MjAxNjQ0MzdaFw0yNzA2 +MjAxNjU0MzdaMFUxEjAQBgoJkiaJk/IsZAEZFgJydTEWMBQGCgmSJomT8ixkARkW +BnlhbmRleDESMBAGCgmSJomT8ixkARkWAmxkMRMwEQYDVQQDEwpZYW5kZXhDTENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqgNnjk0JKPcbsk1+KG2t +eM1AfMnEe5RkAJuBBuwVV49snhcvO1jhKBx/pCnjr6biICc1/oAFDVgU8yVYYPwp +WZ2vH3ZtscjJ/RAT/NS9OKKG7kKknhFhVYxua5xhoIQmm6usBNYYiTcWoFm1eHC8 +I9oddOLSscZYbh3unVRvt+3V+drVmUx9oSUKpqMgfysiv1MN6zB3vq9TFkbhz53E +k0tEcV+W2NnDaeFhLKy284FDKLvOdTDj1EDsSAihxl7sNEKpupNuhgyy2siOqUb+ +d5mO/CRfaAKGg3E6hDM3pEi48E506dJdjPXWfHKSvuguMLRlb2RWdVocRZuyWxOh +0QIDAQABo4HkMIHhMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRMU5uItjx+ +TOicX1+ovC1Xq2PSnzAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8E +BAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSrucX/oe/mUx0zOSKE +0XbUN04tajBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vY3Jscy55YW5kZXgucnUv +WWFuZGV4SW50ZXJuYWxSb290Q0EvWWFuZGV4SW50ZXJuYWxSb290Q0EuY3JsMA0G +CSqGSIb3DQEBDQUAA4ICAQAsR5Lb4Pv2FD0Kk+4oc1GEOnehxKLsQtdV81nrU+IV +l9pr2oNMdi8lwIolvHZRllLM4Ba5AcRH6YJ5fe7AjKm+5EdSkhqVWo2UOllRCbtS +wmL50+erOAkxstSlRkO6b8x1L0MOBKv54E5YcQ/Wwt27ldSb6RkEmJBGvmxObAaf +5zc51pqSqao9tnldYaCblEQ/Zmy43FliIpa2eUJoh8DqK8bVo2gcI3wbQ32tWs9u +wvKk8fo4lAdhCwhv+QHuqau1VAY9hPU106bsFIDUmijTMxjAobKBi6CkIX6EbNHU +Jv4DzYVLlDd2y0CADdn2F6I70xpCBn5cquSGuvFbqZjQDmIHwb7WQSxadkiGRWfc +zVTnmiHjJONJJIpE2t+FOV3hc+8o98OzOtNaH2QQ9j6dnKvtIGKGFeNSDp0vXPOi +QhHiIyuB7eWx+g2whktQ74UCpGDSXYnEW3s8w5wezVWIEmouq7q4rCEkTNvJ7Ico +43AgUdPzAFS2zYktw1C+cbUALM8smvXbXrXOBzMmscjIhtXvLMrpPeh23VfdJfQB +0rN2BmRCLUE8JOV+o0k98XMm83oN+lGkL1l+hyoj3ok1uI3JrsWOcDyjOds3ptcN +KimJLm27ndjcxDNo/iA6gefMJuCxFRaqI+eF4P0jSkMgnnQqZkvLGFuHCw8eRDhm +bw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFGTCCAwGgAwIBAgIQJMM7ZIy2SYxCBgK7WcFwnjANBgkqhkiG9w0BAQ0FADAf +MR0wGwYDVQQDExRZYW5kZXhJbnRlcm5hbFJvb3RDQTAeFw0xMzAyMTExMzQxNDNa +Fw0zMzAyMTExMzUxNDJaMB8xHTAbBgNVBAMTFFlhbmRleEludGVybmFsUm9vdENB +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAgb4xoQjBQ7oEFk8EHVGy +1pDEmPWw0Wgw5nX9RM7LL2xQWyUuEq+Lf9Dgh+O725aZ9+SO2oEs47DHHt81/fne +5N6xOftRrCpy8hGtUR/A3bvjnQgjs+zdXvcO9cTuuzzPTFSts/iZATZsAruiepMx +SGj9S1fGwvYws/yiXWNoNBz4Tu1Tlp0g+5fp/ADjnxc6DqNk6w01mJRDbx+6rlBO +aIH2tQmJXDVoFdrhmBK9qOfjxWlIYGy83TnrvdXwi5mKTMtpEREMgyNLX75UjpvO +NkZgBvEXPQq+g91wBGsWIE2sYlguXiBniQgAJOyRuSdTxcJoG8tZkLDPRi5RouWY +gxXr13edn1TRDGco2hkdtSUBlajBMSvAq+H0hkslzWD/R+BXkn9dh0/DFnxVt4XU +5JbFyd/sKV/rF4Vygfw9ssh1ZIWdqkfZ2QXOZ2gH4AEeoN/9vEfUPwqPVzL0XEZK +r4s2WjU9mE5tHrVsQOZ80wnvYHYi2JHbl0hr5ghs4RIyJwx6LEEnj2tzMFec4f7o +dQeSsZpgRJmpvpAfRTxhIRjZBrKxnMytedAkUPguBQwjVCn7+EaKiJfpu42JG8Mm ++/dHi+Q9Tc+0tX5pKOIpQMlMxMHw8MfPmUjC3AAd9lsmCtuybYoeN2IRdbzzchJ8 +l1ZuoI3gH7pcIeElfVSqSBkCAwEAAaNRME8wCwYDVR0PBAQDAgGGMA8GA1UdEwEB +/wQFMAMBAf8wHQYDVR0OBBYEFKu5xf+h7+ZTHTM5IoTRdtQ3Ti1qMBAGCSsGAQQB +gjcVAQQDAgEAMA0GCSqGSIb3DQEBDQUAA4ICAQAVpyJ1qLjqRLC34F1UXkC3vxpO +nV6WgzpzA+DUNog4Y6RhTnh0Bsir+I+FTl0zFCm7JpT/3NP9VjfEitMkHehmHhQK +c7cIBZSF62K477OTvLz+9ku2O/bGTtYv9fAvR4BmzFfyPDoAKOjJSghD1p/7El+1 +eSjvcUBzLnBUtxO/iYXRNo7B3+1qo4F5Hz7rPRLI0UWW/0UAfVCO2fFtyF6C1iEY +/q0Ldbf3YIaMkf2WgGhnX9yH/8OiIij2r0LVNHS811apyycjep8y/NkG4q1Z9jEi +VEX3P6NEL8dWtXQlvlNGMcfDT3lmB+tS32CPEUwce/Ble646rukbERRwFfxXojpf +C6ium+LtJc7qnK6ygnYF4D6mz4H+3WaxJd1S1hGQxOb/3WVw63tZFnN62F6/nc5g +6T44Yb7ND6y3nVcygLpbQsws6HsjX65CoSjrrPn0YhKxNBscF7M7tLTW/5LK9uhk +yjRCkJ0YagpeLxfV1l1ZJZaTPZvY9+ylHnWHhzlq0FzcrooSSsp4i44DB2K7O2ID +87leymZkKUY6PMDa4GkDJx0dG4UXDhRETMf+NkYgtLJ+UIzMNskwVDcxO4kVL+Hi +Pj78bnC5yCw8P5YylR45LdxLzLO68unoXOyFz1etGXzszw8lJI9LNubYxk77mK8H +LpuQKbSbIERsmR+QqQ== +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Subject: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Label: "GlobalSign Root CA" +# Serial: 4835703278459707669005204 +# MD5 Fingerprint: 3e:45:52:15:09:51:92:e1:b7:5d:37:9f:b1:87:29:8a +# SHA1 Fingerprint: b1:bc:96:8b:d4:f4:9d:62:2a:a8:9a:81:f2:15:01:52:a4:1d:82:9c +# SHA256 Fingerprint: eb:d4:10:40:e4:bb:3e:c7:42:c9:e3:81:d3:1e:f2:a4:1a:48:b6:68:5c:96:e7:ce:f3:c1:df:6c:d4:33:1c:99 +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG +A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv +b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw +MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i +YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT +aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ +jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp +xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp +1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG +snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ +U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 +9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B +AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz +yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE +38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP +AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad +DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME +HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Label: "GlobalSign Root CA - R2" +# Serial: 4835703278459682885658125 +# MD5 Fingerprint: 94:14:77:7e:3e:5e:fd:8f:30:bd:41:b0:cf:e7:d0:30 +# SHA1 Fingerprint: 75:e0:ab:b6:13:85:12:27:1c:04:f8:5f:dd:de:38:e4:b7:24:2e:fe +# SHA256 Fingerprint: ca:42:dd:41:74:5f:d0:b8:1e:b9:02:36:2c:f9:d8:bf:71:9d:a1:bd:1b:1e:fc:94:6f:5b:4c:99:f4:2c:1b:9e +-----BEGIN CERTIFICATE----- +MIIDujCCAqKgAwIBAgILBAAAAAABD4Ym5g0wDQYJKoZIhvcNAQEFBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjIxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDYxMjE1MDgwMDAwWhcNMjExMjE1 +MDgwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAKbPJA6+Lm8omUVCxKs+IVSbC9N/hHD6ErPL +v4dfxn+G07IwXNb9rfF73OX4YJYJkhD10FPe+3t+c4isUoh7SqbKSaZeqKeMWhG8 +eoLrvozps6yWJQeXSpkqBy+0Hne/ig+1AnwblrjFuTosvNYSuetZfeLQBoZfXklq +tTleiDTsvHgMCJiEbKjNS7SgfQx5TfC4LcshytVsW33hoCmEofnTlEnLJGKRILzd +C9XZzPnqJworc5HGnRusyMvo4KD0L5CLTfuwNhv2GXqF4G3yYROIXJ/gkwpRl4pa +zq+r1feqCapgvdzZX99yqWATXgAByUr6P6TqBwMhAo6CygPCm48CAwEAAaOBnDCB +mTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUm+IH +V2ccHsBqBt5ZtJot39wZhi4wNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5n +bG9iYWxzaWduLm5ldC9yb290LXIyLmNybDAfBgNVHSMEGDAWgBSb4gdXZxwewGoG +3lm0mi3f3BmGLjANBgkqhkiG9w0BAQUFAAOCAQEAmYFThxxol4aR7OBKuEQLq4Gs +J0/WwbgcQ3izDJr86iw8bmEbTUsp9Z8FHSbBuOmDAGJFtqkIk7mpM0sYmsL4h4hO +291xNBrBVNpGP+DTKqttVCL1OmLNIG+6KYnX3ZHu01yiPqFbQfXf5WRDLenVOavS +ot+3i9DAgBkcRcAtjOj4LaR0VknFBbVPFd5uRHg5h6h+u/N5GJG79G+dwfCMNYxd +AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7 +TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg== +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Label: "Verisign Class 3 Public Primary Certification Authority - G3" +# Serial: 206684696279472310254277870180966723415 +# MD5 Fingerprint: cd:68:b6:a7:c7:c4:ce:75:e0:1d:4f:57:44:61:92:09 +# SHA1 Fingerprint: 13:2d:0d:45:53:4b:69:97:cd:b2:d5:c3:39:e2:55:76:60:9b:5c:c6 +# SHA256 Fingerprint: eb:04:cf:5e:b1:f3:9a:fa:76:2f:2b:b1:20:f2:96:cb:a5:20:c1:b9:7d:b1:58:95:65:b8:1c:b9:a1:7b:72:44 +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b +N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t +KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu +kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm +CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ +Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu +imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te +2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe +DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC +/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p +F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt +TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== +-----END CERTIFICATE----- + +# Issuer: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Subject: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Label: "Entrust.net Premium 2048 Secure Server CA" +# Serial: 946069240 +# MD5 Fingerprint: ee:29:31:bc:32:7e:9a:e6:e8:b5:f7:51:b4:34:71:90 +# SHA1 Fingerprint: 50:30:06:09:1d:97:d4:f5:ae:39:f7:cb:e7:92:7d:7d:65:2d:34:31 +# SHA256 Fingerprint: 6d:c4:71:72:e0:1c:bc:b0:bf:62:58:0d:89:5f:e2:b8:ac:9a:d4:f8:73:80:1e:0c:10:b9:c8:37:d2:1e:b1:77 +-----BEGIN CERTIFICATE----- +MIIEKjCCAxKgAwIBAgIEOGPe+DANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChML +RW50cnVzdC5uZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBp +bmNvcnAuIGJ5IHJlZi4gKGxpbWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5 +IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNVBAMTKkVudHJ1c3QubmV0IENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQxNzUwNTFaFw0yOTA3 +MjQxNDE1MTJaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3d3d3 +LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxp +YWIuKTElMCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEG +A1UEAxMqRW50cnVzdC5uZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgp +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArU1LqRKGsuqjIAcVFmQq +K0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOLGp18EzoOH1u3Hs/lJBQe +sYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSrhRSGlVuX +MlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVT +XTzWnLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/ +HoZdenoVve8AjhUiVBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH +4QIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUVeSB0RGAvtiJuQijMfmhJAkWuXAwDQYJKoZIhvcNAQEFBQADggEBADub +j1abMOdTmXx6eadNl9cZlZD7Bh/KM3xGY4+WZiT6QBshJ8rmcnPyT/4xmf3IDExo +U8aAghOY+rat2l098c5u9hURlIIM7j+VrxGrD9cv3h8Dj1csHsm7mhpElesYT6Yf +zX1XEC+bBAlahLVu2B064dae0Wx5XnkcFMXj0EyTO2U87d89vqbllRrDtRnDvV5b +u/8j72gZyxKTJ1wDLW8w0B62GqzeWvfRqqgnpv55gcR5mTNXuhKwqeBCbJPKVt7+ +bYQLCIt+jerXmCHG8+c8eS9enNFMFY3h7CI3zJpDC5fcgJCNs2ebb0gIFVbPv/Er +fF6adulZkMV8gzURZVE= +-----END CERTIFICATE----- + +# Issuer: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Subject: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Label: "Baltimore CyberTrust Root" +# Serial: 33554617 +# MD5 Fingerprint: ac:b6:94:a5:9c:17:e0:d7:91:52:9b:b1:97:06:a6:e4 +# SHA1 Fingerprint: d4:de:20:d0:5e:66:fc:53:fe:1a:50:88:2c:78:db:28:52:ca:e4:74 +# SHA256 Fingerprint: 16:af:57:a9:f6:76:b0:ab:12:60:95:aa:5e:ba:de:f2:2a:b3:11:19:d6:44:ac:95:cd:4b:93:db:f3:f2:6a:eb +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ +RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD +VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX +DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y +ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy +VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr +mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr +IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK +mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu +XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy +dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye +jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 +BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 +DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 +9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx +jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 +Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz +ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS +R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp +-----END CERTIFICATE----- + +# Issuer: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network +# Subject: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network +# Label: "AddTrust External Root" +# Serial: 1 +# MD5 Fingerprint: 1d:35:54:04:85:78:b0:3f:42:42:4d:bf:20:73:0a:3f +# SHA1 Fingerprint: 02:fa:f3:e2:91:43:54:68:60:78:57:69:4d:f5:e4:5b:68:85:18:68 +# SHA256 Fingerprint: 68:7f:a4:51:38:22:78:ff:f0:c8:b1:1f:8d:43:d5:76:67:1c:6e:b2:bc:ea:b4:13:fb:83:d9:65:d0:6d:2f:f2 +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFs +IFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290 +MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFowbzELMAkGA1UEBhMCU0Ux +FDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRUcnVzdCBFeHRlcm5h +bCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0EgUm9v +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvt +H7xsD821+iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9 +uMq/NzgtHj6RQa1wVsfwTz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzX +mk6vBbOmcZSccbNQYArHE504B4YCqOmoaSYYkKtMsE8jqzpPhNjfzp/haW+710LX +a0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy2xSoRcRdKn23tNbE7qzN +E0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv77+ldU9U0 +WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYD +VR0PBAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0 +Jvf6xCZU7wO94CTLVBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRU +cnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsx +IjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENBIFJvb3SCAQEwDQYJKoZIhvcN +AQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZlj7DYd7usQWxH +YINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5 +6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvC +Nr4TDea9Y355e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEX +c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a +mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ= +-----END CERTIFICATE----- + +# Issuer: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Subject: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Label: "Entrust Root Certification Authority" +# Serial: 1164660820 +# MD5 Fingerprint: d6:a5:c3:ed:5d:dd:3e:00:c1:3d:87:92:1f:1d:3f:e4 +# SHA1 Fingerprint: b3:1e:b1:b7:40:e3:6c:84:02:da:dc:37:d4:4d:f5:d4:67:49:52:f9 +# SHA256 Fingerprint: 73:c1:76:43:4f:1b:c6:d5:ad:f4:5b:0e:76:e7:27:28:7c:8d:e5:76:16:c1:e6:e6:14:1a:2b:2c:bc:7d:8e:4c +-----BEGIN CERTIFICATE----- +MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0 +Lm5ldC9DUFMgaXMgaW5jb3Jwb3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMW +KGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsGA1UEAxMkRW50cnVzdCBSb290IENl +cnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0MloXDTI2MTEyNzIw +NTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMTkw +NwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSBy +ZWZlcmVuY2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNV +BAMTJEVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALaVtkNC+sZtKm9I35RMOVcF7sN5EUFo +Nu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYszA9u3g3s+IIRe7bJWKKf4 +4LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOwwCj0Yzfv9 +KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGI +rb68j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi +94DkZfs0Nw4pgHBNrziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOB +sDCBrTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAi +gA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1MzQyWjAfBgNVHSMEGDAWgBRo +kORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DHhmak8fdLQ/uE +vW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA +A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9t +O1KzKtvn1ISMY/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6Zua +AGAT/3B+XxFNSRuzFVJ7yVTav52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP +9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTSW3iDVuycNsMm4hH2Z0kdkquM++v/ +eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m +0vdXcDazv/wor3ElhVsT/h5/WrQ8 +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Global CA O=GeoTrust Inc. +# Subject: CN=GeoTrust Global CA O=GeoTrust Inc. +# Label: "GeoTrust Global CA" +# Serial: 144470 +# MD5 Fingerprint: f7:75:ab:29:fb:51:4e:b7:77:5e:ff:05:3c:99:8e:f5 +# SHA1 Fingerprint: de:28:f4:a4:ff:e5:b9:2f:a3:c5:03:d1:a3:49:a7:f9:96:2a:82:12 +# SHA256 Fingerprint: ff:85:6a:2d:25:1d:cd:88:d3:66:56:f4:50:12:67:98:cf:ab:aa:de:40:79:9c:72:2d:e4:d2:b5:db:36:a7:3a +-----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Universal CA O=GeoTrust Inc. +# Subject: CN=GeoTrust Universal CA O=GeoTrust Inc. +# Label: "GeoTrust Universal CA" +# Serial: 1 +# MD5 Fingerprint: 92:65:58:8b:a2:1a:31:72:73:68:5c:b4:a5:7a:07:48 +# SHA1 Fingerprint: e6:21:f3:35:43:79:05:9a:4b:68:30:9d:8a:2f:74:22:15:87:ec:79 +# SHA256 Fingerprint: a0:45:9b:9f:63:b2:25:59:f5:fa:5d:4c:6d:b3:f9:f7:2f:f1:93:42:03:35:78:f0:73:bf:1d:1b:46:cb:b9:12 +-----BEGIN CERTIFICATE----- +MIIFaDCCA1CgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJVUzEW +MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEeMBwGA1UEAxMVR2VvVHJ1c3QgVW5pdmVy +c2FsIENBMB4XDTA0MDMwNDA1MDAwMFoXDTI5MDMwNDA1MDAwMFowRTELMAkGA1UE +BhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xHjAcBgNVBAMTFUdlb1RydXN0 +IFVuaXZlcnNhbCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKYV +VaCjxuAfjJ0hUNfBvitbtaSeodlyWL0AG0y/YckUHUWCq8YdgNY96xCcOq9tJPi8 +cQGeBvV8Xx7BDlXKg5pZMK4ZyzBIle0iN430SppyZj6tlcDgFgDgEB8rMQ7XlFTT +QjOgNB0eRXbdT8oYN+yFFXoZCPzVx5zw8qkuEKmS5j1YPakWaDwvdSEYfyh3peFh +F7em6fgemdtzbvQKoiFs7tqqhZJmr/Z6a4LauiIINQ/PQvE1+mrufislzDoR5G2v +c7J2Ha3QsnhnGqQ5HFELZ1aD/ThdDc7d8Lsrlh/eezJS/R27tQahsiFepdaVaH/w +mZ7cRQg+59IJDTWU3YBOU5fXtQlEIGQWFwMCTFMNaN7VqnJNk22CDtucvc+081xd +VHppCZbW2xHBjXWotM85yM48vCR85mLK4b19p71XZQvk/iXttmkQ3CgaRr0BHdCX +teGYO8A3ZNY9lO4L4fUorgtWv3GLIylBjobFS1J72HGrH4oVpjuDWtdYAVHGTEHZ +f9hBZ3KiKN9gg6meyHv8U3NyWfWTehd2Ds735VzZC1U0oqpbtWpU5xPKV+yXbfRe +Bi9Fi1jUIxaS5BZuKGNZMN9QAZxjiRqf2xeUgnA3wySemkfWWspOqGmJch+RbNt+ +nhutxx9z3SxPGWX9f5NAEC7S8O08ni4oPmkmM8V7AgMBAAGjYzBhMA8GA1UdEwEB +/wQFMAMBAf8wHQYDVR0OBBYEFNq7LqqwDLiIJlF0XG0D08DYj3rWMB8GA1UdIwQY +MBaAFNq7LqqwDLiIJlF0XG0D08DYj3rWMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG +9w0BAQUFAAOCAgEAMXjmx7XfuJRAyXHEqDXsRh3ChfMoWIawC/yOsjmPRFWrZIRc +aanQmjg8+uUfNeVE44B5lGiku8SfPeE0zTBGi1QrlaXv9z+ZhP015s8xxtxqv6fX +IwjhmF7DWgh2qaavdy+3YL1ERmrvl/9zlcGO6JP7/TG37FcREUWbMPEaiDnBTzyn +ANXH/KttgCJwpQzgXQQpAvvLoJHRfNbDflDVnVi+QTjruXU8FdmbyUqDWcDaU/0z +uzYYm4UPFd3uLax2k7nZAY1IEKj79TiG8dsKxr2EoyNB3tZ3b4XUhRxQ4K5RirqN +Pnbiucon8l+f725ZDQbYKxek0nxru18UGkiPGkzns0ccjkxFKyDuSN/n3QmOGKja +QI2SJhFTYXNd673nxE0pN2HrrDktZy4W1vUAg4WhzH92xH3kt0tm7wNFYGm2DFKW +koRepqO1pD4r2czYG0eq8kTaT/kD6PAUyz/zg97QwVTjt+gKN02LIFkDMBmhLMi9 +ER/frslKxfMnZmaGrGiR/9nmUxwPi1xpZQomyB40w11Re9epnAahNt3ViZS82eQt +DF4JbAiXfKM9fJP/P6EUp8+1Xevb2xzEdt+Iub1FBZUbrvxGakyvSOPOrg/Sfuvm +bJxPgWp6ZKy7PtXny3YuxadIwVyQD8vIP/rmMuGNG2+k5o7Y+SlIis5z/iw= +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Universal CA 2 O=GeoTrust Inc. +# Subject: CN=GeoTrust Universal CA 2 O=GeoTrust Inc. +# Label: "GeoTrust Universal CA 2" +# Serial: 1 +# MD5 Fingerprint: 34:fc:b8:d0:36:db:9e:14:b3:c2:f2:db:8f:e4:94:c7 +# SHA1 Fingerprint: 37:9a:19:7b:41:85:45:35:0c:a6:03:69:f3:3c:2e:af:47:4f:20:79 +# SHA256 Fingerprint: a0:23:4f:3b:c8:52:7c:a5:62:8e:ec:81:ad:5d:69:89:5d:a5:68:0d:c9:1d:1c:b8:47:7f:33:f8:78:b9:5b:0b +-----BEGIN CERTIFICATE----- +MIIFbDCCA1SgAwIBAgIBATANBgkqhkiG9w0BAQUFADBHMQswCQYDVQQGEwJVUzEW +MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1c3QgVW5pdmVy +c2FsIENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMjkwMzA0MDUwMDAwWjBHMQswCQYD +VQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1 +c3QgVW5pdmVyc2FsIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQCzVFLByT7y2dyxUxpZKeexw0Uo5dfR7cXFS6GqdHtXr0om/Nj1XqduGdt0DE81 +WzILAePb63p3NeqqWuDW6KFXlPCQo3RWlEQwAx5cTiuFJnSCegx2oG9NzkEtoBUG +FF+3Qs17j1hhNNwqCPkuwwGmIkQcTAeC5lvO0Ep8BNMZcyfwqph/Lq9O64ceJHdq +XbboW0W63MOhBW9Wjo8QJqVJwy7XQYci4E+GymC16qFjwAGXEHm9ADwSbSsVsaxL +se4YuU6W3Nx2/zu+z18DwPw76L5GG//aQMJS9/7jOvdqdzXQ2o3rXhhqMcceujwb +KNZrVMaqW9eiLBsZzKIC9ptZvTdrhrVtgrrY6slWvKk2WP0+GfPtDCapkzj4T8Fd +IgbQl+rhrcZV4IErKIM6+vR7IVEAvlI4zs1meaj0gVbi0IMJR1FbUGrP20gaXT73 +y/Zl92zxlfgCOzJWgjl6W70viRu/obTo/3+NjN8D8WBOWBFM66M/ECuDmgFz2ZRt +hAAnZqzwcEAJQpKtT5MNYQlRJNiS1QuUYbKHsu3/mjX/hVTK7URDrBs8FmtISgoc +QIgfksILAAX/8sgCSqSqqcyZlpwvWOB94b67B9xfBHJcMTTD7F8t4D1kkCLm0ey4 +Lt1ZrtmhN79UNdxzMk+MBB4zsslG8dhcyFVQyWi9qLo2CQIDAQABo2MwYTAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAfBgNV +HSMEGDAWgBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAOBgNVHQ8BAf8EBAMCAYYwDQYJ +KoZIhvcNAQEFBQADggIBAGbBxiPz2eAubl/oz66wsCVNK/g7WJtAJDday6sWSf+z +dXkzoS9tcBc0kf5nfo/sm+VegqlVHy/c1FEHEv6sFj4sNcZj/NwQ6w2jqtB8zNHQ +L1EuxBRa3ugZ4T7GzKQp5y6EqgYweHZUcyiYWTjgAA1i00J9IZ+uPTqM1fp3DRgr +Fg5fNuH8KrUwJM/gYwx7WBr+mbpCErGR9Hxo4sjoryzqyX6uuyo9DRXcNJW2GHSo +ag/HtPQTxORb7QrSpJdMKu0vbBKJPfEncKpqA1Ihn0CoZ1Dy81of398j9tx4TuaY +T1U6U+Pv8vSfx3zYWK8pIpe44L2RLrB27FcRz+8pRPPphXpgY+RdM4kX2TGq2tbz +GDVyz4crL2MjhF2EjD9XoIj8mZEoJmmZ1I+XRL6O1UixpCgp8RW04eWe3fiPpm8m +1wk8OhwRDqZsN/etRIcsKMfYdIKz0G9KV7s1KSegi+ghp4dkNl3M2Basx7InQJJV +OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH +6aLcr34YEoP9VhdBLtUpgn2Z9DH2canPLAEnpQW5qrJITirvn5NSUZU8UnOOVkwX +QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS +-----END CERTIFICATE----- + +# Issuer: CN=AAA Certificate Services O=Comodo CA Limited +# Subject: CN=AAA Certificate Services O=Comodo CA Limited +# Label: "Comodo AAA Services root" +# Serial: 1 +# MD5 Fingerprint: 49:79:04:b0:eb:87:19:ac:47:b0:bc:11:51:9b:74:d0 +# SHA1 Fingerprint: d1:eb:23:a4:6d:17:d6:8f:d9:25:64:c2:f1:f1:60:17:64:d8:e3:49 +# SHA256 Fingerprint: d7:a7:a0:fb:5d:7e:27:31:d7:71:e9:48:4e:bc:de:f7:1d:5f:0c:3e:0a:29:48:78:2b:c8:3e:e0:ea:69:9e:f4 +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root Certification Authority O=QuoVadis Limited OU=Root Certification Authority +# Subject: CN=QuoVadis Root Certification Authority O=QuoVadis Limited OU=Root Certification Authority +# Label: "QuoVadis Root CA" +# Serial: 985026699 +# MD5 Fingerprint: 27:de:36:fe:72:b7:00:03:00:9d:f4:f0:1e:6c:04:24 +# SHA1 Fingerprint: de:3f:40:bd:50:93:d3:9b:6c:60:f6:da:bc:07:62:01:00:89:76:c9 +# SHA256 Fingerprint: a4:5e:de:3b:bb:f0:9c:8a:e1:5c:72:ef:c0:72:68:d6:93:a2:1c:99:6f:d5:1e:67:ca:07:94:60:fd:6d:88:73 +-----BEGIN CERTIFICATE----- +MIIF0DCCBLigAwIBAgIEOrZQizANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJC +TTEZMBcGA1UEChMQUXVvVmFkaXMgTGltaXRlZDElMCMGA1UECxMcUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTEuMCwGA1UEAxMlUXVvVmFkaXMgUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wMTAzMTkxODMzMzNaFw0yMTAzMTcxODMz +MzNaMH8xCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMSUw +IwYDVQQLExxSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MS4wLAYDVQQDEyVR +dW9WYWRpcyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv2G1lVO6V/z68mcLOhrfEYBklbTRvM16z/Yp +li4kVEAkOPcahdxYTMukJ0KX0J+DisPkBgNbAKVRHnAEdOLB1Dqr1607BxgFjv2D +rOpm2RgbaIr1VxqYuvXtdj182d6UajtLF8HVj71lODqV0D1VNk7feVcxKh7YWWVJ +WCCYfqtffp/p1k3sg3Spx2zY7ilKhSoGFPlU5tPaZQeLYzcS19Dsw3sgQUSj7cug +F+FxZc4dZjH3dgEZyH0DWLaVSR2mEiboxgx24ONmy+pdpibu5cxfvWenAScOospU +xbF6lR1xHkopigPcakXBpBlebzbNw6Kwt/5cOOJSvPhEQ+aQuwIDAQABo4ICUjCC +Ak4wPQYIKwYBBQUHAQEEMTAvMC0GCCsGAQUFBzABhiFodHRwczovL29jc3AucXVv +dmFkaXNvZmZzaG9yZS5jb20wDwYDVR0TAQH/BAUwAwEB/zCCARoGA1UdIASCAREw +ggENMIIBCQYJKwYBBAG+WAABMIH7MIHUBggrBgEFBQcCAjCBxxqBxFJlbGlhbmNl +IG9uIHRoZSBRdW9WYWRpcyBSb290IENlcnRpZmljYXRlIGJ5IGFueSBwYXJ0eSBh +c3N1bWVzIGFjY2VwdGFuY2Ugb2YgdGhlIHRoZW4gYXBwbGljYWJsZSBzdGFuZGFy +ZCB0ZXJtcyBhbmQgY29uZGl0aW9ucyBvZiB1c2UsIGNlcnRpZmljYXRpb24gcHJh +Y3RpY2VzLCBhbmQgdGhlIFF1b1ZhZGlzIENlcnRpZmljYXRlIFBvbGljeS4wIgYI +KwYBBQUHAgEWFmh0dHA6Ly93d3cucXVvdmFkaXMuYm0wHQYDVR0OBBYEFItLbe3T +KbkGGew5Oanwl4Rqy+/fMIGuBgNVHSMEgaYwgaOAFItLbe3TKbkGGew5Oanwl4Rq +y+/foYGEpIGBMH8xCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1p +dGVkMSUwIwYDVQQLExxSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MS4wLAYD +VQQDEyVRdW9WYWRpcyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggQ6tlCL +MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOCAQEAitQUtf70mpKnGdSk +fnIYj9lofFIk3WdvOXrEql494liwTXCYhGHoG+NpGA7O+0dQoE7/8CQfvbLO9Sf8 +7C9TqnN7Az10buYWnuulLsS/VidQK2K6vkscPFVcQR0kvoIgR13VRH56FmjffU1R +cHhXHTMe/QKZnAzNCgVPx7uOpHX6Sm2xgI4JVrmcGmD+XcHXetwReNDWXcG31a0y +mQM6isxUJTkxgXsTIlG6Rmyhu576BGxJJnSP0nPrzDCi5upZIof4l/UO/erMkqQW +xFIY6iHOsfHmhIHluqmGKPJDWl0Snawe2ajlCmqnf6CHKc/yiU3U7MXi5nrQNiOK +SnQ2+Q== +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 2 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 2 O=QuoVadis Limited +# Label: "QuoVadis Root CA 2" +# Serial: 1289 +# MD5 Fingerprint: 5e:39:7b:dd:f8:ba:ec:82:e9:ac:62:ba:0c:54:00:2b +# SHA1 Fingerprint: ca:3a:fb:cf:12:40:36:4b:44:b2:16:20:88:80:48:39:19:93:7c:f7 +# SHA256 Fingerprint: 85:a0:dd:7d:d7:20:ad:b7:ff:05:f8:3d:54:2b:20:9d:c7:ff:45:28:f7:d6:77:b1:83:89:fe:a5:e5:c4:9e:86 +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICBQkwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x +GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv +b3QgQ0EgMjAeFw0wNjExMjQxODI3MDBaFw0zMTExMjQxODIzMzNaMEUxCzAJBgNV +BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W +YWRpcyBSb290IENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCa +GMpLlA0ALa8DKYrwD4HIrkwZhR0In6spRIXzL4GtMh6QRr+jhiYaHv5+HBg6XJxg +Fyo6dIMzMH1hVBHL7avg5tKifvVrbxi3Cgst/ek+7wrGsxDp3MJGF/hd/aTa/55J +WpzmM+Yklvc/ulsrHHo1wtZn/qtmUIttKGAr79dgw8eTvI02kfN/+NsRE8Scd3bB +rrcCaoF6qUWD4gXmuVbBlDePSHFjIuwXZQeVikvfj8ZaCuWw419eaxGrDPmF60Tp ++ARz8un+XJiM9XOva7R+zdRcAitMOeGylZUtQofX1bOQQ7dsE/He3fbE+Ik/0XX1 +ksOR1YqI0JDs3G3eicJlcZaLDQP9nL9bFqyS2+r+eXyt66/3FsvbzSUr5R/7mp/i +Ucw6UwxI5g69ybR2BlLmEROFcmMDBOAENisgGQLodKcftslWZvB1JdxnwQ5hYIiz +PtGo/KPaHbDRsSNU30R2be1B2MGyIrZTHN81Hdyhdyox5C315eXbyOD/5YDXC2Og +/zOhD7osFRXql7PSorW+8oyWHhqPHWykYTe5hnMz15eWniN9gqRMgeKh0bpnX5UH +oycR7hYQe7xFSkyyBNKr79X9DFHOUGoIMfmR2gyPZFwDwzqLID9ujWc9Otb+fVuI +yV77zGHcizN300QyNQliBJIWENieJ0f7OyHj+OsdWwIDAQABo4GwMIGtMA8GA1Ud +EwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMB0GA1UdDgQWBBQahGK8SEwzJQTU7tD2 +A8QZRtGUazBuBgNVHSMEZzBlgBQahGK8SEwzJQTU7tD2A8QZRtGUa6FJpEcwRTEL +MAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMT +ElF1b1ZhZGlzIFJvb3QgQ0EgMoICBQkwDQYJKoZIhvcNAQEFBQADggIBAD4KFk2f +BluornFdLwUvZ+YTRYPENvbzwCYMDbVHZF34tHLJRqUDGCdViXh9duqWNIAXINzn +g/iN/Ae42l9NLmeyhP3ZRPx3UIHmfLTJDQtyU/h2BwdBR5YM++CCJpNVjP4iH2Bl +fF/nJrP3MpCYUNQ3cVX2kiF495V5+vgtJodmVjB3pjd4M1IQWK4/YY7yarHvGH5K +WWPKjaJW1acvvFYfzznB4vsKqBUsfU16Y8Zsl0Q80m/DShcK+JDSV6IZUaUtl0Ha +B0+pUNqQjZRG4T7wlP0QADj1O+hA4bRuVhogzG9Yje0uRY/W6ZM/57Es3zrWIozc +hLsib9D45MY56QSIPMO661V6bYCZJPVsAfv4l7CUW+v90m/xd2gNNWQjrLhVoQPR +TUIZ3Ph1WVaj+ahJefivDrkRoHy3au000LYmYjgahwz46P0u05B/B5EqHdZ+XIWD +mbA4CD/pXvk1B+TJYm5Xf6dQlfe6yJvmjqIBxdZmv3lh8zwc4bmCXF2gw+nYSL0Z +ohEUGW6yhhtoPkg3Goi3XZZenMfvJ2II4pEZXNLxId26F0KCl3GBUzGpn/Z9Yr9y +4aOTHcyKJloJONDO1w2AFrR4pTqHTI2KpdVGl/IsELm8VCLAAVBpQ570su9t+Oza +8eOx79+Rj1QqCyXBJhnEUhAFZdWCEOrCMc0u +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 3" +# Serial: 1478 +# MD5 Fingerprint: 31:85:3c:62:94:97:63:b9:aa:fd:89:4e:af:6f:e0:cf +# SHA1 Fingerprint: 1f:49:14:f7:d8:74:95:1d:dd:ae:02:c0:be:fd:3a:2d:82:75:51:85 +# SHA256 Fingerprint: 18:f1:fc:7f:20:5d:f8:ad:dd:eb:7f:e0:07:dd:57:e3:af:37:5a:9c:4d:8d:73:54:6b:f4:f1:fe:d1:e1:8d:35 +-----BEGIN CERTIFICATE----- +MIIGnTCCBIWgAwIBAgICBcYwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x +GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv +b3QgQ0EgMzAeFw0wNjExMjQxOTExMjNaFw0zMTExMjQxOTA2NDRaMEUxCzAJBgNV +BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W +YWRpcyBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDM +V0IWVJzmmNPTTe7+7cefQzlKZbPoFog02w1ZkXTPkrgEQK0CSzGrvI2RaNggDhoB +4hp7Thdd4oq3P5kazethq8Jlph+3t723j/z9cI8LoGe+AaJZz3HmDyl2/7FWeUUr +H556VOijKTVopAFPD6QuN+8bv+OPEKhyq1hX51SGyMnzW9os2l2ObjyjPtr7guXd +8lyyBTNvijbO0BNO/79KDDRMpsMhvVAEVeuxu537RR5kFd5VAYwCdrXLoT9Cabwv +vWhDFlaJKjdhkf2mrk7AyxRllDdLkgbvBNDInIjbC3uBr7E9KsRlOni27tyAsdLT +mZw67mtaa7ONt9XOnMK+pUsvFrGeaDsGb659n/je7Mwpp5ijJUMv7/FfJuGITfhe +btfZFG4ZM2mnO4SJk8RTVROhUXhA+LjJou57ulJCg54U7QVSWllWp5f8nT8KKdjc +T5EOE7zelaTfi5m+rJsziO+1ga8bxiJTyPbH7pcUsMV8eFLI8M5ud2CEpukqdiDt +WAEXMJPpGovgc2PZapKUSU60rUqFxKMiMPwJ7Wgic6aIDFUhWMXhOp8q3crhkODZ +c6tsgLjoC2SToJyMGf+z0gzskSaHirOi4XCPLArlzW1oUevaPwV/izLmE1xr/l9A +4iLItLRkT9a6fUg+qGkM17uGcclzuD87nSVL2v9A6wIDAQABo4IBlTCCAZEwDwYD +VR0TAQH/BAUwAwEB/zCB4QYDVR0gBIHZMIHWMIHTBgkrBgEEAb5YAAMwgcUwgZMG +CCsGAQUFBwICMIGGGoGDQW55IHVzZSBvZiB0aGlzIENlcnRpZmljYXRlIGNvbnN0 +aXR1dGVzIGFjY2VwdGFuY2Ugb2YgdGhlIFF1b1ZhZGlzIFJvb3QgQ0EgMyBDZXJ0 +aWZpY2F0ZSBQb2xpY3kgLyBDZXJ0aWZpY2F0aW9uIFByYWN0aWNlIFN0YXRlbWVu +dC4wLQYIKwYBBQUHAgEWIWh0dHA6Ly93d3cucXVvdmFkaXNnbG9iYWwuY29tL2Nw +czALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFPLAE+CCQz777i9nMpY1XNu4ywLQMG4G +A1UdIwRnMGWAFPLAE+CCQz777i9nMpY1XNu4ywLQoUmkRzBFMQswCQYDVQQGEwJC +TTEZMBcGA1UEChMQUXVvVmFkaXMgTGltaXRlZDEbMBkGA1UEAxMSUXVvVmFkaXMg +Um9vdCBDQSAzggIFxjANBgkqhkiG9w0BAQUFAAOCAgEAT62gLEz6wPJv92ZVqyM0 +7ucp2sNbtrCD2dDQ4iH782CnO11gUyeim/YIIirnv6By5ZwkajGxkHon24QRiSem +d1o417+shvzuXYO8BsbRd2sPbSQvS3pspweWyuOEn62Iix2rFo1bZhfZFvSLgNLd ++LJ2w/w4E6oM3kJpK27zPOuAJ9v1pkQNn1pVWQvVDVJIxa6f8i+AxeoyUDUSly7B +4f/xI4hROJ/yZlZ25w9Rl6VSDE1JUZU2Pb+iSwwQHYaZTKrzchGT5Or2m9qoXadN +t54CrnMAyNojA+j56hl0YgCUyyIgvpSnWbWCar6ZeXqp8kokUvd0/bpO5qgdAm6x +DYBEwa7TIzdfu4V8K5Iu6H6li92Z4b8nby1dqnuH/grdS/yO9SbkbnBCbjPsMZ57 +k8HkyWkaPcBrTiJt7qtYTcbQQcEr6k8Sh17rRdhs9ZgC06DYVYoGmRmioHfRMJ6s +zHXug/WwYjnPbFfiTNKRCw51KBuav/0aQ/HKd/s7j2G4aSgWQgRecCocIdiP4b0j +Wy10QJLZYxkNc91pvGJHvOB0K7Lrfb5BG7XARsWhIstfTsEokt4YutUqKLsRixeT +mJlglFwjz1onl14LBQaTNx47aTbrqZ5hHY8y2o4M1nQ+ewkk2gF3R8Q7zTSMmfXK +4SVhM7JZG+Ju1zdXtg2pEto= +-----END CERTIFICATE----- + +# Issuer: O=SECOM Trust.net OU=Security Communication RootCA1 +# Subject: O=SECOM Trust.net OU=Security Communication RootCA1 +# Label: "Security Communication Root CA" +# Serial: 0 +# MD5 Fingerprint: f1:bc:63:6a:54:e0:b5:27:f5:cd:e7:1a:e3:4d:6e:4a +# SHA1 Fingerprint: 36:b1:2b:49:f9:81:9e:d7:4c:9e:bc:38:0f:c6:56:8f:5d:ac:b2:f7 +# SHA256 Fingerprint: e7:5e:72:ed:9f:56:0e:ec:6e:b4:80:00:73:a4:3f:c3:ad:19:19:5a:39:22:82:01:78:95:97:4a:99:02:6b:6c +-----BEGIN CERTIFICATE----- +MIIDWjCCAkKgAwIBAgIBADANBgkqhkiG9w0BAQUFADBQMQswCQYDVQQGEwJKUDEY +MBYGA1UEChMPU0VDT00gVHJ1c3QubmV0MScwJQYDVQQLEx5TZWN1cml0eSBDb21t +dW5pY2F0aW9uIFJvb3RDQTEwHhcNMDMwOTMwMDQyMDQ5WhcNMjMwOTMwMDQyMDQ5 +WjBQMQswCQYDVQQGEwJKUDEYMBYGA1UEChMPU0VDT00gVHJ1c3QubmV0MScwJQYD +VQQLEx5TZWN1cml0eSBDb21tdW5pY2F0aW9uIFJvb3RDQTEwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCzs/5/022x7xZ8V6UMbXaKL0u/ZPtM7orw8yl8 +9f/uKuDp6bpbZCKamm8sOiZpUQWZJtzVHGpxxpp9Hp3dfGzGjGdnSj74cbAZJ6kJ +DKaVv0uMDPpVmDvY6CKhS3E4eayXkmmziX7qIWgGmBSWh9JhNrxtJ1aeV+7AwFb9 +Ms+k2Y7CI9eNqPPYJayX5HA49LY6tJ07lyZDo6G8SVlyTCMwhwFY9k6+HGhWZq/N +QV3Is00qVUarH9oe4kA92819uZKAnDfdDJZkndwi92SL32HeFZRSFaB9UslLqCHJ +xrHty8OVYNEP8Ktw+N/LTX7s1vqr2b1/VPKl6Xn62dZ2JChzAgMBAAGjPzA9MB0G +A1UdDgQWBBSgc0mZaNyFW2XjmygvV5+9M7wHSDALBgNVHQ8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAaECpqLvkT115swW1F7NgE+vG +kl3g0dNq/vu+m22/xwVtWSDEHPC32oRYAmP6SBbvT6UL90qY8j+eG61Ha2POCEfr +Uj94nK9NrvjVT8+amCoQQTlSxN3Zmw7vkwGusi7KaEIkQmywszo+zenaSMQVy+n5 +Bw+SUEmK3TGXX8npN6o7WWWXlDLJs58+OmJYxUmtYg5xpTKqL8aJdkNAExNnPaJU +JRDL8Try2frbSVa7pv6nQTXD4IhhyYjH3zYQIphZ6rBK+1YWc26sTfcioU+tHXot +RSflMMFe8toTyyVCUZVHA4xsIcx0Qu1T/zOLjw9XARYvz6buyXAiFL39vmwLAw== +-----END CERTIFICATE----- + +# Issuer: CN=Sonera Class2 CA O=Sonera +# Subject: CN=Sonera Class2 CA O=Sonera +# Label: "Sonera Class 2 Root CA" +# Serial: 29 +# MD5 Fingerprint: a3:ec:75:0f:2e:88:df:fa:48:01:4e:0b:5c:48:6f:fb +# SHA1 Fingerprint: 37:f7:6d:e6:07:7c:90:c5:b1:3e:93:1a:b7:41:10:b4:f2:e4:9a:27 +# SHA256 Fingerprint: 79:08:b4:03:14:c1:38:10:0b:51:8d:07:35:80:7f:fb:fc:f8:51:8a:00:95:33:71:05:ba:38:6b:15:3d:d9:27 +-----BEGIN CERTIFICATE----- +MIIDIDCCAgigAwIBAgIBHTANBgkqhkiG9w0BAQUFADA5MQswCQYDVQQGEwJGSTEP +MA0GA1UEChMGU29uZXJhMRkwFwYDVQQDExBTb25lcmEgQ2xhc3MyIENBMB4XDTAx +MDQwNjA3Mjk0MFoXDTIxMDQwNjA3Mjk0MFowOTELMAkGA1UEBhMCRkkxDzANBgNV +BAoTBlNvbmVyYTEZMBcGA1UEAxMQU29uZXJhIENsYXNzMiBDQTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAJAXSjWdyvANlsdE+hY3/Ei9vX+ALTU74W+o +Z6m/AxxNjG8yR9VBaKQTBME1DJqEQ/xcHf+Js+gXGM2RX/uJ4+q/Tl18GybTdXnt +5oTjV+WtKcT0OijnpXuENmmz/V52vaMtmdOQTiMofRhj8VQ7Jp12W5dCsv+u8E7s +3TmVToMGf+dJQMjFAbJUWmYdPfz56TwKnoG4cPABi+QjVHzIrviQHgCWctRUz2Ej +vOr7nQKV0ba5cTppCD8PtOFCx4j1P5iop7oc4HFx71hXgVB6XGt0Rg6DA5jDjqhu +8nYybieDwnPz3BjotJPqdURrBGAgcVeHnfO+oJAjPYok4doh28MCAwEAAaMzMDEw +DwYDVR0TAQH/BAUwAwEB/zARBgNVHQ4ECgQISqCqWITTXjwwCwYDVR0PBAQDAgEG +MA0GCSqGSIb3DQEBBQUAA4IBAQBazof5FnIVV0sd2ZvnoiYw7JNn39Yt0jSv9zil +zqsWuasvfDXLrNAPtEwr/IDva4yRXzZ299uzGxnq9LIR/WFxRL8oszodv7ND6J+/ +3DEIcbCdjdY0RzKQxmUk96BKfARzjzlvF4xytb1LyHr4e4PDKE6cCepnP7JnBBvD +FNr450kkkdAdavphOe9r5yF1BgfYErQhIHBCcYHaPJo2vqZbDWpsmh+Re/n570K6 +Tk6ezAyNlNzZRZxe7EJQY670XcSxEtzKO6gunRRaBXW37Ndj4ro1tgQIkejanZz2 +ZrUYrAqmVCY0M9IbwdR/GjqOC6oybtv8TyWf2TLHllpwrN9M +-----END CERTIFICATE----- + +# Issuer: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com +# Subject: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com +# Label: "XRamp Global CA Root" +# Serial: 107108908803651509692980124233745014957 +# MD5 Fingerprint: a1:0b:44:b3:ca:10:d8:00:6e:9d:0f:d8:0f:92:0a:d1 +# SHA1 Fingerprint: b8:01:86:d1:eb:9c:86:a5:41:04:cf:30:54:f3:4c:52:b7:e5:58:c6 +# SHA256 Fingerprint: ce:cd:dc:90:50:99:d8:da:df:c5:b1:d2:09:b7:37:cb:e2:c1:8c:fb:2c:10:c0:ff:0b:cf:0d:32:86:fc:1a:a2 +-----BEGIN CERTIFICATE----- +MIIEMDCCAxigAwIBAgIQUJRs7Bjq1ZxN1ZfvdY+grTANBgkqhkiG9w0BAQUFADCB +gjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3dy54cmFtcHNlY3VyaXR5LmNvbTEk +MCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2VydmljZXMgSW5jMS0wKwYDVQQDEyRY +UmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQxMTAxMTcx +NDA0WhcNMzUwMTAxMDUzNzE5WjCBgjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3 +dy54cmFtcHNlY3VyaXR5LmNvbTEkMCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2Vy +dmljZXMgSW5jMS0wKwYDVQQDEyRYUmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBB +dXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYJB69FbS6 +38eMpSe2OAtp87ZOqCwuIR1cRN8hXX4jdP5efrRKt6atH67gBhbim1vZZ3RrXYCP +KZ2GG9mcDZhtdhAoWORlsH9KmHmf4MMxfoArtYzAQDsRhtDLooY2YKTVMIJt2W7Q +DxIEM5dfT2Fa8OT5kavnHTu86M/0ay00fOJIYRyO82FEzG+gSqmUsE3a56k0enI4 +qEHMPJQRfevIpoy3hsvKMzvZPTeL+3o+hiznc9cKV6xkmxnr9A8ECIqsAxcZZPRa +JSKNNCyy9mgdEm3Tih4U2sSPpuIjhdV6Db1q4Ons7Be7QhtnqiXtRYMh/MHJfNVi +PvryxS3T/dRlAgMBAAGjgZ8wgZwwEwYJKwYBBAGCNxQCBAYeBABDAEEwCwYDVR0P +BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMZPoj0GY4QJnM5i5ASs +jVy16bYbMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwueHJhbXBzZWN1cml0 +eS5jb20vWEdDQS5jcmwwEAYJKwYBBAGCNxUBBAMCAQEwDQYJKoZIhvcNAQEFBQAD +ggEBAJEVOQMBG2f7Shz5CmBbodpNl2L5JFMn14JkTpAuw0kbK5rc/Kh4ZzXxHfAR +vbdI4xD2Dd8/0sm2qlWkSLoC295ZLhVbO50WfUfXN+pfTXYSNrsf16GBBEYgoyxt +qZ4Bfj8pzgCT3/3JknOJiWSe5yvkHJEs0rnOfc5vMZnT5r7SHpDwCRR5XCOrTdLa +IR9NmXmd4c8nnxCbHIgNsIpkQTG4DmyQJKSbXHGPurt+HBvbaoAPIbzp26a3QPSy +i6mx5O+aGtA9aZnuqCij4Tyz8LIRnM98QObd50N9otg6tamN8jSZxNQQ4Qb9CYQQ +O+7ETPTsJ3xCwnR8gooJybQDJbw= +-----END CERTIFICATE----- + +# Issuer: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Subject: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Label: "Go Daddy Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 91:de:06:25:ab:da:fd:32:17:0c:bb:25:17:2a:84:67 +# SHA1 Fingerprint: 27:96:ba:e6:3f:18:01:e2:77:26:1b:a0:d7:77:70:02:8f:20:ee:e4 +# SHA256 Fingerprint: c3:84:6b:f2:4b:9e:93:ca:64:27:4c:0e:c6:7c:1e:cc:5e:02:4f:fc:ac:d2:d7:40:19:35:0e:81:fe:54:6a:e4 +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= +-----END CERTIFICATE----- + +# Issuer: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Subject: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Label: "Starfield Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 32:4a:4b:bb:c8:63:69:9b:be:74:9a:c6:dd:1d:46:24 +# SHA1 Fingerprint: ad:7e:1c:28:b0:64:ef:8f:60:03:40:20:14:c3:d0:e3:37:0e:b5:8a +# SHA256 Fingerprint: 14:65:fa:20:53:97:b8:76:fa:a6:f0:a9:95:8e:55:90:e4:0f:cc:7f:aa:4f:b7:c2:c8:67:75:21:fb:5f:b6:58 +-----BEGIN CERTIFICATE----- +MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl +MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp +U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw +NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp +ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 +DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf +8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN ++lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 +X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa +K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA +1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G +A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR +zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 +YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD +bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 +L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D +eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl +xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp +VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY +WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= +-----END CERTIFICATE----- + +# Issuer: O=Government Root Certification Authority +# Subject: O=Government Root Certification Authority +# Label: "Taiwan GRCA" +# Serial: 42023070807708724159991140556527066870 +# MD5 Fingerprint: 37:85:44:53:32:45:1f:20:f0:f3:95:e1:25:c4:43:4e +# SHA1 Fingerprint: f4:8b:11:bf:de:ab:be:94:54:20:71:e6:41:de:6b:be:88:2b:40:b9 +# SHA256 Fingerprint: 76:00:29:5e:ef:e8:5b:9e:1f:d6:24:db:76:06:2a:aa:ae:59:81:8a:54:d2:77:4c:d4:c0:b2:c0:11:31:e1:b3 +-----BEGIN CERTIFICATE----- +MIIFcjCCA1qgAwIBAgIQH51ZWtcvwgZEpYAIaeNe9jANBgkqhkiG9w0BAQUFADA/ +MQswCQYDVQQGEwJUVzEwMC4GA1UECgwnR292ZXJubWVudCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5MB4XDTAyMTIwNTEzMjMzM1oXDTMyMTIwNTEzMjMzM1ow +PzELMAkGA1UEBhMCVFcxMDAuBgNVBAoMJ0dvdmVybm1lbnQgUm9vdCBDZXJ0aWZp +Y2F0aW9uIEF1dGhvcml0eTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +AJoluOzMonWoe/fOW1mKydGGEghU7Jzy50b2iPN86aXfTEc2pBsBHH8eV4qNw8XR +IePaJD9IK/ufLqGU5ywck9G/GwGHU5nOp/UKIXZ3/6m3xnOUT0b3EEk3+qhZSV1q +gQdW8or5BtD3cCJNtLdBuTK4sfCxw5w/cP1T3YGq2GN49thTbqGsaoQkclSGxtKy +yhwOeYHWtXBiCAEuTk8O1RGvqa/lmr/czIdtJuTJV6L7lvnM4T9TjGxMfptTCAts +F/tnyMKtsc2AtJfcdgEWFelq16TheEfOhtX7MfP6Mb40qij7cEwdScevLJ1tZqa2 +jWR+tSBqnTuBto9AAGdLiYa4zGX+FVPpBMHWXx1E1wovJ5pGfaENda1UhhXcSTvx +ls4Pm6Dso3pdvtUqdULle96ltqqvKKyskKw4t9VoNSZ63Pc78/1Fm9G7Q3hub/FC +VGqY8A2tl+lSXunVanLeavcbYBT0peS2cWeqH+riTcFCQP5nRhc4L0c/cZyu5SHK +YS1tB6iEfC3uUSXxY5Ce/eFXiGvviiNtsea9P63RPZYLhY3Naye7twWb7LuRqQoH +EgKXTiCQ8P8NHuJBO9NAOueNXdpm5AKwB1KYXA6OM5zCppX7VRluTI6uSw+9wThN +Xo+EHWbNxWCWtFJaBYmOlXqYwZE8lSOyDvR5tMl8wUohAgMBAAGjajBoMB0GA1Ud +DgQWBBTMzO/MKWCkO7GStjz6MmKPrCUVOzAMBgNVHRMEBTADAQH/MDkGBGcqBwAE +MTAvMC0CAQAwCQYFKw4DAhoFADAHBgVnKgMAAAQUA5vwIhP/lSg209yewDL7MTqK +UWUwDQYJKoZIhvcNAQEFBQADggIBAECASvomyc5eMN1PhnR2WPWus4MzeKR6dBcZ +TulStbngCnRiqmjKeKBMmo4sIy7VahIkv9Ro04rQ2JyftB8M3jh+Vzj8jeJPXgyf +qzvS/3WXy6TjZwj/5cAWtUgBfen5Cv8b5Wppv3ghqMKnI6mGq3ZW6A4M9hPdKmaK +ZEk9GhiHkASfQlK3T8v+R0F2Ne//AHY2RTKbxkaFXeIksB7jSJaYV0eUVXoPQbFE +JPPB/hprv4j9wabak2BegUqZIJxIZhm1AHlUD7gsL0u8qV1bYH+Mh6XgUmMqvtg7 +hUAV/h62ZT/FS9p+tXo1KaMuephgIqP0fSdOLeq0dDzpD6QzDxARvBMB1uUO07+1 +EqLhRSPAzAhuYbeJq4PjJB7mXQfnHyA+z2fI56wwbSdLaG5LKlwCCDTb+HbkZ6Mm +nD+iMsJKxYEYMRBWqoTvLQr/uB930r+lWKBi5NdLkXWNiYCYfm3LU05er/ayl4WX +udpVBrkk7tfGOB5jGxI7leFYrPLfhNVfmS8NVVvmONsuP3LpSIXLuykTjx44Vbnz +ssQwmSNOXfJIoRIM3BKQCZBUkQM8R+XVyWXgt0t97EfTsws+rZ7QdAAO671RrcDe +LMDDav7v3Aun+kbfYNucpllQdSNpc5Oy+fwC00fmcc4QAu4njIT/rEUNE1yDMuAl +pYYsfPQS +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root CA" +# Serial: 17154717934120587862167794914071425081 +# MD5 Fingerprint: 87:ce:0b:7b:2a:0e:49:00:e1:58:71:9b:37:a8:93:72 +# SHA1 Fingerprint: 05:63:b8:63:0d:62:d7:5a:bb:c8:ab:1e:4b:df:b5:a8:99:b2:4d:43 +# SHA256 Fingerprint: 3e:90:99:b5:01:5e:8f:48:6c:00:bc:ea:9d:11:1e:e7:21:fa:ba:35:5a:89:bc:f1:df:69:56:1e:3d:c6:32:5c +-----BEGIN CERTIFICATE----- +MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c +JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP +mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ +wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 +VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ +AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB +AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun +pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC +dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf +fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm +NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx +H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe ++o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root CA" +# Serial: 10944719598952040374951832963794454346 +# MD5 Fingerprint: 79:e4:a9:84:0d:7d:3a:96:d7:c0:4f:e2:43:4c:89:2e +# SHA1 Fingerprint: a8:98:5d:3a:65:e5:e5:c4:b2:d7:d6:6d:40:c6:dd:2f:b1:9c:54:36 +# SHA256 Fingerprint: 43:48:a0:e9:44:4c:78:cb:26:5e:05:8d:5e:89:44:b4:d8:4f:96:62:bd:26:db:25:7f:89:34:a4:43:c7:01:61 +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD +QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB +CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97 +nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt +43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P +T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4 +gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR +TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw +DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr +hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg +06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF +PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls +YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk +CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert High Assurance EV Root CA" +# Serial: 3553400076410547919724730734378100087 +# MD5 Fingerprint: d4:74:de:57:5c:39:b2:d3:9c:85:83:c5:c0:65:49:8a +# SHA1 Fingerprint: 5f:b7:ee:06:33:e2:59:db:ad:0c:4c:9a:e6:d3:8f:1a:61:c7:dc:25 +# SHA256 Fingerprint: 74:31:e5:f4:c3:c1:ce:46:90:77:4f:0b:61:e0:54:40:88:3b:a9:a0:1e:d0:0b:a6:ab:d7:80:6e:d3:b1:18:cf +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm ++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW +PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM +xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB +Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 +hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg +EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA +FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec +nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z +eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF +hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 +Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep ++OkuE6N36B9K +-----END CERTIFICATE----- + +# Issuer: CN=Class 2 Primary CA O=Certplus +# Subject: CN=Class 2 Primary CA O=Certplus +# Label: "Certplus Class 2 Primary CA" +# Serial: 177770208045934040241468760488327595043 +# MD5 Fingerprint: 88:2c:8c:52:b8:a2:3c:f3:f7:bb:03:ea:ae:ac:42:0b +# SHA1 Fingerprint: 74:20:74:41:72:9c:dd:92:ec:79:31:d8:23:10:8d:c2:81:92:e2:bb +# SHA256 Fingerprint: 0f:99:3c:8a:ef:97:ba:af:56:87:14:0e:d5:9a:d1:82:1b:b4:af:ac:f0:aa:9a:58:b5:d5:7a:33:8a:3a:fb:cb +-----BEGIN CERTIFICATE----- +MIIDkjCCAnqgAwIBAgIRAIW9S/PY2uNp9pTXX8OlRCMwDQYJKoZIhvcNAQEFBQAw +PTELMAkGA1UEBhMCRlIxETAPBgNVBAoTCENlcnRwbHVzMRswGQYDVQQDExJDbGFz +cyAyIFByaW1hcnkgQ0EwHhcNOTkwNzA3MTcwNTAwWhcNMTkwNzA2MjM1OTU5WjA9 +MQswCQYDVQQGEwJGUjERMA8GA1UEChMIQ2VydHBsdXMxGzAZBgNVBAMTEkNsYXNz +IDIgUHJpbWFyeSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANxQ +ltAS+DXSCHh6tlJw/W/uz7kRy1134ezpfgSN1sxvc0NXYKwzCkTsA18cgCSR5aiR +VhKC9+Ar9NuuYS6JEI1rbLqzAr3VNsVINyPi8Fo3UjMXEuLRYE2+L0ER4/YXJQyL +kcAbmXuZVg2v7tK8R1fjeUl7NIknJITesezpWE7+Tt9avkGtrAjFGA7v0lPubNCd +EgETjdyAYveVqUSISnFOYFWe2yMZeVYHDD9jC1yw4r5+FfyUM1hBOHTE4Y+L3yas +H7WLO7dDWWuwJKZtkIvEcupdM5i3y95ee++U8Rs+yskhwcWYAqqi9lt3m/V+llU0 +HGdpwPFC40es/CgcZlUCAwEAAaOBjDCBiTAPBgNVHRMECDAGAQH/AgEKMAsGA1Ud +DwQEAwIBBjAdBgNVHQ4EFgQU43Mt38sOKAze3bOkynm4jrvoMIkwEQYJYIZIAYb4 +QgEBBAQDAgEGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHA6Ly93d3cuY2VydHBsdXMu +Y29tL0NSTC9jbGFzczIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCnVM+IRBnL39R/ +AN9WM2K191EBkOvDP9GIROkkXe/nFL0gt5o8AP5tn9uQ3Nf0YtaLcF3n5QRIqWh8 +yfFC82x/xXp8HVGIutIKPidd3i1RTtMTZGnkLuPT55sJmabglZvOGtd/vjzOUrMR +FcEPF80Du5wlFbqidon8BvEY0JNLDnyCt6X09l/+7UCmnYR0ObncHoUW2ikbhiMA +ybuJfm6AiB4vFLQDJKgybwOaRywwvlbGp0ICcBvqQNi6BQNwB6SW//1IMwrh3KWB +kJtN3X3n57LNXMhqlfil9o3EXXgIvnsG1knPGTZQIy4I5p4FTUcY1Rbpsda2ENW7 +l7+ijrRU +-----END CERTIFICATE----- + +# Issuer: CN=DST Root CA X3 O=Digital Signature Trust Co. +# Subject: CN=DST Root CA X3 O=Digital Signature Trust Co. +# Label: "DST Root CA X3" +# Serial: 91299735575339953335919266965803778155 +# MD5 Fingerprint: 41:03:52:dc:0f:f7:50:1b:16:f0:02:8e:ba:6f:45:c5 +# SHA1 Fingerprint: da:c9:02:4f:54:d8:f6:df:94:93:5f:b1:73:26:38:ca:6a:d7:7c:13 +# SHA256 Fingerprint: 06:87:26:03:31:a7:24:03:d9:09:f1:05:e6:9b:cf:0d:32:e1:bd:24:93:ff:c6:d9:20:6d:11:bc:d6:77:07:39 +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIQRK+wgNajJ7qJMDmGLvhAazANBgkqhkiG9w0BAQUFADA/ +MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT +DkRTVCBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDkzMDE0MDExNVow +PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD +Ew5EU1QgUm9vdCBDQSBYMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AN+v6ZdQCINXtMxiZfaQguzH0yxrMMpb7NnDfcdAwRgUi+DoM3ZJKuM/IUmTrE4O +rz5Iy2Xu/NMhD2XSKtkyj4zl93ewEnu1lcCJo6m67XMuegwGMoOifooUMM0RoOEq +OLl5CjH9UL2AZd+3UWODyOKIYepLYYHsUmu5ouJLGiifSKOeDNoJjj4XLh7dIN9b +xiqKqy69cK3FCxolkHRyxXtqqzTWMIn/5WgTe1QLyNau7Fqckh49ZLOMxt+/yUFw +7BZy1SbsOFU5Q9D8/RhcQPGX69Wam40dutolucbY38EVAjqr2m7xPi71XAicPNaD +aeQQmxkqtilX4+U9m5/wAl0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFMSnsaR7LHH62+FLkHX/xBVghYkQMA0GCSqG +SIb3DQEBBQUAA4IBAQCjGiybFwBcqR7uKGY3Or+Dxz9LwwmglSBd49lZRNI+DT69 +ikugdB/OEIKcdBodfpga3csTS7MgROSR6cz8faXbauX+5v3gTt23ADq1cEmv8uXr +AvHRAosZy5Q6XkjEGB5YGV8eAlrwDPGxrancWYaLbumR9YbK+rlmM6pZW87ipxZz +R8srzJmwN0jP41ZL9c8PDHIyh8bwRLtTcm1D9SZImlJnt1ir/md2cXjbDaJWFBM5 +JDGFoqgCWjBH4d1QB7wCCZAA62RjYJsWvIjJEubSfZGL+T0yjWW06XyxV3bqxbYo +Ob8VZRzI9neWagqNdwvYkQsEjgfbKbYK7p2CNTUQ +-----END CERTIFICATE----- + +# Issuer: CN=SwissSign Gold CA - G2 O=SwissSign AG +# Subject: CN=SwissSign Gold CA - G2 O=SwissSign AG +# Label: "SwissSign Gold CA - G2" +# Serial: 13492815561806991280 +# MD5 Fingerprint: 24:77:d9:a8:91:d1:3b:fa:88:2d:c2:ff:f8:cd:33:93 +# SHA1 Fingerprint: d8:c5:38:8a:b7:30:1b:1b:6e:d4:7a:e6:45:25:3a:6f:9f:1a:27:61 +# SHA256 Fingerprint: 62:dd:0b:e9:b9:f5:0a:16:3e:a0:f8:e7:5c:05:3b:1e:ca:57:ea:55:c8:68:8f:64:7c:68:81:f2:c8:35:7b:95 +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJALtAHEP1Xk+wMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkNIMRUwEwYDVQQKEwxTd2lzc1NpZ24gQUcxHzAdBgNVBAMTFlN3aXNzU2ln +biBHb2xkIENBIC0gRzIwHhcNMDYxMDI1MDgzMDM1WhcNMzYxMDI1MDgzMDM1WjBF +MQswCQYDVQQGEwJDSDEVMBMGA1UEChMMU3dpc3NTaWduIEFHMR8wHQYDVQQDExZT +d2lzc1NpZ24gR29sZCBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAr+TufoskDhJuqVAtFkQ7kpJcyrhdhJJCEyq8ZVeCQD5XJM1QiyUqt2/8 +76LQwB8CJEoTlo8jE+YoWACjR8cGp4QjK7u9lit/VcyLwVcfDmJlD909Vopz2q5+ +bbqBHH5CjCA12UNNhPqE21Is8w4ndwtrvxEvcnifLtg+5hg3Wipy+dpikJKVyh+c +6bM8K8vzARO/Ws/BtQpgvd21mWRTuKCWs2/iJneRjOBiEAKfNA+k1ZIzUd6+jbqE +emA8atufK+ze3gE/bk3lUIbLtK/tREDFylqM2tIrfKjuvqblCqoOpd8FUrdVxyJd +MmqXl2MT28nbeTZ7hTpKxVKJ+STnnXepgv9VHKVxaSvRAiTysybUa9oEVeXBCsdt +MDeQKuSeFDNeFhdVxVu1yzSJkvGdJo+hB9TGsnhQ2wwMC3wLjEHXuendjIj3o02y +MszYF9rNt85mndT9Xv+9lz4pded+p2JYryU0pUHHPbwNUMoDAw8IWh+Vc3hiv69y +FGkOpeUDDniOJihC8AcLYiAQZzlG+qkDzAQ4embvIIO1jEpWjpEA/I5cgt6IoMPi +aG59je883WX0XaxR7ySArqpWl2/5rX3aYT+YdzylkbYcjCbaZaIJbcHiVOO5ykxM +gI93e2CaHt+28kgeDrpOVG2Y4OGiGqJ3UM/EY5LsRxmd6+ZrzsECAwEAAaOBrDCB +qTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWyV7 +lqRlUX64OfPAeGZe6Drn8O4wHwYDVR0jBBgwFoAUWyV7lqRlUX64OfPAeGZe6Drn +8O4wRgYDVR0gBD8wPTA7BglghXQBWQECAQEwLjAsBggrBgEFBQcCARYgaHR0cDov +L3JlcG9zaXRvcnkuc3dpc3NzaWduLmNvbS8wDQYJKoZIhvcNAQEFBQADggIBACe6 +45R88a7A3hfm5djV9VSwg/S7zV4Fe0+fdWavPOhWfvxyeDgD2StiGwC5+OlgzczO +UYrHUDFu4Up+GC9pWbY9ZIEr44OE5iKHjn3g7gKZYbge9LgriBIWhMIxkziWMaa5 +O1M/wySTVltpkuzFwbs4AOPsF6m43Md8AYOfMke6UiI0HTJ6CVanfCU2qT1L2sCC +bwq7EsiHSycR+R4tx5M/nttfJmtS2S6K8RTGRI0Vqbe/vd6mGu6uLftIdxf+u+yv +GPUqUfA5hJeVbG4bwyvEdGB5JbAKJ9/fXtI5z0V9QkvfsywexcZdylU6oJxpmo/a +77KwPJ+HbBIrZXAVUjEaJM9vMSNQH4xPjyPDdEFjHFWoFN0+4FFQz/EbMFYOkrCC +hdiDyyJkvC24JdVUorgG6q2SpCSgwYa1ShNqR88uC1aVVMvOmttqtKay20EIhid3 +92qgQmwLOM7XdVAyksLfKzAiSNDVQTglXaTpXZ/GlHXQRf0wl0OPkKsKx4ZzYEpp +Ld6leNcG2mqeSz53OiATIgHQv2ieY2BrNU0LbbqhPcCT4H8js1WtciVORvnSFu+w +ZMEBnunKoGqYDs/YYPIvSbjkQuE4NRb0yG5P94FW6LqjviOvrv1vA+ACOzB2+htt +Qc8Bsem4yWb02ybzOqR08kkkW8mw0FfB+j564ZfJ +-----END CERTIFICATE----- + +# Issuer: CN=SwissSign Silver CA - G2 O=SwissSign AG +# Subject: CN=SwissSign Silver CA - G2 O=SwissSign AG +# Label: "SwissSign Silver CA - G2" +# Serial: 5700383053117599563 +# MD5 Fingerprint: e0:06:a1:c9:7d:cf:c9:fc:0d:c0:56:75:96:d8:62:13 +# SHA1 Fingerprint: 9b:aa:e5:9f:56:ee:21:cb:43:5a:be:25:93:df:a7:f0:40:d1:1d:cb +# SHA256 Fingerprint: be:6c:4d:a2:bb:b9:ba:59:b6:f3:93:97:68:37:42:46:c3:c0:05:99:3f:a9:8f:02:0d:1d:ed:be:d4:8a:81:d5 +-----BEGIN CERTIFICATE----- +MIIFvTCCA6WgAwIBAgIITxvUL1S7L0swDQYJKoZIhvcNAQEFBQAwRzELMAkGA1UE +BhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzEhMB8GA1UEAxMYU3dpc3NTaWdu +IFNpbHZlciBDQSAtIEcyMB4XDTA2MTAyNTA4MzI0NloXDTM2MTAyNTA4MzI0Nlow +RzELMAkGA1UEBhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzEhMB8GA1UEAxMY +U3dpc3NTaWduIFNpbHZlciBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A +MIICCgKCAgEAxPGHf9N4Mfc4yfjDmUO8x/e8N+dOcbpLj6VzHVxumK4DV644N0Mv +Fz0fyM5oEMF4rhkDKxD6LHmD9ui5aLlV8gREpzn5/ASLHvGiTSf5YXu6t+WiE7br +YT7QbNHm+/pe7R20nqA1W6GSy/BJkv6FCgU+5tkL4k+73JU3/JHpMjUi0R86TieF +nbAVlDLaYQ1HTWBCrpJH6INaUFjpiou5XaHc3ZlKHzZnu0jkg7Y360g6rw9njxcH +6ATK72oxh9TAtvmUcXtnZLi2kUpCe2UuMGoM9ZDulebyzYLs2aFK7PayS+VFheZt +eJMELpyCbTapxDFkH4aDCyr0NQp4yVXPQbBH6TCfmb5hqAaEuSh6XzjZG6k4sIN/ +c8HDO0gqgg8hm7jMqDXDhBuDsz6+pJVpATqJAHgE2cn0mRmrVn5bi4Y5FZGkECwJ +MoBgs5PAKrYYC51+jUnyEEp/+dVGLxmSo5mnJqy7jDzmDrxHB9xzUfFwZC8I+bRH +HTBsROopN4WSaGa8gzj+ezku01DwH/teYLappvonQfGbGHLy9YR0SslnxFSuSGTf +jNFusB3hB48IHpmccelM2KX3RxIfdNFRnobzwqIjQAtz20um53MGjMGg6cFZrEb6 +5i/4z3GcRm25xBWNOHkDRUjvxF3XCO6HOSKGsg0PWEP3calILv3q1h8CAwEAAaOB +rDCBqTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +F6DNweRBtjpbO8tFnb0cwpj6hlgwHwYDVR0jBBgwFoAUF6DNweRBtjpbO8tFnb0c +wpj6hlgwRgYDVR0gBD8wPTA7BglghXQBWQEDAQEwLjAsBggrBgEFBQcCARYgaHR0 +cDovL3JlcG9zaXRvcnkuc3dpc3NzaWduLmNvbS8wDQYJKoZIhvcNAQEFBQADggIB +AHPGgeAn0i0P4JUw4ppBf1AsX19iYamGamkYDHRJ1l2E6kFSGG9YrVBWIGrGvShp +WJHckRE1qTodvBqlYJ7YH39FkWnZfrt4csEGDyrOj4VwYaygzQu4OSlWhDJOhrs9 +xCrZ1x9y7v5RoSJBsXECYxqCsGKrXlcSH9/L3XWgwF15kIwb4FDm3jH+mHtwX6WQ +2K34ArZv02DdQEsixT2tOnqfGhpHkXkzuoLcMmkDlm4fS/Bx/uNncqCxv1yL5PqZ +IseEuRuNI5c/7SXgz2W79WEE790eslpBIlqhn10s6FvJbakMDHiqYMZWjwFaDGi8 +aRl5xB9+lwW/xekkUV7U1UtT7dkjWjYDZaPBA61BMPNGG4WQr2W11bHkFlt4dR2X +em1ZqSqPe97Dh4kQmUlzeMg9vVE1dCrV8X5pGyq7O70luJpaPXJhkGaH7gzWTdQR +dAtq/gsD/KNVV4n+SsuuWxcFyPKNIzFTONItaj+CuY0IavdeQXRuwxF+B6wpYJE/ +OMpXEA29MC/HpeZBoNquBYeaoKRlbEwJDIm6uNO5wJOKMPqN5ZprFQFOZ6raYlY+ +hAhm0sQ2fac+EPyI4NSA5QC9qvNOBqN6avlicuMJT+ubDgEj8Z+7fNzcbBGXJbLy +tGMU0gYqZ4yD9c7qB9iaah7s5Aq7KkzrCWA5zspi2C5u +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc. +# Subject: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc. +# Label: "GeoTrust Primary Certification Authority" +# Serial: 32798226551256963324313806436981982369 +# MD5 Fingerprint: 02:26:c3:01:5e:08:30:37:43:a9:d0:7d:cf:37:e6:bf +# SHA1 Fingerprint: 32:3c:11:8e:1b:f7:b8:b6:52:54:e2:e2:10:0d:d6:02:90:37:f0:96 +# SHA256 Fingerprint: 37:d5:10:06:c5:12:ea:ab:62:64:21:f1:ec:8c:92:01:3f:c5:f8:2a:e9:8e:e5:33:eb:46:19:b8:de:b4:d0:6c +-----BEGIN CERTIFICATE----- +MIIDfDCCAmSgAwIBAgIQGKy1av1pthU6Y2yv2vrEoTANBgkqhkiG9w0BAQUFADBY +MQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjExMC8GA1UEAxMo +R2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEx +MjcwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMFgxCzAJBgNVBAYTAlVTMRYwFAYDVQQK +Ew1HZW9UcnVzdCBJbmMuMTEwLwYDVQQDEyhHZW9UcnVzdCBQcmltYXJ5IENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAvrgVe//UfH1nrYNke8hCUy3f9oQIIGHWAVlqnEQRr+92/ZV+zmEwu3qDXwK9 +AWbK7hWNb6EwnL2hhZ6UOvNWiAAxz9juapYC2e0DjPt1befquFUWBRaa9OBesYjA +ZIVcFU2Ix7e64HXprQU9nceJSOC7KMgD4TCTZF5SwFlwIjVXiIrxlQqD17wxcwE0 +7e9GceBrAqg1cmuXm2bgyxx5X9gaBGgeRwLmnWDiNpcB3841kt++Z8dtd1k7j53W +kBWUvEI0EME5+bEnPn7WinXFsq+W06Lem+SYvn3h6YGttm/81w7a4DSwDRp35+MI +mO9Y+pyEtzavwt+s0vQQBnBxNQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4G +A1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQULNVQQZcVi/CPNmFbSvtr2ZnJM5IwDQYJ +KoZIhvcNAQEFBQADggEBAFpwfyzdtzRP9YZRqSa+S7iq8XEN3GHHoOo0Hnp3DwQ1 +6CePbJC/kRYkRj5KTs4rFtULUh38H2eiAkUxT87z+gOneZ1TatnaYzr4gNfTmeGl +4b7UVXGYNTq+k+qurUKykG/g/CFNNWMziUnWm07Kx+dOCQD32sfvmWKZd7aVIl6K +oKv0uHiYyjgZmclynnjNS6yvGaBzEi38wkG6gZHaFloxt/m0cYASSJlyc1pZU8Fj +UjPtp8nSOQJw+uCxQmYpqptR7TBUIhRf2asdweSU8Pj1K/fqynhG1riR/aYNKxoU +AT6A8EKglQdebc3MS6RFjasS6LPeWuWgfOgPIh1a6Vk= +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA" +# Serial: 69529181992039203566298953787712940909 +# MD5 Fingerprint: 8c:ca:dc:0b:22:ce:f5:be:72:ac:41:1a:11:a8:d8:12 +# SHA1 Fingerprint: 91:c6:d6:ee:3e:8a:c8:63:84:e5:48:c2:99:29:5c:75:6c:81:7b:81 +# SHA256 Fingerprint: 8d:72:2f:81:a9:c1:13:c0:79:1d:f1:36:a2:96:6d:b2:6c:95:0a:97:1d:b4:6b:41:99:f4:ea:54:b7:8b:fb:9f +-----BEGIN CERTIFICATE----- +MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB +qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV +BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw +NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j +LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG +A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl +IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs +W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta +3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk +6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6 +Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J +NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA +MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP +r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU +DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz +YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX +xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2 +/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/ +LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 +jVaMaA== +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Class 3 Public Primary Certification Authority - G5" +# Serial: 33037644167568058970164719475676101450 +# MD5 Fingerprint: cb:17:e4:31:67:3e:e2:09:fe:45:57:93:f3:0a:fa:1c +# SHA1 Fingerprint: 4e:b6:d5:78:49:9b:1c:cf:5f:58:1e:ad:56:be:3d:9b:67:44:a5:e5 +# SHA256 Fingerprint: 9a:cf:ab:7e:43:c8:d8:80:d0:6b:26:2a:94:de:ee:e4:b4:65:99:89:c3:d0:ca:f1:9b:af:64:05:e4:1a:b7:df +-----BEGIN CERTIFICATE----- +MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB +yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW +ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1 +nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex +t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz +SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG +BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+ +rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/ +NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH +BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy +aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv +MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE +p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y +5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK +WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ +4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N +hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq +-----END CERTIFICATE----- + +# Issuer: CN=SecureTrust CA O=SecureTrust Corporation +# Subject: CN=SecureTrust CA O=SecureTrust Corporation +# Label: "SecureTrust CA" +# Serial: 17199774589125277788362757014266862032 +# MD5 Fingerprint: dc:32:c3:a7:6d:25:57:c7:68:09:9d:ea:2d:a9:a2:d1 +# SHA1 Fingerprint: 87:82:c6:c3:04:35:3b:cf:d2:96:92:d2:59:3e:7d:44:d9:34:ff:11 +# SHA256 Fingerprint: f1:c1:b5:0a:e5:a2:0d:d8:03:0e:c9:f6:bc:24:82:3d:d3:67:b5:25:57:59:b4:e7:1b:61:fc:e9:f7:37:5d:73 +-----BEGIN CERTIFICATE----- +MIIDuDCCAqCgAwIBAgIQDPCOXAgWpa1Cf/DrJxhZ0DANBgkqhkiG9w0BAQUFADBI +MQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3QgQ29ycG9yYXRpb24x +FzAVBgNVBAMTDlNlY3VyZVRydXN0IENBMB4XDTA2MTEwNzE5MzExOFoXDTI5MTIz +MTE5NDA1NVowSDELMAkGA1UEBhMCVVMxIDAeBgNVBAoTF1NlY3VyZVRydXN0IENv +cnBvcmF0aW9uMRcwFQYDVQQDEw5TZWN1cmVUcnVzdCBDQTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAKukgeWVzfX2FI7CT8rU4niVWJxB4Q2ZQCQXOZEz +Zum+4YOvYlyJ0fwkW2Gz4BERQRwdbvC4u/jep4G6pkjGnx29vo6pQT64lO0pGtSO +0gMdA+9tDWccV9cGrcrI9f4Or2YlSASWC12juhbDCE/RRvgUXPLIXgGZbf2IzIao +wW8xQmxSPmjL8xk037uHGFaAJsTQ3MBv396gwpEWoGQRS0S8Hvbn+mPeZqx2pHGj +7DaUaHp3pLHnDi+BeuK1cobvomuL8A/b01k/unK8RCSc43Oz969XL0Imnal0ugBS +8kvNU3xHCzaFDmapCJcWNFfBZveA4+1wVMeT4C4oFVmHursCAwEAAaOBnTCBmjAT +BgkrBgEEAYI3FAIEBh4EAEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUQjK2FvoE/f5dS3rD/fdMQB1aQ68wNAYDVR0fBC0wKzApoCeg +JYYjaHR0cDovL2NybC5zZWN1cmV0cnVzdC5jb20vU1RDQS5jcmwwEAYJKwYBBAGC +NxUBBAMCAQAwDQYJKoZIhvcNAQEFBQADggEBADDtT0rhWDpSclu1pqNlGKa7UTt3 +6Z3q059c4EVlew3KW+JwULKUBRSuSceNQQcSc5R+DCMh/bwQf2AQWnL1mA6s7Ll/ +3XpvXdMc9P+IBWlCqQVxyLesJugutIxq/3HcuLHfmbx8IVQr5Fiiu1cprp6poxkm +D5kuCLDv/WnPmRoJjeOnnyvJNjR7JLN4TJUXpAYmHrZkUjZfYGfZnMUFdAvnZyPS +CPyI6a6Lf+Ew9Dd+/cYy2i2eRDAwbO4H3tI0/NL/QPZL9GZGBlSm8jIKYyYwa5vR +3ItHuuG51WLQoqD0ZwV4KWMabwTW+MZMo5qxN7SN5ShLHZ4swrhovO0C7jE= +-----END CERTIFICATE----- + +# Issuer: CN=Secure Global CA O=SecureTrust Corporation +# Subject: CN=Secure Global CA O=SecureTrust Corporation +# Label: "Secure Global CA" +# Serial: 9751836167731051554232119481456978597 +# MD5 Fingerprint: cf:f4:27:0d:d4:ed:dc:65:16:49:6d:3d:da:bf:6e:de +# SHA1 Fingerprint: 3a:44:73:5a:e5:81:90:1f:24:86:61:46:1e:3b:9c:c4:5f:f5:3a:1b +# SHA256 Fingerprint: 42:00:f5:04:3a:c8:59:0e:bb:52:7d:20:9e:d1:50:30:29:fb:cb:d4:1c:a1:b5:06:ec:27:f1:5a:de:7d:ac:69 +-----BEGIN CERTIFICATE----- +MIIDvDCCAqSgAwIBAgIQB1YipOjUiolN9BPI8PjqpTANBgkqhkiG9w0BAQUFADBK +MQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3QgQ29ycG9yYXRpb24x +GTAXBgNVBAMTEFNlY3VyZSBHbG9iYWwgQ0EwHhcNMDYxMTA3MTk0MjI4WhcNMjkx +MjMxMTk1MjA2WjBKMQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3Qg +Q29ycG9yYXRpb24xGTAXBgNVBAMTEFNlY3VyZSBHbG9iYWwgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvNS7YrGxVaQZx5RNoJLNP2MwhR/jxYDiJ +iQPpvepeRlMJ3Fz1Wuj3RSoC6zFh1ykzTM7HfAo3fg+6MpjhHZevj8fcyTiW89sa +/FHtaMbQbqR8JNGuQsiWUGMu4P51/pinX0kuleM5M2SOHqRfkNJnPLLZ/kG5VacJ +jnIFHovdRIWCQtBJwB1g8NEXLJXr9qXBkqPFwqcIYA1gBBCWeZ4WNOaptvolRTnI +HmX5k/Wq8VLcmZg9pYYaDDUz+kulBAYVHDGA76oYa8J719rO+TMg1fW9ajMtgQT7 +sFzUnKPiXB3jqUJ1XnvUd+85VLrJChgbEplJL4hL/VBi0XPnj3pDAgMBAAGjgZ0w +gZowEwYJKwYBBAGCNxQCBAYeBABDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFK9EBMJBfkiD2045AuzshHrmzsmkMDQGA1UdHwQtMCsw +KaAnoCWGI2h0dHA6Ly9jcmwuc2VjdXJldHJ1c3QuY29tL1NHQ0EuY3JsMBAGCSsG +AQQBgjcVAQQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQBjGghAfaReUw132HquHw0L +URYD7xh8yOOvaliTFGCRsoTciE6+OYo68+aCiV0BN7OrJKQVDpI1WkpEXk5X+nXO +H0jOZvQ8QCaSmGwb7iRGDBezUqXbpZGRzzfTb+cnCDpOGR86p1hcF895P4vkp9Mm +I50mD1hp/Ed+stCNi5O/KU9DaXR2Z0vPB4zmAve14bRDtUstFJ/53CYNv6ZHdAbY +iNE6KTCEztI5gGIbqMdXSbxqVVFnFUq+NQfk1XWYN3kwFNspnWzFacxHVaIw98xc +f8LDmBxrThaA63p4ZUWiABqvDA1VZDRIuJK58bRQKfJPIx/abKwfROHdI3hRW8cW +-----END CERTIFICATE----- + +# Issuer: CN=COMODO Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO Certification Authority O=COMODO CA Limited +# Label: "COMODO Certification Authority" +# Serial: 104350513648249232941998508985834464573 +# MD5 Fingerprint: 5c:48:dc:f7:42:72:ec:56:94:6d:1c:cc:71:35:80:75 +# SHA1 Fingerprint: 66:31:bf:9e:f7:4f:9e:b6:c9:d5:a6:0c:ba:6a:be:d1:f7:bd:ef:7b +# SHA256 Fingerprint: 0c:2c:d6:3d:f7:80:6f:a3:99:ed:e8:09:11:6b:57:5b:f8:79:89:f0:65:18:f9:80:8c:86:05:03:17:8b:af:66 +-----BEGIN CERTIFICATE----- +MIIEHTCCAwWgAwIBAgIQToEtioJl4AsC7j41AkblPTANBgkqhkiG9w0BAQUFADCB +gTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNV +BAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEyMDEwMDAw +MDBaFw0yOTEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEbMBkGA1UECBMSR3Jl +YXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFDT01P +RE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3 +UcEbVASY06m/weaKXTuH+7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI +2GqGd0S7WWaXUF601CxwRM/aN5VCaTwwxHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8 +Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV4EajcNxo2f8ESIl33rXp ++2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA1KGzqSX+ +DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5O +nKVIrLsm9wIDAQABo4GOMIGLMB0GA1UdDgQWBBQLWOWLxkwVN6RAqTCpIb5HNlpW +/zAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zBJBgNVHR8EQjBAMD6g +PKA6hjhodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9DZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDANBgkqhkiG9w0BAQUFAAOCAQEAPpiem/Yb6dc5t3iuHXIY +SdOH5EOC6z/JqvWote9VfCFSZfnVDeFs9D6Mk3ORLgLETgdxb8CPOGEIqB6BCsAv +IC9Bi5HcSEW88cbeunZrM8gALTFGTO3nnc+IlP8zwFboJIYmuNg4ON8qa90SzMc/ +RxdMosIGlgnW2/4/PEZB31jiVg88O8EckzXZOFKs7sjsLjBOlDW0JB9LeGna8gI4 +zJVSk/BwJVmcIGfE7vmLV2H0knZ9P4SNVbfo5azV8fUZVqZa+5Acr5Pr5RzUZ5dd +BA6+C4OmF4O5MBKgxTMVBbkN+8cFduPYSo38NBejxiEovjBFMR7HeL5YYTisO+IB +ZQ== +-----END CERTIFICATE----- + +# Issuer: CN=Network Solutions Certificate Authority O=Network Solutions L.L.C. +# Subject: CN=Network Solutions Certificate Authority O=Network Solutions L.L.C. +# Label: "Network Solutions Certificate Authority" +# Serial: 116697915152937497490437556386812487904 +# MD5 Fingerprint: d3:f3:a6:16:c0:fa:6b:1d:59:b1:2d:96:4d:0e:11:2e +# SHA1 Fingerprint: 74:f8:a3:c3:ef:e7:b3:90:06:4b:83:90:3c:21:64:60:20:e5:df:ce +# SHA256 Fingerprint: 15:f0:ba:00:a3:ac:7a:f3:ac:88:4c:07:2b:10:11:a0:77:bd:77:c0:97:f4:01:64:b2:f8:59:8a:bd:83:86:0c +-----BEGIN CERTIFICATE----- +MIID5jCCAs6gAwIBAgIQV8szb8JcFuZHFhfjkDFo4DANBgkqhkiG9w0BAQUFADBi +MQswCQYDVQQGEwJVUzEhMB8GA1UEChMYTmV0d29yayBTb2x1dGlvbnMgTC5MLkMu +MTAwLgYDVQQDEydOZXR3b3JrIFNvbHV0aW9ucyBDZXJ0aWZpY2F0ZSBBdXRob3Jp +dHkwHhcNMDYxMjAxMDAwMDAwWhcNMjkxMjMxMjM1OTU5WjBiMQswCQYDVQQGEwJV +UzEhMB8GA1UEChMYTmV0d29yayBTb2x1dGlvbnMgTC5MLkMuMTAwLgYDVQQDEydO +ZXR3b3JrIFNvbHV0aW9ucyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDkvH6SMG3G2I4rC7xGzuAnlt7e+foS0zwz +c7MEL7xxjOWftiJgPl9dzgn/ggwbmlFQGiaJ3dVhXRncEg8tCqJDXRfQNJIg6nPP +OCwGJgl6cvf6UDL4wpPTaaIjzkGxzOTVHzbRijr4jGPiFFlp7Q3Tf2vouAPlT2rl +mGNpSAW+Lv8ztumXWWn4Zxmuk2GWRBXTcrA/vGp97Eh/jcOrqnErU2lBUzS1sLnF +BgrEsEX1QV1uiUV7PTsmjHTC5dLRfbIR1PtYMiKagMnc/Qzpf14Dl847ABSHJ3A4 +qY5usyd2mFHgBeMhqxrVhSI8KbWaFsWAqPS7azCPL0YCorEMIuDTAgMBAAGjgZcw +gZQwHQYDVR0OBBYEFCEwyfsA106Y2oeqKtCnLrFAMadMMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MFIGA1UdHwRLMEkwR6BFoEOGQWh0dHA6Ly9jcmwu +bmV0c29sc3NsLmNvbS9OZXR3b3JrU29sdXRpb25zQ2VydGlmaWNhdGVBdXRob3Jp +dHkuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQC7rkvnt1frf6ott3NHhWrB5KUd5Oc8 +6fRZZXe1eltajSU24HqXLjjAV2CDmAaDn7l2em5Q4LqILPxFzBiwmZVRDuwduIj/ +h1AcgsLj4DKAv6ALR8jDMe+ZZzKATxcheQxpXN5eNK4CtSbqUN9/GGUsyfJj4akH +/nxxH2szJGoeBfcFaMBqEssuXmHLrijTfsK0ZpEmXzwuJF/LWA/rKOyvEZbz3Htv +wKeI8lN3s2Berq4o2jUsbzRF0ybh3uxbTydrFny9RAQYgrOJeRcQcT16ohZO9QHN +pGxlaKFJdlxDydi8NmdspZS11My5vWo1ViHe2MPr+8ukYEywVaCge1ey +-----END CERTIFICATE----- + +# Issuer: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Label: "COMODO ECC Certification Authority" +# Serial: 41578283867086692638256921589707938090 +# MD5 Fingerprint: 7c:62:ff:74:9d:31:53:5e:68:4a:d5:78:aa:1e:bf:23 +# SHA1 Fingerprint: 9f:74:4e:9f:2b:4d:ba:ec:0f:31:2c:50:b6:56:3b:8e:2d:93:c3:11 +# SHA256 Fingerprint: 17:93:92:7a:06:14:54:97:89:ad:ce:2f:8f:34:f7:f0:b6:6d:0f:3a:e3:a3:b8:4d:21:ec:15:db:ba:4f:ad:c7 +-----BEGIN CERTIFICATE----- +MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT +IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw +MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy +ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N +T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR +FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J +cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW +BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm +fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv +GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GA CA O=WISeKey OU=Copyright (c) 2005/OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GA CA O=WISeKey OU=Copyright (c) 2005/OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GA CA" +# Serial: 86718877871133159090080555911823548314 +# MD5 Fingerprint: bc:6c:51:33:a7:e9:d3:66:63:54:15:72:1b:21:92:93 +# SHA1 Fingerprint: 59:22:a1:e1:5a:ea:16:35:21:f8:98:39:6a:46:46:b0:44:1b:0f:a9 +# SHA256 Fingerprint: 41:c9:23:86:6a:b4:ca:d6:b7:ad:57:80:81:58:2e:02:07:97:a6:cb:df:4f:ff:78:ce:83:96:b3:89:37:d7:f5 +-----BEGIN CERTIFICATE----- +MIID8TCCAtmgAwIBAgIQQT1yx/RrH4FDffHSKFTfmjANBgkqhkiG9w0BAQUFADCB +ijELMAkGA1UEBhMCQ0gxEDAOBgNVBAoTB1dJU2VLZXkxGzAZBgNVBAsTEkNvcHly +aWdodCAoYykgMjAwNTEiMCAGA1UECxMZT0lTVEUgRm91bmRhdGlvbiBFbmRvcnNl +ZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwgUm9vdCBHQSBDQTAeFw0w +NTEyMTExNjAzNDRaFw0zNzEyMTExNjA5NTFaMIGKMQswCQYDVQQGEwJDSDEQMA4G +A1UEChMHV0lTZUtleTEbMBkGA1UECxMSQ29weXJpZ2h0IChjKSAyMDA1MSIwIAYD +VQQLExlPSVNURSBGb3VuZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBX +SVNlS2V5IEdsb2JhbCBSb290IEdBIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAy0+zAJs9Nt350UlqaxBJH+zYK7LG+DKBKUOVTJoZIyEVRd7jyBxR +VVuuk+g3/ytr6dTqvirdqFEr12bDYVxgAsj1znJ7O7jyTmUIms2kahnBAbtzptf2 +w93NvKSLtZlhuAGio9RN1AU9ka34tAhxZK9w8RxrfvbDd50kc3vkDIzh2TbhmYsF +mQvtRTEJysIA2/dyoJaqlYfQjse2YXMNdmaM3Bu0Y6Kff5MTMPGhJ9vZ/yxViJGg +4E8HsChWjBgbl0SOid3gF27nKu+POQoxhILYQBRJLnpB5Kf+42TMwVlxSywhp1t9 +4B3RLoGbw9ho972WG6xwsRYUC9tguSYBBQIDAQABo1EwTzALBgNVHQ8EBAMCAYYw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUswN+rja8sHnR3JQmthG+IbJphpQw +EAYJKwYBBAGCNxUBBAMCAQAwDQYJKoZIhvcNAQEFBQADggEBAEuh/wuHbrP5wUOx +SPMowB0uyQlB+pQAHKSkq0lPjz0e701vvbyk9vImMMkQyh2I+3QZH4VFvbBsUfk2 +ftv1TDI6QU9bR8/oCy22xBmddMVHxjtqD6wU2zz0c5ypBd8A3HR4+vg1YFkCExh8 +vPtNsCBtQ7tgMHpnM1zFmdH4LTlSc/uMqpclXHLZCB6rTjzjgTGfA6b7wP4piFXa +hNVQA7bihKOmNqoROgHhGEvWRGizPflTdISzRpFGlgC3gCy24eMQ4tui5yiPAZZi +Fj4A4xylNoEYokxSdsARo27mHbrjWr42U8U+dY+GaSlYU7Wcu2+fXMUY7N0v4ZjJ +/L7fCg0= +-----END CERTIFICATE----- + +# Issuer: CN=Certigna O=Dhimyotis +# Subject: CN=Certigna O=Dhimyotis +# Label: "Certigna" +# Serial: 18364802974209362175 +# MD5 Fingerprint: ab:57:a6:5b:7d:42:82:19:b5:d8:58:26:28:5e:fd:ff +# SHA1 Fingerprint: b1:2e:13:63:45:86:a4:6f:1a:b2:60:68:37:58:2d:c4:ac:fd:94:97 +# SHA256 Fingerprint: e3:b6:a2:db:2e:d7:ce:48:84:2f:7a:c5:32:41:c7:b7:1d:54:14:4b:fb:40:c1:1f:3f:1d:0b:42:f5:ee:a1:2d +-----BEGIN CERTIFICATE----- +MIIDqDCCApCgAwIBAgIJAP7c4wEPyUj/MA0GCSqGSIb3DQEBBQUAMDQxCzAJBgNV +BAYTAkZSMRIwEAYDVQQKDAlEaGlteW90aXMxETAPBgNVBAMMCENlcnRpZ25hMB4X +DTA3MDYyOTE1MTMwNVoXDTI3MDYyOTE1MTMwNVowNDELMAkGA1UEBhMCRlIxEjAQ +BgNVBAoMCURoaW15b3RpczERMA8GA1UEAwwIQ2VydGlnbmEwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDIaPHJ1tazNHUmgh7stL7qXOEm7RFHYeGifBZ4 +QCHkYJ5ayGPhxLGWkv8YbWkj4Sti993iNi+RB7lIzw7sebYs5zRLcAglozyHGxny +gQcPOJAZ0xH+hrTy0V4eHpbNgGzOOzGTtvKg0KmVEn2lmsxryIRWijOp5yIVUxbw +zBfsV1/pogqYCd7jX5xv3EjjhQsVWqa6n6xI4wmy9/Qy3l40vhx4XUJbzg4ij02Q +130yGLMLLGq/jj8UEYkgDncUtT2UCIf3JR7VsmAA7G8qKCVuKj4YYxclPz5EIBb2 +JsglrgVKtOdjLPOMFlN+XPsRGgjBRmKfIrjxwo1p3Po6WAbfAgMBAAGjgbwwgbkw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUGu3+QTmQtCRZvgHyUtVF9lo53BEw +ZAYDVR0jBF0wW4AUGu3+QTmQtCRZvgHyUtVF9lo53BGhOKQ2MDQxCzAJBgNVBAYT +AkZSMRIwEAYDVQQKDAlEaGlteW90aXMxETAPBgNVBAMMCENlcnRpZ25hggkA/tzj +AQ/JSP8wDgYDVR0PAQH/BAQDAgEGMBEGCWCGSAGG+EIBAQQEAwIABzANBgkqhkiG +9w0BAQUFAAOCAQEAhQMeknH2Qq/ho2Ge6/PAD/Kl1NqV5ta+aDY9fm4fTIrv0Q8h +bV6lUmPOEvjvKtpv6zf+EwLHyzs+ImvaYS5/1HI93TDhHkxAGYwP15zRgzB7mFnc +fca5DClMoTOi62c6ZYTTluLtdkVwj7Ur3vkj1kluPBS1xp81HlDQwY9qcEQCYsuu +HWhBp6pX6FOqB9IG9tUUBguRA3UsbHK1YZWaDYu5Def131TN3ubY1gkIl2PlwS6w +t0QmwCbAr1UwnjvVNioZBPRcHv/PLLf/0P2HQBHVESO7SMAhqaQoLf0V+LBOK/Qw +WyH8EZE0vkHve52Xdf+XlcCWWC/qu0bXu+TZLg== +-----END CERTIFICATE----- + +# Issuer: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center +# Subject: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center +# Label: "Deutsche Telekom Root CA 2" +# Serial: 38 +# MD5 Fingerprint: 74:01:4a:91:b1:08:c4:58:ce:47:cd:f0:dd:11:53:08 +# SHA1 Fingerprint: 85:a4:08:c0:9c:19:3e:5d:51:58:7d:cd:d6:13:30:fd:8c:de:37:bf +# SHA256 Fingerprint: b6:19:1a:50:d0:c3:97:7f:7d:a9:9b:cd:aa:c8:6a:22:7d:ae:b9:67:9e:c7:0b:a3:b0:c9:d9:22:71:c1:70:d3 +-----BEGIN CERTIFICATE----- +MIIDnzCCAoegAwIBAgIBJjANBgkqhkiG9w0BAQUFADBxMQswCQYDVQQGEwJERTEc +MBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxlU2Vj +IFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290IENB +IDIwHhcNOTkwNzA5MTIxMTAwWhcNMTkwNzA5MjM1OTAwWjBxMQswCQYDVQQGEwJE +RTEcMBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxl +U2VjIFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290 +IENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrC6M14IspFLEU +ha88EOQ5bzVdSq7d6mGNlUn0b2SjGmBmpKlAIoTZ1KXleJMOaAGtuU1cOs7TuKhC +QN/Po7qCWWqSG6wcmtoIKyUn+WkjR/Hg6yx6m/UTAtB+NHzCnjwAWav12gz1Mjwr +rFDa1sPeg5TKqAyZMg4ISFZbavva4VhYAUlfckE8FQYBjl2tqriTtM2e66foai1S +NNs671x1Udrb8zH57nGYMsRUFUQM+ZtV7a3fGAigo4aKSe5TBY8ZTNXeWHmb0moc +QqvF1afPaA+W5OFhmHZhyJF81j4A4pFQh+GdCuatl9Idxjp9y7zaAzTVjlsB9WoH +txa2bkp/AgMBAAGjQjBAMB0GA1UdDgQWBBQxw3kbuvVT1xfgiXotF2wKsyudMzAP +BgNVHRMECDAGAQH/AgEFMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOC +AQEAlGRZrTlk5ynrE/5aw4sTV8gEJPB0d8Bg42f76Ymmg7+Wgnxu1MM9756Abrsp +tJh6sTtU6zkXR34ajgv8HzFZMQSyzhfzLMdiNlXiItiJVbSYSKpk+tYcNthEeFpa +IzpXl/V6ME+un2pMSyuOoAPjPuCp1NJ70rOo4nI8rZ7/gFnkm0W09juwzTkZmDLl +6iFhkOQxIY40sfcvNUqFENrnijchvllj4PKFiDFT1FQUhXB59C4Gdyd1Lx+4ivn+ +xbrYNuSD7Odlt79jWvNGr4GUN9RBjNYj1h7P9WgbRGOiWrqnNVmh5XAFmw4jV5mU +Cm26OWMohpLzGITY+9HPBVZkVw== +-----END CERTIFICATE----- + +# Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc +# Subject: CN=Cybertrust Global Root O=Cybertrust, Inc +# Label: "Cybertrust Global Root" +# Serial: 4835703278459682877484360 +# MD5 Fingerprint: 72:e4:4a:87:e3:69:40:80:77:ea:bc:e3:f4:ff:f0:e1 +# SHA1 Fingerprint: 5f:43:e5:b1:bf:f8:78:8c:ac:1c:c7:ca:4a:9a:c6:22:2b:cc:34:c6 +# SHA256 Fingerprint: 96:0a:df:00:63:e9:63:56:75:0c:29:65:dd:0a:08:67:da:0b:9c:bd:6e:77:71:4a:ea:fb:23:49:ab:39:3d:a3 +-----BEGIN CERTIFICATE----- +MIIDoTCCAomgAwIBAgILBAAAAAABD4WqLUgwDQYJKoZIhvcNAQEFBQAwOzEYMBYG +A1UEChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2Jh +bCBSb290MB4XDTA2MTIxNTA4MDAwMFoXDTIxMTIxNTA4MDAwMFowOzEYMBYGA1UE +ChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2JhbCBS +b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+Mi8vRRQZhP/8NN5 +7CPytxrHjoXxEnOmGaoQ25yiZXRadz5RfVb23CO21O1fWLE3TdVJDm71aofW0ozS +J8bi/zafmGWgE07GKmSb1ZASzxQG9Dvj1Ci+6A74q05IlG2OlTEQXO2iLb3VOm2y +HLtgwEZLAfVJrn5GitB0jaEMAs7u/OePuGtm839EAL9mJRQr3RAwHQeWP032a7iP +t3sMpTjr3kfb1V05/Iin89cqdPHoWqI7n1C6poxFNcJQZZXcY4Lv3b93TZxiyWNz +FtApD0mpSPCzqrdsxacwOUBdrsTiXSZT8M4cIwhhqJQZugRiQOwfOHB3EgZxpzAY +XSUnpQIDAQABo4GlMIGiMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ +MB0GA1UdDgQWBBS2CHsNesysIEyGVjJez6tuhS1wVzA/BgNVHR8EODA2MDSgMqAw +hi5odHRwOi8vd3d3Mi5wdWJsaWMtdHJ1c3QuY29tL2NybC9jdC9jdHJvb3QuY3Js +MB8GA1UdIwQYMBaAFLYIew16zKwgTIZWMl7Pq26FLXBXMA0GCSqGSIb3DQEBBQUA +A4IBAQBW7wojoFROlZfJ+InaRcHUowAl9B8Tq7ejhVhpwjCt2BWKLePJzYFa+HMj +Wqd8BfP9IjsO0QbE2zZMcwSO5bAi5MXzLqXZI+O4Tkogp24CJJ8iYGd7ix1yCcUx +XOl5n4BHPa2hCwcUPUf/A2kaDAtE52Mlp3+yybh2hO0j9n0Hq0V+09+zv+mKts2o +omcrUtW3ZfA5TGOgkXmTUg9U3YO7n9GPp1Nzw8v/MOx8BLjYRB+TX3EJIrduPuoc +A06dGiBh+4E37F78CkWr1+cXVdCg6mCbpvbjjFspwgZgFJ0tl0ypkxWdYcQBX0jW +WL1WMRJOEcgh4LMRkWXbtKaIOM5V +-----END CERTIFICATE----- + +# Issuer: O=Chunghwa Telecom Co., Ltd. OU=ePKI Root Certification Authority +# Subject: O=Chunghwa Telecom Co., Ltd. OU=ePKI Root Certification Authority +# Label: "ePKI Root Certification Authority" +# Serial: 28956088682735189655030529057352760477 +# MD5 Fingerprint: 1b:2e:00:ca:26:06:90:3d:ad:fe:6f:15:68:d3:6b:b3 +# SHA1 Fingerprint: 67:65:0d:f1:7e:8e:7e:5b:82:40:a4:f4:56:4b:cf:e2:3d:69:c6:f0 +# SHA256 Fingerprint: c0:a6:f4:dc:63:a2:4b:fd:cf:54:ef:2a:6a:08:2a:0a:72:de:35:80:3e:2f:f5:ff:52:7a:e5:d8:72:06:df:d5 +-----BEGIN CERTIFICATE----- +MIIFsDCCA5igAwIBAgIQFci9ZUdcr7iXAF7kBtK8nTANBgkqhkiG9w0BAQUFADBe +MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 +ZC4xKjAoBgNVBAsMIWVQS0kgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe +Fw0wNDEyMjAwMjMxMjdaFw0zNDEyMjAwMjMxMjdaMF4xCzAJBgNVBAYTAlRXMSMw +IQYDVQQKDBpDaHVuZ2h3YSBUZWxlY29tIENvLiwgTHRkLjEqMCgGA1UECwwhZVBL +SSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA4SUP7o3biDN1Z82tH306Tm2d0y8U82N0ywEhajfqhFAH +SyZbCUNsIZ5qyNUD9WBpj8zwIuQf5/dqIjG3LBXy4P4AakP/h2XGtRrBp0xtInAh +ijHyl3SJCRImHJ7K2RKilTza6We/CKBk49ZCt0Xvl/T29de1ShUCWH2YWEtgvM3X +DZoTM1PRYfl61dd4s5oz9wCGzh1NlDivqOx4UXCKXBCDUSH3ET00hl7lSM2XgYI1 +TBnsZfZrxQWh7kcT1rMhJ5QQCtkkO7q+RBNGMD+XPNjX12ruOzjjK9SXDrkb5wdJ +fzcq+Xd4z1TtW0ado4AOkUPB1ltfFLqfpo0kR0BZv3I4sjZsN/+Z0V0OWQqraffA +sgRFelQArr5T9rXn4fg8ozHSqf4hUmTFpmfwdQcGlBSBVcYn5AGPF8Fqcde+S/uU +WH1+ETOxQvdibBjWzwloPn9s9h6PYq2lY9sJpx8iQkEeb5mKPtf5P0B6ebClAZLS +nT0IFaUQAS2zMnaolQ2zepr7BxB4EW/hj8e6DyUadCrlHJhBmd8hh+iVBmoKs2pH +dmX2Os+PYhcZewoozRrSgx4hxyy/vv9haLdnG7t4TY3OZ+XkwY63I2binZB1NJip +NiuKmpS5nezMirH4JYlcWrYvjB9teSSnUmjDhDXiZo1jDiVN1Rmy5nk3pyKdVDEC +AwEAAaNqMGgwHQYDVR0OBBYEFB4M97Zn8uGSJglFwFU5Lnc/QkqiMAwGA1UdEwQF +MAMBAf8wOQYEZyoHAAQxMC8wLQIBADAJBgUrDgMCGgUAMAcGBWcqAwAABBRFsMLH +ClZ87lt4DJX5GFPBphzYEDANBgkqhkiG9w0BAQUFAAOCAgEACbODU1kBPpVJufGB +uvl2ICO1J2B01GqZNF5sAFPZn/KmsSQHRGoqxqWOeBLoR9lYGxMqXnmbnwoqZ6Yl +PwZpVnPDimZI+ymBV3QGypzqKOg4ZyYr8dW1P2WT+DZdjo2NQCCHGervJ8A9tDkP +JXtoUHRVnAxZfVo9QZQlUgjgRywVMRnVvwdVxrsStZf0X4OFunHB2WyBEXYKCrC/ +gpf36j36+uwtqSiUO1bd0lEursC9CBWMd1I0ltabrNMdjmEPNXubrjlpC2JgQCA2 +j6/7Nu4tCEoduL+bXPjqpRugc6bY+G7gMwRfaKonh+3ZwZCc7b3jajWvY9+rGNm6 +5ulK6lCKD2GTHuItGeIwlDWSXQ62B68ZgI9HkFFLLk3dheLSClIKF5r8GrBQAuUB +o2M3IUxExJtRmREOc5wGj1QupyheRDmHVi03vYVElOEMSyycw5KFNGHLD7ibSkNS +/jQ6fbjpKdx2qcgw+BRxgMYeNkh0IkFch4LoGHGLQYlE535YW6i4jRPpp2zDR+2z +Gp1iro2C6pSe3VkQw63d4k3jMdXH7OjysP6SHhYKGvzZ8/gntsm+HbRsZJB/9OTE +W9c3rkIO3aQab3yIVMUWbuF6aC74Or8NpDyJO3inTmODBCEIZ43ygknQW/2xzQ+D +hNQ+IIX3Sj0rnP0qCglN6oH4EZw= +-----END CERTIFICATE----- + +# Issuer: O=certSIGN OU=certSIGN ROOT CA +# Subject: O=certSIGN OU=certSIGN ROOT CA +# Label: "certSIGN ROOT CA" +# Serial: 35210227249154 +# MD5 Fingerprint: 18:98:c0:d6:e9:3a:fc:f9:b0:f5:0c:f7:4b:01:44:17 +# SHA1 Fingerprint: fa:b7:ee:36:97:26:62:fb:2d:b0:2a:f6:bf:03:fd:e8:7c:4b:2f:9b +# SHA256 Fingerprint: ea:a9:62:c4:fa:4a:6b:af:eb:e4:15:19:6d:35:1c:cd:88:8d:4f:53:f3:fa:8a:e6:d7:c4:66:a9:4e:60:42:bb +-----BEGIN CERTIFICATE----- +MIIDODCCAiCgAwIBAgIGIAYFFnACMA0GCSqGSIb3DQEBBQUAMDsxCzAJBgNVBAYT +AlJPMREwDwYDVQQKEwhjZXJ0U0lHTjEZMBcGA1UECxMQY2VydFNJR04gUk9PVCBD +QTAeFw0wNjA3MDQxNzIwMDRaFw0zMTA3MDQxNzIwMDRaMDsxCzAJBgNVBAYTAlJP +MREwDwYDVQQKEwhjZXJ0U0lHTjEZMBcGA1UECxMQY2VydFNJR04gUk9PVCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALczuX7IJUqOtdu0KBuqV5Do +0SLTZLrTk+jUrIZhQGpgV2hUhE28alQCBf/fm5oqrl0Hj0rDKH/v+yv6efHHrfAQ +UySQi2bJqIirr1qjAOm+ukbuW3N7LBeCgV5iLKECZbO9xSsAfsT8AzNXDe3i+s5d +RdY4zTW2ssHQnIFKquSyAVwdj1+ZxLGt24gh65AIgoDzMKND5pCCrlUoSe1b16kQ +OA7+j0xbm0bqQfWwCHTD0IgztnzXdN/chNFDDnU5oSVAKOp4yw4sLjmdjItuFhwv +JoIQ4uNllAoEwF73XVv4EOLQunpL+943AAAaWyjj0pxzPjKHmKHJUS/X3qwzs08C +AwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAcYwHQYDVR0O +BBYEFOCMm9slSbPxfIbWskKHC9BroNnkMA0GCSqGSIb3DQEBBQUAA4IBAQA+0hyJ +LjX8+HXd5n9liPRyTMks1zJO890ZeUe9jjtbkw9QSSQTaxQGcu8J06Gh40CEyecY +MnQ8SG4Pn0vU9x7Tk4ZkVJdjclDVVc/6IJMCopvDI5NOFlV2oHB5bc0hH88vLbwZ +44gx+FkagQnIl6Z0x2DEW8xXjrJ1/RsCCdtZb3KTafcxQdaIOL+Hsr0Wefmq5L6I +Jd1hJyMctTEHBDa0GpC9oHRxUIltvBTjD4au8as+x6AJzKNI0eDbZOeStc+vckNw +i/nDhDwTqn6Sm1dTk/pwwpEOMfmbZ13pljheX7NzTogVZ96edhBiIL5VaZVDADlN +9u6wWk5JRFRYX0KD +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only +# Subject: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only +# Label: "GeoTrust Primary Certification Authority - G3" +# Serial: 28809105769928564313984085209975885599 +# MD5 Fingerprint: b5:e8:34:36:c9:10:44:58:48:70:6d:2e:83:d4:b8:05 +# SHA1 Fingerprint: 03:9e:ed:b8:0b:e7:a0:3c:69:53:89:3b:20:d2:d9:32:3a:4c:2a:fd +# SHA256 Fingerprint: b4:78:b8:12:25:0d:f8:78:63:5c:2a:a7:ec:7d:15:5e:aa:62:5e:e8:29:16:e2:cd:29:43:61:88:6c:d1:fb:d4 +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIQFaxulBmyeUtB9iepwxgPHzANBgkqhkiG9w0BAQsFADCB +mDELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsT +MChjKSAyMDA4IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTE2MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhv +cml0eSAtIEczMB4XDTA4MDQwMjAwMDAwMFoXDTM3MTIwMTIzNTk1OVowgZgxCzAJ +BgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykg +MjAwOCBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0 +BgNVBAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANziXmJYHTNXOTIz ++uvLh4yn1ErdBojqZI4xmKU4kB6Yzy5jK/BGvESyiaHAKAxJcCGVn2TAppMSAmUm +hsalifD614SgcK9PGpc/BkTVyetyEH3kMSj7HGHmKAdEc5IiaacDiGydY8hS2pgn +5whMcD60yRLBxWeDXTPzAxHsatBT4tG6NmCUgLthY2xbF37fQJQeqw3CIShwiP/W +JmxsYAQlTlV+fe+/lEjetx3dcI0FX4ilm/LC7urRQEFtYjgdVgbFA0dRIBn8exAL +DmKudlW/X3e+PkkBUz2YJQN2JFodtNuJ6nnltrM7P7pMKEF/BqxqjsHQ9gUdfeZC +huOl1UcCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +HQYDVR0OBBYEFMR5yo6hTgMdHNxr2zFblD4/MH8tMA0GCSqGSIb3DQEBCwUAA4IB +AQAtxRPPVoB7eni9n64smefv2t+UXglpp+duaIy9cr5HqQ6XErhK8WTTOd8lNNTB +zU6B8A8ExCSzNJbGpqow32hhc9f5joWJ7w5elShKKiePEI4ufIbEAp7aDHdlDkQN +kv39sxY2+hENHYwOB4lqKVb3cvTdFZx3NWZXqxNT2I7BQMXXExZacse3aQHEerGD +AWh9jUGhlBjBJVz88P6DAod8DQ3PLghcSkANPuyBYeYk28rgDi0Hsj5W3I31QYUH +SJsMC8tJP33st/3LjWeJGqvtux6jAAgIFyqCXDFdRootD4abdNlF+9RAsXqqaC2G +spki4cErx5z481+oghLrGREt +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA - G2" +# Serial: 71758320672825410020661621085256472406 +# MD5 Fingerprint: 74:9d:ea:60:24:c4:fd:22:53:3e:cc:3a:72:d9:29:4f +# SHA1 Fingerprint: aa:db:bc:22:23:8f:c4:01:a1:27:bb:38:dd:f4:1d:db:08:9e:f0:12 +# SHA256 Fingerprint: a4:31:0d:50:af:18:a6:44:71:90:37:2a:86:af:af:8b:95:1f:fb:43:1d:83:7f:1e:56:88:b4:59:71:ed:15:57 +-----BEGIN CERTIFICATE----- +MIICiDCCAg2gAwIBAgIQNfwmXNmET8k9Jj1Xm67XVjAKBggqhkjOPQQDAzCBhDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjE4MDYGA1UECxMvKGMp +IDIwMDcgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAi +BgNVBAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMjAeFw0wNzExMDUwMDAw +MDBaFw0zODAxMTgyMzU5NTlaMIGEMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhh +d3RlLCBJbmMuMTgwNgYDVQQLEy8oYykgMjAwNyB0aGF3dGUsIEluYy4gLSBGb3Ig +YXV0aG9yaXplZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9v +dCBDQSAtIEcyMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEotWcgnuVnfFSeIf+iha/ +BebfowJPDQfGAFG6DAJSLSKkQjnE/o/qycG+1E3/n3qe4rF8mq2nhglzh9HnmuN6 +papu+7qzcMBniKI11KOasf2twu8x+qi58/sIxpHR+ymVo0IwQDAPBgNVHRMBAf8E +BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUmtgAMADna3+FGO6Lts6K +DPgR4bswCgYIKoZIzj0EAwMDaQAwZgIxAN344FdHW6fmCsO99YCKlzUNG4k8VIZ3 +KMqh9HneteY4sPBlcIx/AlTCv//YoT7ZzwIxAMSNlPzcU9LcnXgWHxUzI1NS41ox +XZ3Krr0TKUQNJ1uo52icEvdYPy5yAlejj6EULg== +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA - G3" +# Serial: 127614157056681299805556476275995414779 +# MD5 Fingerprint: fb:1b:5d:43:8a:94:cd:44:c6:76:f2:43:4b:47:e7:31 +# SHA1 Fingerprint: f1:8b:53:8d:1b:e9:03:b6:a6:f0:56:43:5b:17:15:89:ca:f3:6b:f2 +# SHA256 Fingerprint: 4b:03:f4:58:07:ad:70:f2:1b:fc:2c:ae:71:c9:fd:e4:60:4c:06:4c:f5:ff:b6:86:ba:e5:db:aa:d7:fd:d3:4c +-----BEGIN CERTIFICATE----- +MIIEKjCCAxKgAwIBAgIQYAGXt0an6rS0mtZLL/eQ+zANBgkqhkiG9w0BAQsFADCB +rjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDggdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAiBgNV +BAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMzAeFw0wODA0MDIwMDAwMDBa +Fw0zNzEyMDEyMzU5NTlaMIGuMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhhd3Rl +LCBJbmMuMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9uIFNlcnZpY2VzIERpdmlzaW9u +MTgwNgYDVQQLEy8oYykgMjAwOCB0aGF3dGUsIEluYy4gLSBGb3IgYXV0aG9yaXpl +ZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9vdCBDQSAtIEcz +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsr8nLPvb2FvdeHsbnndm +gcs+vHyu86YnmjSjaDFxODNi5PNxZnmxqWWjpYvVj2AtP0LMqmsywCPLLEHd5N/8 +YZzic7IilRFDGF/Eth9XbAoFWCLINkw6fKXRz4aviKdEAhN0cXMKQlkC+BsUa0Lf +b1+6a4KinVvnSr0eAXLbS3ToO39/fR8EtCab4LRarEc9VbjXsCZSKAExQGbY2SS9 +9irY7CFJXJv2eul/VTV+lmuNk5Mny5K76qxAwJ/C+IDPXfRa3M50hqY+bAtTyr2S +zhkGcuYMXDhpxwTWvGzOW/b3aJzcJRVIiKHpqfiYnODz1TEoYRFsZ5aNOZnLwkUk +OQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV +HQ4EFgQUrWyqlGCc7eT/+j4KdCtjA/e2Wb8wDQYJKoZIhvcNAQELBQADggEBABpA +2JVlrAmSicY59BDlqQ5mU1143vokkbvnRFHfxhY0Cu9qRFHqKweKA3rD6z8KLFIW +oCtDuSWQP3CpMyVtRRooOyfPqsMpQhvfO0zAMzRbQYi/aytlryjvsvXDqmbOe1bu +t8jLZ8HJnBoYuMTDSQPxYA5QzUbF83d597YV4Djbxy8ooAw/dyZ02SUS2jHaGh7c +KUGRIjxpp7sC8rZcJwOJ9Abqm+RyguOhCcHpABnTPtRwa7pxpqpYrvS76Wy274fM +m7v/OeZWYdMKp8RcTGB7BXcmer/YB1IsYvdwY9k5vG8cwnncdimvzsUsZAReiDZu +MdRAGmI0Nj81Aa6sY6A= +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only +# Subject: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only +# Label: "GeoTrust Primary Certification Authority - G2" +# Serial: 80682863203381065782177908751794619243 +# MD5 Fingerprint: 01:5e:d8:6b:bd:6f:3d:8e:a1:31:f8:12:e0:98:73:6a +# SHA1 Fingerprint: 8d:17:84:d5:37:f3:03:7d:ec:70:fe:57:8b:51:9a:99:e6:10:d7:b0 +# SHA256 Fingerprint: 5e:db:7a:c4:3b:82:a0:6a:87:61:e8:d7:be:49:79:eb:f2:61:1f:7d:d7:9b:f9:1c:1c:6b:56:6a:21:9e:d7:66 +-----BEGIN CERTIFICATE----- +MIICrjCCAjWgAwIBAgIQPLL0SAoA4v7rJDteYD7DazAKBggqhkjOPQQDAzCBmDEL +MAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsTMChj +KSAyMDA3IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTE2 +MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eSAtIEcyMB4XDTA3MTEwNTAwMDAwMFoXDTM4MDExODIzNTk1OVowgZgxCzAJBgNV +BAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykgMjAw +NyBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0BgNV +BAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH +MjB2MBAGByqGSM49AgEGBSuBBAAiA2IABBWx6P0DFUPlrOuHNxFi79KDNlJ9RVcL +So17VDs6bl8VAsBQps8lL33KSLjHUGMcKiEIfJo22Av+0SbFWDEwKCXzXV2juLal +tJLtbCyf691DiaI8S0iRHVDsJt/WYC69IaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBVfNVdRVfslsq0DafwBo/q+EVXVMAoG +CCqGSM49BAMDA2cAMGQCMGSWWaboCd6LuvpaiIjwH5HTRqjySkwCY/tsXzjbLkGT +qQ7mndwxHLKgpxgceeHHNgIwOlavmnRs9vuD4DPTCF+hnMJbn0bWtsuRBmOiBucz +rD6ogRLQy7rQkgu2npaqBA+K +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Universal Root Certification Authority" +# Serial: 85209574734084581917763752644031726877 +# MD5 Fingerprint: 8e:ad:b5:01:aa:4d:81:e4:8c:1d:d1:e1:14:00:95:19 +# SHA1 Fingerprint: 36:79:ca:35:66:87:72:30:4d:30:a5:fb:87:3b:0f:a7:7b:b7:0d:54 +# SHA256 Fingerprint: 23:99:56:11:27:a5:71:25:de:8c:ef:ea:61:0d:df:2f:a0:78:b5:c8:06:7f:4e:82:82:90:bf:b8:60:e8:4b:3c +-----BEGIN CERTIFICATE----- +MIIEuTCCA6GgAwIBAgIQQBrEZCGzEyEDDrvkEhrFHTANBgkqhkiG9w0BAQsFADCB +vTELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwOCBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MTgwNgYDVQQDEy9W +ZXJpU2lnbiBVbml2ZXJzYWwgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe +Fw0wODA0MDIwMDAwMDBaFw0zNzEyMDEyMzU5NTlaMIG9MQswCQYDVQQGEwJVUzEX +MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlTaWduIFRydXN0 +IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAyMDA4IFZlcmlTaWduLCBJbmMuIC0gRm9y +IGF1dGhvcml6ZWQgdXNlIG9ubHkxODA2BgNVBAMTL1ZlcmlTaWduIFVuaXZlcnNh +bCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAx2E3XrEBNNti1xWb/1hajCMj1mCOkdeQmIN65lgZOIzF +9uVkhbSicfvtvbnazU0AtMgtc6XHaXGVHzk8skQHnOgO+k1KxCHfKWGPMiJhgsWH +H26MfF8WIFFE0XBPV+rjHOPMee5Y2A7Cs0WTwCznmhcrewA3ekEzeOEz4vMQGn+H +LL729fdC4uW/h2KJXwBL38Xd5HVEMkE6HnFuacsLdUYI0crSK5XQz/u5QGtkjFdN +/BMReYTtXlT2NJ8IAfMQJQYXStrxHXpma5hgZqTZ79IugvHw7wnqRMkVauIDbjPT +rJ9VAMf2CGqUuV/c4DPxhGD5WycRtPwW8rtWaoAljQIDAQABo4GyMIGvMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMG0GCCsGAQUFBwEMBGEwX6FdoFsw +WTBXMFUWCWltYWdlL2dpZjAhMB8wBwYFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgs +exkuMCUWI2h0dHA6Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMB0GA1Ud +DgQWBBS2d/ppSEefUxLVwuoHMnYH0ZcHGTANBgkqhkiG9w0BAQsFAAOCAQEASvj4 +sAPmLGd75JR3Y8xuTPl9Dg3cyLk1uXBPY/ok+myDjEedO2Pzmvl2MpWRsXe8rJq+ +seQxIcaBlVZaDrHC1LGmWazxY8u4TB1ZkErvkBYoH1quEPuBUDgMbMzxPcP1Y+Oz +4yHJJDnp/RVmRvQbEdBNc6N9Rvk97ahfYtTxP/jgdFcrGJ2BtMQo2pSXpXDrrB2+ +BxHw1dvd5Yzw1TKwg+ZX4o+/vqGqvz0dtdQ46tewXDpPaj+PwGZsY6rp2aQW9IHR +lRQOfc2VNNnSj3BzgXucfr2YYdhFh5iQxeuGMMY1v/D/w1WIg0vvBZIGcfK4mJO3 +7M2CYfE45k+XmCpajQ== +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Class 3 Public Primary Certification Authority - G4" +# Serial: 63143484348153506665311985501458640051 +# MD5 Fingerprint: 3a:52:e1:e7:fd:6f:3a:e3:6f:f3:6f:99:1b:f9:22:41 +# SHA1 Fingerprint: 22:d5:d8:df:8f:02:31:d1:8d:f7:9d:b7:cf:8a:2d:64:c9:3f:6c:3a +# SHA256 Fingerprint: 69:dd:d7:ea:90:bb:57:c9:3e:13:5d:c8:5e:a6:fc:d5:48:0b:60:32:39:bd:c4:54:fc:75:8b:2a:26:cf:7f:79 +-----BEGIN CERTIFICATE----- +MIIDhDCCAwqgAwIBAgIQL4D+I4wOIg9IZxIokYesszAKBggqhkjOPQQDAzCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzQwHhcNMDcxMTA1MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCByjELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJp +U2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2lnbiwg +SW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2ln +biBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5 +IC0gRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASnVnp8Utpkmw4tXNherJI9/gHm +GUo9FANL+mAnINmDiWn6VMaaGF5VKmTeBvaNSjutEDxlPZCIBIngMGGzrl0Bp3ve +fLK+ymVhAIau2o970ImtTR1ZmkGxvEeA3J5iw/mjgbIwga8wDwYDVR0TAQH/BAUw +AwEB/zAOBgNVHQ8BAf8EBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJ +aW1hZ2UvZ2lmMCEwHzAHBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYj +aHR0cDovL2xvZ28udmVyaXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFLMW +kf3upm7ktS5Jj4d4gYDs5bG1MAoGCCqGSM49BAMDA2gAMGUCMGYhDBgmYFo4e1ZC +4Kf8NoRRkSAsdk1DPcQdhCPQrNZ8NQbOzWm9kA3bbEhCHQ6qQgIxAJw9SDkjOVga +FRJZap7v1VmyHVIsmXHNxynfGyphe3HR3vPA5Q06Sqotp9iGKt0uEA== +-----END CERTIFICATE----- + +# Issuer: CN=NetLock Arany (Class Gold) Főtanúsítvány O=NetLock Kft. OU=Tanúsítványkiadók (Certification Services) +# Subject: CN=NetLock Arany (Class Gold) Főtanúsítvány O=NetLock Kft. OU=Tanúsítványkiadók (Certification Services) +# Label: "NetLock Arany (Class Gold) Főtanúsítvány" +# Serial: 80544274841616 +# MD5 Fingerprint: c5:a1:b7:ff:73:dd:d6:d7:34:32:18:df:fc:3c:ad:88 +# SHA1 Fingerprint: 06:08:3f:59:3f:15:a1:04:a0:69:a4:6b:a9:03:d0:06:b7:97:09:91 +# SHA256 Fingerprint: 6c:61:da:c3:a2:de:f0:31:50:6b:e0:36:d2:a6:fe:40:19:94:fb:d1:3d:f9:c8:d4:66:59:92:74:c4:46:ec:98 +-----BEGIN CERTIFICATE----- +MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQG +EwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3 +MDUGA1UECwwuVGFuw7pzw610dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNl +cnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBBcmFueSAoQ2xhc3MgR29sZCkgRsWR +dGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgxMjA2MTUwODIxWjCB +pzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxOZXRM +b2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlm +aWNhdGlvbiBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNz +IEdvbGQpIEbFkXRhbsO6c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAxCRec75LbRTDofTjl5Bu0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrT +lF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw/HpYzY6b7cNGbIRwXdrz +AZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAkH3B5r9s5 +VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRG +ILdwfzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2 +BJtr+UBdADTHLpl1neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAG +AQH/AgEEMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2M +U9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwWqZw8UQCgwBEIBaeZ5m8BiFRh +bvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTtaYtOUZcTh5m2C ++C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC +bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2F +uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2 +XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E= +-----END CERTIFICATE----- + +# Issuer: CN=Staat der Nederlanden Root CA - G2 O=Staat der Nederlanden +# Subject: CN=Staat der Nederlanden Root CA - G2 O=Staat der Nederlanden +# Label: "Staat der Nederlanden Root CA - G2" +# Serial: 10000012 +# MD5 Fingerprint: 7c:a5:0f:f8:5b:9a:7d:6d:30:ae:54:5a:e3:42:a2:8a +# SHA1 Fingerprint: 59:af:82:79:91:86:c7:b4:75:07:cb:cf:03:57:46:eb:04:dd:b7:16 +# SHA256 Fingerprint: 66:8c:83:94:7d:a6:3b:72:4b:ec:e1:74:3c:31:a0:e6:ae:d0:db:8e:c5:b3:1b:e3:77:bb:78:4f:91:b6:71:6f +-----BEGIN CERTIFICATE----- +MIIFyjCCA7KgAwIBAgIEAJiWjDANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJO +TDEeMBwGA1UECgwVU3RhYXQgZGVyIE5lZGVybGFuZGVuMSswKQYDVQQDDCJTdGFh +dCBkZXIgTmVkZXJsYW5kZW4gUm9vdCBDQSAtIEcyMB4XDTA4MDMyNjExMTgxN1oX +DTIwMDMyNTExMDMxMFowWjELMAkGA1UEBhMCTkwxHjAcBgNVBAoMFVN0YWF0IGRl +ciBOZWRlcmxhbmRlbjErMCkGA1UEAwwiU3RhYXQgZGVyIE5lZGVybGFuZGVuIFJv +b3QgQ0EgLSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMVZ5291 +qj5LnLW4rJ4L5PnZyqtdj7U5EILXr1HgO+EASGrP2uEGQxGZqhQlEq0i6ABtQ8Sp +uOUfiUtnvWFI7/3S4GCI5bkYYCjDdyutsDeqN95kWSpGV+RLufg3fNU254DBtvPU +Z5uW6M7XxgpT0GtJlvOjCwV3SPcl5XCsMBQgJeN/dVrlSPhOewMHBPqCYYdu8DvE +pMfQ9XQ+pV0aCPKbJdL2rAQmPlU6Yiile7Iwr/g3wtG61jj99O9JMDeZJiFIhQGp +5Rbn3JBV3w/oOM2ZNyFPXfUib2rFEhZgF1XyZWampzCROME4HYYEhLoaJXhena/M +UGDWE4dS7WMfbWV9whUYdMrhfmQpjHLYFhN9C0lK8SgbIHRrxT3dsKpICT0ugpTN +GmXZK4iambwYfp/ufWZ8Pr2UuIHOzZgweMFvZ9C+X+Bo7d7iscksWXiSqt8rYGPy +5V6548r6f1CGPqI0GAwJaCgRHOThuVw+R7oyPxjMW4T182t0xHJ04eOLoEq9jWYv +6q012iDTiIJh8BIitrzQ1aTsr1SIJSQ8p22xcik/Plemf1WvbibG/ufMQFxRRIEK +eN5KzlW/HdXZt1bv8Hb/C3m1r737qWmRRpdogBQ2HbN/uymYNqUg+oJgYjOk7Na6 +B6duxc8UpufWkjTYgfX8HV2qXB72o007uPc5AgMBAAGjgZcwgZQwDwYDVR0TAQH/ +BAUwAwEB/zBSBgNVHSAESzBJMEcGBFUdIAAwPzA9BggrBgEFBQcCARYxaHR0cDov +L3d3dy5wa2lvdmVyaGVpZC5ubC9wb2xpY2llcy9yb290LXBvbGljeS1HMjAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJFoMocVHYnitfGsNig0jQt8YojrMA0GCSqG +SIb3DQEBCwUAA4ICAQCoQUpnKpKBglBu4dfYszk78wIVCVBR7y29JHuIhjv5tLyS +CZa59sCrI2AGeYwRTlHSeYAz+51IvuxBQ4EffkdAHOV6CMqqi3WtFMTC6GY8ggen +5ieCWxjmD27ZUD6KQhgpxrRW/FYQoAUXvQwjf/ST7ZwaUb7dRUG/kSS0H4zpX897 +IZmflZ85OkYcbPnNe5yQzSipx6lVu6xiNGI1E0sUOlWDuYaNkqbG9AclVMwWVxJK +gnjIFNkXgiYtXSAfea7+1HAWFpWD2DU5/1JddRwWxRNVz0fMdWVSSt7wsKfkCpYL ++63C4iWEst3kvX5ZbJvw8NjnyvLplzh+ib7M+zkXYT9y2zqR2GUBGR2tUKRXCnxL +vJxxcypFURmFzI79R6d0lR2o0a9OF7FpJsKqeFdbxU2n5Z4FF5TKsl+gSRiNNOkm +bEgeqmiSBeGCc1qb3AdbCG19ndeNIdn8FCCqwkXfP+cAslHkwvgFuXkajDTznlvk +N1trSt8sV4pAWja63XVECDdCcAz+3F4hoKOKwJCcaNpQ5kUQR3i2TtJlycM33+FC +Y7BXN0Ute4qcvwXqZVUz9zkQxSgqIXobisQk+T8VyJoVIPVVYpbtbZNQvOSqeK3Z +ywplh6ZmwcSBo3c6WB4L7oOLnR7SUqTMHW+wmG2UMbX4cQrcufx9MmDm66+KAQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 1 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 1 O=Hongkong Post +# Label: "Hongkong Post Root CA 1" +# Serial: 1000 +# MD5 Fingerprint: a8:0d:6f:39:78:b9:43:6d:77:42:6d:98:5a:cc:23:ca +# SHA1 Fingerprint: d6:da:a8:20:8d:09:d2:15:4d:24:b5:2f:cb:34:6e:b2:58:b2:8a:58 +# SHA256 Fingerprint: f9:e6:7d:33:6c:51:00:2a:c0:54:c6:32:02:2d:66:dd:a2:e7:e3:ff:f1:0a:d0:61:ed:31:d8:bb:b4:10:cf:b2 +-----BEGIN CERTIFICATE----- +MIIDMDCCAhigAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwRzELMAkGA1UEBhMCSEsx +FjAUBgNVBAoTDUhvbmdrb25nIFBvc3QxIDAeBgNVBAMTF0hvbmdrb25nIFBvc3Qg +Um9vdCBDQSAxMB4XDTAzMDUxNTA1MTMxNFoXDTIzMDUxNTA0NTIyOVowRzELMAkG +A1UEBhMCSEsxFjAUBgNVBAoTDUhvbmdrb25nIFBvc3QxIDAeBgNVBAMTF0hvbmdr +b25nIFBvc3QgUm9vdCBDQSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEArP84tulmAknjorThkPlAj3n54r15/gK97iSSHSL22oVyaf7XPwnU3ZG1ApzQ +jVrhVcNQhrkpJsLj2aDxaQMoIIBFIi1WpztUlVYiWR8o3x8gPW2iNr4joLFutbEn +PzlTCeqrauh0ssJlXI6/fMN4hM2eFvz1Lk8gKgifd/PFHsSaUmYeSF7jEAaPIpjh +ZY4bXSNmO7ilMlHIhqqhqZ5/dpTCpmy3QfDVyAY45tQM4vM7TG1QjMSDJ8EThFk9 +nnV0ttgCXjqQesBCNnLsak3c78QA3xMYV18meMjWCnl3v/evt3a5pQuEF10Q6m/h +q5URX208o1xNg1vysxmKgIsLhwIDAQABoyYwJDASBgNVHRMBAf8ECDAGAQH/AgED +MA4GA1UdDwEB/wQEAwIBxjANBgkqhkiG9w0BAQUFAAOCAQEADkbVPK7ih9legYsC +mEEIjEy82tvuJxuC52pF7BaLT4Wg87JwvVqWuspube5Gi27nKi6Wsxkz67SfqLI3 +7piol7Yutmcn1KZJ/RyTZXaeQi/cImyaT/JaFTmxcdcrUehtHJjA2Sr0oYJ71clB +oiMBdDhViw+5LmeiIAQ32pwL0xch4I+XeTRvhEgCIDMb5jREn5Fw9IBehEPCKdJs +EhTkYY2sEJCehFC78JZvRZ+K88psT/oROhUVRsPNH4NbLUES7VBnQRM9IauUiqpO +fMGx+6fWtScvl6tu4B3i0RwsH0Ti/L6RoZz71ilTc4afU9hDDl3WY4JxHYB0yvbi +AmvZWg== +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign RootCA11 O=Japan Certification Services, Inc. +# Subject: CN=SecureSign RootCA11 O=Japan Certification Services, Inc. +# Label: "SecureSign RootCA11" +# Serial: 1 +# MD5 Fingerprint: b7:52:74:e2:92:b4:80:93:f2:75:e4:cc:d7:f2:ea:26 +# SHA1 Fingerprint: 3b:c4:9f:48:f8:f3:73:a0:9c:1e:bd:f8:5b:b1:c3:65:c7:d8:11:b3 +# SHA256 Fingerprint: bf:0f:ee:fb:9e:3a:58:1a:d5:f9:e9:db:75:89:98:57:43:d2:61:08:5c:4d:31:4f:6f:5d:72:59:aa:42:16:12 +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIBATANBgkqhkiG9w0BAQUFADBYMQswCQYDVQQGEwJKUDEr +MCkGA1UEChMiSmFwYW4gQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcywgSW5jLjEcMBoG +A1UEAxMTU2VjdXJlU2lnbiBSb290Q0ExMTAeFw0wOTA0MDgwNDU2NDdaFw0yOTA0 +MDgwNDU2NDdaMFgxCzAJBgNVBAYTAkpQMSswKQYDVQQKEyJKYXBhbiBDZXJ0aWZp +Y2F0aW9uIFNlcnZpY2VzLCBJbmMuMRwwGgYDVQQDExNTZWN1cmVTaWduIFJvb3RD +QTExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA/XeqpRyQBTvLTJsz +i1oURaTnkBbR31fSIRCkF/3frNYfp+TbfPfs37gD2pRY/V1yfIw/XwFndBWW4wI8 +h9uuywGOwvNmxoVF9ALGOrVisq/6nL+k5tSAMJjzDbaTj6nU2DbysPyKyiyhFTOV +MdrAG/LuYpmGYz+/3ZMqg6h2uRMft85OQoWPIucuGvKVCbIFtUROd6EgvanyTgp9 +UK31BQ1FT0Zx/Sg+U/sE2C3XZR1KG/rPO7AxmjVuyIsG0wCR8pQIZUyxNAYAeoni +8McDWc/V1uinMrPmmECGxc0nEovMe863ETxiYAcjPitAbpSACW22s293bzUIUPsC +h8U+iQIDAQABo0IwQDAdBgNVHQ4EFgQUW/hNT7KlhtQ60vFjmqC+CfZXt94wDgYD +VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AKChOBZmLqdWHyGcBvod7bkixTgm2E5P7KN/ed5GIaGHd48HCJqypMWvDzKYC3xm +KbabfSVSSUOrTC4rbnpwrxYO4wJs+0LmGJ1F2FXI6Dvd5+H0LgscNFxsWEr7jIhQ +X5Ucv+2rIrVls4W6ng+4reV6G4pQOh29Dbx7VFALuUKvVaAYga1lme++5Jy/xIWr +QbJUb9wlze144o4MjQlJ3WN7WmmWAiGovVJZ6X01y8hSyn+B/tlr0/cR7SXf+Of5 +pPpyl4RTDaXQMhhRdlkUbA/r7F+AjHVDg8OFmP9Mni0N5HeDk061lgeLKBObjBmN +QSdJQO7e5iNEOdyhIta6A/I= +-----END CERTIFICATE----- + +# Issuer: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. +# Subject: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. +# Label: "Microsec e-Szigno Root CA 2009" +# Serial: 14014712776195784473 +# MD5 Fingerprint: f8:49:f4:03:bc:44:2d:83:be:48:69:7d:29:64:fc:b1 +# SHA1 Fingerprint: 89:df:74:fe:5c:f4:0f:4a:80:f9:e3:37:7d:54:da:91:e1:01:31:8e +# SHA256 Fingerprint: 3c:5f:81:fe:a5:fa:b8:2c:64:bf:a2:ea:ec:af:cd:e8:e0:77:fc:86:20:a7:ca:e5:37:16:3d:f3:6e:db:f3:78 +-----BEGIN CERTIFICATE----- +MIIECjCCAvKgAwIBAgIJAMJ+QwRORz8ZMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD +VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 +ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3ppZ25vIFJvb3QgQ0EgMjAwOTEfMB0G +CSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5odTAeFw0wOTA2MTYxMTMwMThaFw0y +OTEyMzAxMTMwMThaMIGCMQswCQYDVQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3Qx +FjAUBgNVBAoMDU1pY3Jvc2VjIEx0ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3pp +Z25vIFJvb3QgQ0EgMjAwOTEfMB0GCSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5o +dTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOn4j/NjrdqG2KfgQvvP +kd6mJviZpWNwrZuuyjNAfW2WbqEORO7hE52UQlKavXWFdCyoDh2Tthi3jCyoz/tc +cbna7P7ofo/kLx2yqHWH2Leh5TvPmUpG0IMZfcChEhyVbUr02MelTTMuhTlAdX4U +fIASmFDHQWe4oIBhVKZsTh/gnQ4H6cm6M+f+wFUoLAKApxn1ntxVUwOXewdI/5n7 +N4okxFnMUBBjjqqpGrCEGob5X7uxUG6k0QrM1XF+H6cbfPVTbiJfyyvm1HxdrtbC +xkzlBQHZ7Vf8wSN5/PrIJIOV87VqUQHQd9bpEqH5GoP7ghu5sJf0dgYzQ0mg/wu1 ++rUCAwEAAaOBgDB+MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G +A1UdDgQWBBTLD8bfQkPMPcu1SCOhGnqmKrs0aDAfBgNVHSMEGDAWgBTLD8bfQkPM +Pcu1SCOhGnqmKrs0aDAbBgNVHREEFDASgRBpbmZvQGUtc3ppZ25vLmh1MA0GCSqG +SIb3DQEBCwUAA4IBAQDJ0Q5eLtXMs3w+y/w9/w0olZMEyL/azXm4Q5DwpL7v8u8h +mLzU1F0G9u5C7DBsoKqpyvGvivo/C3NqPuouQH4frlRheesuCDfXI/OMn74dseGk +ddug4lQUsbocKaQY9hK6ohQU4zE1yED/t+AFdlfBHFny+L/k7SViXITwfn4fs775 +tyERzAMBVnCnEJIeGzSBHq2cGsMEPO0CYdYeBvNfOofyK/FFh+U9rNHHV4S9a67c +2Pm2G2JwCz02yULyMtd6YebS2z3PyKnJm9zbWETXbzivf3jTo60adbocwTZ8jx5t +HMN1Rq41Bab2XD0h7lbwyYIiLXpUq3DDfSJlgnCW +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Label: "GlobalSign Root CA - R3" +# Serial: 4835703278459759426209954 +# MD5 Fingerprint: c5:df:b8:49:ca:05:13:55:ee:2d:ba:1a:c3:3e:b0:28 +# SHA1 Fingerprint: d6:9b:56:11:48:f0:1c:77:c5:45:78:c1:09:26:df:5b:85:69:76:ad +# SHA256 Fingerprint: cb:b5:22:d7:b7:f1:27:ad:6a:01:13:86:5b:df:1c:d4:10:2e:7d:07:59:af:63:5a:7c:f4:72:0d:c9:63:c5:3b +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 +MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 +RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT +gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm +KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd +QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ +XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o +LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU +RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp +jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK +6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX +mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs +Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH +WD9f +-----END CERTIFICATE----- + +# Issuer: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 +# Subject: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 +# Label: "Autoridad de Certificacion Firmaprofesional CIF A62634068" +# Serial: 6047274297262753887 +# MD5 Fingerprint: 73:3a:74:7a:ec:bb:a3:96:a6:c2:e4:e2:c8:9b:c0:c3 +# SHA1 Fingerprint: ae:c5:fb:3f:c8:e1:bf:c4:e5:4f:03:07:5a:9a:e8:00:b7:f7:b6:fa +# SHA256 Fingerprint: 04:04:80:28:bf:1f:28:64:d4:8f:9a:d4:d8:32:94:36:6a:82:88:56:55:3f:3b:14:30:3f:90:14:7f:5d:40:ef +-----BEGIN CERTIFICATE----- +MIIGFDCCA/ygAwIBAgIIU+w77vuySF8wDQYJKoZIhvcNAQEFBQAwUTELMAkGA1UE +BhMCRVMxQjBABgNVBAMMOUF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uIEZpcm1h +cHJvZmVzaW9uYWwgQ0lGIEE2MjYzNDA2ODAeFw0wOTA1MjAwODM4MTVaFw0zMDEy +MzEwODM4MTVaMFExCzAJBgNVBAYTAkVTMUIwQAYDVQQDDDlBdXRvcmlkYWQgZGUg +Q2VydGlmaWNhY2lvbiBGaXJtYXByb2Zlc2lvbmFsIENJRiBBNjI2MzQwNjgwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKlmuO6vj78aI14H9M2uDDUtd9 +thDIAl6zQyrET2qyyhxdKJp4ERppWVevtSBC5IsP5t9bpgOSL/UR5GLXMnE42QQM +cas9UX4PB99jBVzpv5RvwSmCwLTaUbDBPLutN0pcyvFLNg4kq7/DhHf9qFD0sefG +L9ItWY16Ck6WaVICqjaY7Pz6FIMMNx/Jkjd/14Et5cS54D40/mf0PmbR0/RAz15i +NA9wBj4gGFrO93IbJWyTdBSTo3OxDqqHECNZXyAFGUftaI6SEspd/NYrspI8IM/h +X68gvqB2f3bl7BqGYTM+53u0P6APjqK5am+5hyZvQWyIplD9amML9ZMWGxmPsu2b +m8mQ9QEM3xk9Dz44I8kvjwzRAv4bVdZO0I08r0+k8/6vKtMFnXkIoctXMbScyJCy +Z/QYFpM6/EfY0XiWMR+6KwxfXZmtY4laJCB22N/9q06mIqqdXuYnin1oKaPnirja +EbsXLZmdEyRG98Xi2J+Of8ePdG1asuhy9azuJBCtLxTa/y2aRnFHvkLfuwHb9H/T +KI8xWVvTyQKmtFLKbpf7Q8UIJm+K9Lv9nyiqDdVF8xM6HdjAeI9BZzwelGSuewvF +6NkBiDkal4ZkQdU7hwxu+g/GvUgUvzlN1J5Bto+WHWOWk9mVBngxaJ43BjuAiUVh +OSPHG0SjFeUc+JIwuwIDAQABo4HvMIHsMBIGA1UdEwEB/wQIMAYBAf8CAQEwDgYD +VR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRlzeurNR4APn7VdMActHNHDhpkLzCBpgYD +VR0gBIGeMIGbMIGYBgRVHSAAMIGPMC8GCCsGAQUFBwIBFiNodHRwOi8vd3d3LmZp +cm1hcHJvZmVzaW9uYWwuY29tL2NwczBcBggrBgEFBQcCAjBQHk4AUABhAHMAZQBv +ACAAZABlACAAbABhACAAQgBvAG4AYQBuAG8AdgBhACAANAA3ACAAQgBhAHIAYwBl +AGwAbwBuAGEAIAAwADgAMAAxADcwDQYJKoZIhvcNAQEFBQADggIBABd9oPm03cXF +661LJLWhAqvdpYhKsg9VSytXjDvlMd3+xDLx51tkljYyGOylMnfX40S2wBEqgLk9 +am58m9Ot/MPWo+ZkKXzR4Tgegiv/J2Wv+xYVxC5xhOW1//qkR71kMrv2JYSiJ0L1 +ILDCExARzRAVukKQKtJE4ZYm6zFIEv0q2skGz3QeqUvVhyj5eTSSPi5E6PaPT481 +PyWzOdxjKpBrIF/EUhJOlywqrJ2X3kjyo2bbwtKDlaZmp54lD+kLM5FlClrD2VQS +3a/DTg4fJl4N3LON7NWBcN7STyQF82xO9UxJZo3R/9ILJUFI/lGExkKvgATP0H5k +SeTy36LssUzAKh3ntLFlosS88Zj0qnAHY7S42jtM+kAiMFsRpvAFDsYCA0irhpuF +3dvd6qJ2gHN99ZwExEWN57kci57q13XRcrHedUTnQn3iV2t93Jm8PYMo6oCTjcVM +ZcFwgbg4/EMxsvYDNEeyrPsiBsse3RdHHF9mudMaotoRsaS8I8nkvof/uZS2+F0g +StRf571oe2XyFR7SOqkt6dhrJKyXWERHrVkY8SFlcN7ONGCoQPHzPKTDKCOM/icz +Q0CgFzzr6juwcqajuUpLXhZI9LK8yIySxZ2frHI2vDSANGupi5LAuBft7HZT9SQB +jLMi6Et8Vcad+qMUu2WFbm5PEn4KPJ2V +-----END CERTIFICATE----- + +# Issuer: CN=Izenpe.com O=IZENPE S.A. +# Subject: CN=Izenpe.com O=IZENPE S.A. +# Label: "Izenpe.com" +# Serial: 917563065490389241595536686991402621 +# MD5 Fingerprint: a6:b0:cd:85:80:da:5c:50:34:a3:39:90:2f:55:67:73 +# SHA1 Fingerprint: 2f:78:3d:25:52:18:a7:4a:65:39:71:b5:2c:a2:9c:45:15:6f:e9:19 +# SHA256 Fingerprint: 25:30:cc:8e:98:32:15:02:ba:d9:6f:9b:1f:ba:1b:09:9e:2d:29:9e:0f:45:48:bb:91:4f:36:3b:c0:d4:53:1f +-----BEGIN CERTIFICATE----- +MIIF8TCCA9mgAwIBAgIQALC3WhZIX7/hy/WL1xnmfTANBgkqhkiG9w0BAQsFADA4 +MQswCQYDVQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6 +ZW5wZS5jb20wHhcNMDcxMjEzMTMwODI4WhcNMzcxMjEzMDgyNzI1WjA4MQswCQYD +VQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6ZW5wZS5j +b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ03rKDx6sp4boFmVq +scIbRTJxldn+EFvMr+eleQGPicPK8lVx93e+d5TzcqQsRNiekpsUOqHnJJAKClaO +xdgmlOHZSOEtPtoKct2jmRXagaKH9HtuJneJWK3W6wyyQXpzbm3benhB6QiIEn6H +LmYRY2xU+zydcsC8Lv/Ct90NduM61/e0aL6i9eOBbsFGb12N4E3GVFWJGjMxCrFX +uaOKmMPsOzTFlUFpfnXCPCDFYbpRR6AgkJOhkEvzTnyFRVSa0QUmQbC1TR0zvsQD +yCV8wXDbO/QJLVQnSKwv4cSsPsjLkkxTOTcj7NMB+eAJRE1NZMDhDVqHIrytG6P+ +JrUV86f8hBnp7KGItERphIPzidF0BqnMC9bC3ieFUCbKF7jJeodWLBoBHmy+E60Q +rLUk9TiRodZL2vG70t5HtfG8gfZZa88ZU+mNFctKy6lvROUbQc/hhqfK0GqfvEyN +BjNaooXlkDWgYlwWTvDjovoDGrQscbNYLN57C9saD+veIR8GdwYDsMnvmfzAuU8L +hij+0rnq49qlw0dpEuDb8PYZi+17cNcC1u2HGCgsBCRMd+RIihrGO5rUD8r6ddIB +QFqNeb+Lz0vPqhbBleStTIo+F5HUsWLlguWABKQDfo2/2n+iD5dPDNMN+9fR5XJ+ +HMh3/1uaD7euBUbl8agW7EekFwIDAQABo4H2MIHzMIGwBgNVHREEgagwgaWBD2lu +Zm9AaXplbnBlLmNvbaSBkTCBjjFHMEUGA1UECgw+SVpFTlBFIFMuQS4gLSBDSUYg +QTAxMzM3MjYwLVJNZXJjLlZpdG9yaWEtR2FzdGVpeiBUMTA1NSBGNjIgUzgxQzBB +BgNVBAkMOkF2ZGEgZGVsIE1lZGl0ZXJyYW5lbyBFdG9yYmlkZWEgMTQgLSAwMTAx +MCBWaXRvcmlhLUdhc3RlaXowDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFB0cZQ6o8iV7tJHP5LGx5r1VdGwFMA0GCSqGSIb3DQEBCwUA +A4ICAQB4pgwWSp9MiDrAyw6lFn2fuUhfGI8NYjb2zRlrrKvV9pF9rnHzP7MOeIWb +laQnIUdCSnxIOvVFfLMMjlF4rJUT3sb9fbgakEyrkgPH7UIBzg/YsfqikuFgba56 +awmqxinuaElnMIAkejEWOVt+8Rwu3WwJrfIxwYJOubv5vr8qhT/AQKM6WfxZSzwo +JNu0FXWuDYi6LnPAvViH5ULy617uHjAimcs30cQhbIHsvm0m5hzkQiCeR7Csg1lw +LDXWrzY0tM07+DKo7+N4ifuNRSzanLh+QBxh5z6ikixL8s36mLYp//Pye6kfLqCT +VyvehQP5aTfLnnhqBbTFMXiJ7HqnheG5ezzevh55hM6fcA5ZwjUukCox2eRFekGk +LhObNA5me0mrZJfQRsN5nXJQY6aYWwa9SG3YOYNw6DXwBdGqvOPbyALqfP2C2sJb +UjWumDqtujWTI6cfSN01RpiyEGjkpTHCClguGYEQyVB1/OpaFs4R1+7vUIgtYf8/ +QnMFlEPVjjxOAToZpR9GTnfQXeWBIiGH/pR9hNiTrdZoQ0iy2+tzJOeRf1SktoA+ +naM8THLCV8Sg1Mw4J87VBp6iSNnpn86CcDaTmjvfliHjWbcM2pE38P1ZWrOZyGls +QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw== +-----END CERTIFICATE----- + +# Issuer: CN=Chambers of Commerce Root - 2008 O=AC Camerfirma S.A. +# Subject: CN=Chambers of Commerce Root - 2008 O=AC Camerfirma S.A. +# Label: "Chambers of Commerce Root - 2008" +# Serial: 11806822484801597146 +# MD5 Fingerprint: 5e:80:9e:84:5a:0e:65:0b:17:02:f3:55:18:2a:3e:d7 +# SHA1 Fingerprint: 78:6a:74:ac:76:ab:14:7f:9c:6a:30:50:ba:9e:a8:7e:fe:9a:ce:3c +# SHA256 Fingerprint: 06:3e:4a:fa:c4:91:df:d3:32:f3:08:9b:85:42:e9:46:17:d8:93:d7:fe:94:4e:10:a7:93:7e:e2:9d:96:93:c0 +-----BEGIN CERTIFICATE----- +MIIHTzCCBTegAwIBAgIJAKPaQn6ksa7aMA0GCSqGSIb3DQEBBQUAMIGuMQswCQYD +VQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3VycmVudCBhZGRyZXNzIGF0 +IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAGA1UEBRMJQTgyNzQzMjg3 +MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xKTAnBgNVBAMTIENoYW1iZXJz +IG9mIENvbW1lcmNlIFJvb3QgLSAyMDA4MB4XDTA4MDgwMTEyMjk1MFoXDTM4MDcz +MTEyMjk1MFowga4xCzAJBgNVBAYTAkVVMUMwQQYDVQQHEzpNYWRyaWQgKHNlZSBj +dXJyZW50IGFkZHJlc3MgYXQgd3d3LmNhbWVyZmlybWEuY29tL2FkZHJlc3MpMRIw +EAYDVQQFEwlBODI3NDMyODcxGzAZBgNVBAoTEkFDIENhbWVyZmlybWEgUy5BLjEp +MCcGA1UEAxMgQ2hhbWJlcnMgb2YgQ29tbWVyY2UgUm9vdCAtIDIwMDgwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCvAMtwNyuAWko6bHiUfaN/Gh/2NdW9 +28sNRHI+JrKQUrpjOyhYb6WzbZSm891kDFX29ufyIiKAXuFixrYp4YFs8r/lfTJq +VKAyGVn+H4vXPWCGhSRv4xGzdz4gljUha7MI2XAuZPeEklPWDrCQiorjh40G072Q +DuKZoRuGDtqaCrsLYVAGUvGef3bsyw/QHg3PmTA9HMRFEFis1tPo1+XqxQEHd9ZR +5gN/ikilTWh1uem8nk4ZcfUyS5xtYBkL+8ydddy/Js2Pk3g5eXNeJQ7KXOt3EgfL +ZEFHcpOrUMPrCXZkNNI5t3YRCQ12RcSprj1qr7V9ZS+UWBDsXHyvfuK2GNnQm05a +Sd+pZgvMPMZ4fKecHePOjlO+Bd5gD2vlGts/4+EhySnB8esHnFIbAURRPHsl18Tl +UlRdJQfKFiC4reRB7noI/plvg6aRArBsNlVq5331lubKgdaX8ZSD6e2wsWsSaR6s ++12pxZjptFtYer49okQ6Y1nUCyXeG0+95QGezdIp1Z8XGQpvvwyQ0wlf2eOKNcx5 +Wk0ZN5K3xMGtr/R5JJqyAQuxr1yW84Ay+1w9mPGgP0revq+ULtlVmhduYJ1jbLhj +ya6BXBg14JC7vjxPNyK5fuvPnnchpj04gftI2jE9K+OJ9dC1vX7gUMQSibMjmhAx +hduub+84Mxh2EQIDAQABo4IBbDCCAWgwEgYDVR0TAQH/BAgwBgEB/wIBDDAdBgNV +HQ4EFgQU+SSsD7K1+HnA+mCIG8TZTQKeFxkwgeMGA1UdIwSB2zCB2IAU+SSsD7K1 ++HnA+mCIG8TZTQKeFxmhgbSkgbEwga4xCzAJBgNVBAYTAkVVMUMwQQYDVQQHEzpN +YWRyaWQgKHNlZSBjdXJyZW50IGFkZHJlc3MgYXQgd3d3LmNhbWVyZmlybWEuY29t +L2FkZHJlc3MpMRIwEAYDVQQFEwlBODI3NDMyODcxGzAZBgNVBAoTEkFDIENhbWVy +ZmlybWEgUy5BLjEpMCcGA1UEAxMgQ2hhbWJlcnMgb2YgQ29tbWVyY2UgUm9vdCAt +IDIwMDiCCQCj2kJ+pLGu2jAOBgNVHQ8BAf8EBAMCAQYwPQYDVR0gBDYwNDAyBgRV +HSAAMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9wb2xpY3kuY2FtZXJmaXJtYS5jb20w +DQYJKoZIhvcNAQEFBQADggIBAJASryI1wqM58C7e6bXpeHxIvj99RZJe6dqxGfwW +PJ+0W2aeaufDuV2I6A+tzyMP3iU6XsxPpcG1Lawk0lgH3qLPaYRgM+gQDROpI9CF +5Y57pp49chNyM/WqfcZjHwj0/gF/JM8rLFQJ3uIrbZLGOU8W6jx+ekbURWpGqOt1 +glanq6B8aBMz9p0w8G8nOSQjKpD9kCk18pPfNKXG9/jvjA9iSnyu0/VU+I22mlaH +FoI6M6taIgj3grrqLuBHmrS1RaMFO9ncLkVAO+rcf+g769HsJtg1pDDFOqxXnrN2 +pSB7+R5KBWIBpih1YJeSDW4+TTdDDZIVnBgizVGZoCkaPF+KMjNbMMeJL0eYD6MD +xvbxrN8y8NmBGuScvfaAFPDRLLmF9dijscilIeUcE5fuDr3fKanvNFNb0+RqE4QG +tjICxFKuItLcsiFCGtpA8CnJ7AoMXOLQusxI0zcKzBIKinmwPQN/aUv0NCB9szTq +jktk9T79syNnFQ0EuPAtwQlRPLJsFfClI9eDdOTlLsn+mCdCxqvGnrDQWzilm1De +fhiYtUU79nm06PcaewaD+9CL2rvHvRirCG88gGtAPxkZumWK5r7VXNM21+9AUiRg +OGcEMeyP84LG3rlV8zsxkVrctQgVrXYlCg17LofiDKYGvCYQbTed7N14jHyAxfDZ +d0jQ +-----END CERTIFICATE----- + +# Issuer: CN=Global Chambersign Root - 2008 O=AC Camerfirma S.A. +# Subject: CN=Global Chambersign Root - 2008 O=AC Camerfirma S.A. +# Label: "Global Chambersign Root - 2008" +# Serial: 14541511773111788494 +# MD5 Fingerprint: 9e:80:ff:78:01:0c:2e:c1:36:bd:fe:96:90:6e:08:f3 +# SHA1 Fingerprint: 4a:bd:ee:ec:95:0d:35:9c:89:ae:c7:52:a1:2c:5b:29:f6:d6:aa:0c +# SHA256 Fingerprint: 13:63:35:43:93:34:a7:69:80:16:a0:d3:24:de:72:28:4e:07:9d:7b:52:20:bb:8f:bd:74:78:16:ee:be:ba:ca +-----BEGIN CERTIFICATE----- +MIIHSTCCBTGgAwIBAgIJAMnN0+nVfSPOMA0GCSqGSIb3DQEBBQUAMIGsMQswCQYD +VQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3VycmVudCBhZGRyZXNzIGF0 +IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAGA1UEBRMJQTgyNzQzMjg3 +MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xJzAlBgNVBAMTHkdsb2JhbCBD +aGFtYmVyc2lnbiBSb290IC0gMjAwODAeFw0wODA4MDExMjMxNDBaFw0zODA3MzEx +MjMxNDBaMIGsMQswCQYDVQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3Vy +cmVudCBhZGRyZXNzIGF0IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAG +A1UEBRMJQTgyNzQzMjg3MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xJzAl +BgNVBAMTHkdsb2JhbCBDaGFtYmVyc2lnbiBSb290IC0gMjAwODCCAiIwDQYJKoZI +hvcNAQEBBQADggIPADCCAgoCggIBAMDfVtPkOpt2RbQT2//BthmLN0EYlVJH6xed +KYiONWwGMi5HYvNJBL99RDaxccy9Wglz1dmFRP+RVyXfXjaOcNFccUMd2drvXNL7 +G706tcuto8xEpw2uIRU/uXpbknXYpBI4iRmKt4DS4jJvVpyR1ogQC7N0ZJJ0YPP2 +zxhPYLIj0Mc7zmFLmY/CDNBAspjcDahOo7kKrmCgrUVSY7pmvWjg+b4aqIG7HkF4 +ddPB/gBVsIdU6CeQNR1MM62X/JcumIS/LMmjv9GYERTtY/jKmIhYF5ntRQOXfjyG +HoiMvvKRhI9lNNgATH23MRdaKXoKGCQwoze1eqkBfSbW+Q6OWfH9GzO1KTsXO0G2 +Id3UwD2ln58fQ1DJu7xsepeY7s2MH/ucUa6LcL0nn3HAa6x9kGbo1106DbDVwo3V +yJ2dwW3Q0L9R5OP4wzg2rtandeavhENdk5IMagfeOx2YItaswTXbo6Al/3K1dh3e +beksZixShNBFks4c5eUzHdwHU1SjqoI7mjcv3N2gZOnm3b2u/GSFHTynyQbehP9r +6GsaPMWis0L7iwk+XwhSx2LE1AVxv8Rk5Pihg+g+EpuoHtQ2TS9x9o0o9oOpE9Jh +wZG7SMA0j0GMS0zbaRL/UJScIINZc+18ofLx/d33SdNDWKBWY8o9PeU1VlnpDsog +zCtLkykPAgMBAAGjggFqMIIBZjASBgNVHRMBAf8ECDAGAQH/AgEMMB0GA1UdDgQW +BBS5CcqcHtvTbDprru1U8VuTBjUuXjCB4QYDVR0jBIHZMIHWgBS5CcqcHtvTbDpr +ru1U8VuTBjUuXqGBsqSBrzCBrDELMAkGA1UEBhMCRVUxQzBBBgNVBAcTOk1hZHJp +ZCAoc2VlIGN1cnJlbnQgYWRkcmVzcyBhdCB3d3cuY2FtZXJmaXJtYS5jb20vYWRk +cmVzcykxEjAQBgNVBAUTCUE4Mjc0MzI4NzEbMBkGA1UEChMSQUMgQ2FtZXJmaXJt +YSBTLkEuMScwJQYDVQQDEx5HbG9iYWwgQ2hhbWJlcnNpZ24gUm9vdCAtIDIwMDiC +CQDJzdPp1X0jzjAOBgNVHQ8BAf8EBAMCAQYwPQYDVR0gBDYwNDAyBgRVHSAAMCow +KAYIKwYBBQUHAgEWHGh0dHA6Ly9wb2xpY3kuY2FtZXJmaXJtYS5jb20wDQYJKoZI +hvcNAQEFBQADggIBAICIf3DekijZBZRG/5BXqfEv3xoNa/p8DhxJJHkn2EaqbylZ +UohwEurdPfWbU1Rv4WCiqAm57OtZfMY18dwY6fFn5a+6ReAJ3spED8IXDneRRXoz +X1+WLGiLwUePmJs9wOzL9dWCkoQ10b42OFZyMVtHLaoXpGNR6woBrX/sdZ7LoR/x +fxKxueRkf2fWIyr0uDldmOghp+G9PUIadJpwr2hsUF1Jz//7Dl3mLEfXgTpZALVz +a2Mg9jFFCDkO9HB+QHBaP9BrQql0PSgvAm11cpUJjUhjxsYjV5KTXjXBjfkK9yyd +Yhz2rXzdpjEetrHHfoUm+qRqtdpjMNHvkzeyZi99Bffnt0uYlDXA2TopwZ2yUDMd +SqlapskD7+3056huirRXhOukP9DuqqqHW2Pok+JrqNS4cnhrG+055F3Lm6qH1U9O +AP7Zap88MQ8oAgF9mOinsKJknnn4SPIVqczmyETrP3iZ8ntxPjzxmKfFGBI/5rso +M0LpRQp8bfKGeS/Fghl9CYl8slR2iK7ewfPM4W7bMdaTrpmg7yVqc5iJWzouE4ge +v8CSlDQb4ye3ix5vQv/n6TebUB0tovkC7stYWDpxvGjjqsGvHCgfotwjZT+B6q6Z +09gwzxMNTxXJhLynSC34MCN32EZLeW32jO06f2ARePTpm67VVMB0gNELQp/B +-----END CERTIFICATE----- + +# Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Label: "Go Daddy Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 80:3a:bc:22:c1:e6:fb:8d:9b:3b:27:4a:32:1b:9a:01 +# SHA1 Fingerprint: 47:be:ab:c9:22:ea:e8:0e:78:78:34:62:a7:9f:45:c2:54:fd:e6:8b +# SHA256 Fingerprint: 45:14:0b:32:47:eb:9c:c8:c5:b4:f0:d7:b5:30:91:f7:32:92:08:9e:6e:5a:63:e2:74:9d:d3:ac:a9:19:8e:da +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz +NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE +AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD +E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH +/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy +DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh +GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR +tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA +AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX +WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu +9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr +gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo +2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO +LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI +4uJEvlz36hz1 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: d6:39:81:c6:52:7e:96:69:fc:fc:ca:66:ed:05:f2:96 +# SHA1 Fingerprint: b5:1c:06:7c:ee:2b:0c:3d:f8:55:ab:2d:92:f4:fe:39:d4:e7:0f:0e +# SHA256 Fingerprint: 2c:e1:cb:0b:f9:d2:f9:e1:02:99:3f:be:21:51:52:c3:b2:dd:0c:ab:de:1c:68:e5:31:9b:83:91:54:db:b7:f5 +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs +ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw +MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 +b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj +aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp +Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg +nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 +HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N +Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN +dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 +HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G +CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU +sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 +4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg +8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K +pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 +mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Services Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 17:35:74:af:7b:61:1c:eb:f4:f9:3c:e2:ee:40:f9:a2 +# SHA1 Fingerprint: 92:5a:8f:8d:2c:6d:04:e0:66:5f:59:6a:ff:22:d8:63:e8:25:6f:3f +# SHA256 Fingerprint: 56:8d:69:05:a2:c8:87:08:a4:b3:02:51:90:ed:cf:ed:b1:97:4a:60:6a:13:c6:e5:29:0f:cb:2a:e6:3e:da:b5 +-----BEGIN CERTIFICATE----- +MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs +ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 +MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD +VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy +ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy +dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p +OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 +8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K +Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe +hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk +6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q +AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI +bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB +ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z +qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd +iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn +0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN +sSi6 +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Commercial O=AffirmTrust +# Subject: CN=AffirmTrust Commercial O=AffirmTrust +# Label: "AffirmTrust Commercial" +# Serial: 8608355977964138876 +# MD5 Fingerprint: 82:92:ba:5b:ef:cd:8a:6f:a6:3d:55:f9:84:f6:d6:b7 +# SHA1 Fingerprint: f9:b5:b6:32:45:5f:9c:be:ec:57:5f:80:dc:e9:6e:2c:c7:b2:78:b7 +# SHA256 Fingerprint: 03:76:ab:1d:54:c5:f9:80:3c:e4:b2:e2:01:a0:ee:7e:ef:7b:57:b6:36:e8:a9:3c:9b:8d:48:60:c9:6f:5f:a7 +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIId3cGJyapsXwwDQYJKoZIhvcNAQELBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBDb21tZXJjaWFsMB4XDTEwMDEyOTE0MDYwNloXDTMwMTIzMTE0MDYwNlowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBDb21tZXJjaWFsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA9htPZwcroRX1BiLLHwGy43NFBkRJLLtJJRTWzsO3qyxPxkEylFf6EqdbDuKP +Hx6GGaeqtS25Xw2Kwq+FNXkyLbscYjfysVtKPcrNcV/pQr6U6Mje+SJIZMblq8Yr +ba0F8PrVC8+a5fBQpIs7R6UjW3p6+DM/uO+Zl+MgwdYoic+U+7lF7eNAFxHUdPAL +MeIrJmqbTFeurCA+ukV6BfO9m2kVrn1OIGPENXY6BwLJN/3HR+7o8XYdcxXyl6S1 +yHp52UKqK39c/s4mT6NmgTWvRLpUHhwwMmWd5jyTXlBOeuM61G7MGvv50jeuJCqr +VwMiKA1JdX+3KNp1v47j3A55MQIDAQABo0IwQDAdBgNVHQ4EFgQUnZPGU4teyq8/ +nx4P5ZmVvCT2lI8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQELBQADggEBAFis9AQOzcAN/wr91LoWXym9e2iZWEnStB03TX8nfUYG +XUPGhi4+c7ImfU+TqbbEKpqrIZcUsd6M06uJFdhrJNTxFq7YpFzUf1GO7RgBsZNj +vbz4YYCanrHOQnDiqX0GJX0nof5v7LMeJNrjS1UaADs1tDvZ110w/YETifLCBivt +Z8SOyUOyXGsViQK8YvxO8rUzqrJv0wqiUOP2O+guRMLbZjipM1ZI8W0bM40NjD9g +N53Tym1+NH4Nn3J2ixufcv1SNUFFApYvHLKac0khsUlHRUe072o0EclNmsxZt9YC +nlpOZbWUrhvfKbAW8b8Angc6F2S1BLUjIZkKlTuXfO8= +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Networking O=AffirmTrust +# Subject: CN=AffirmTrust Networking O=AffirmTrust +# Label: "AffirmTrust Networking" +# Serial: 8957382827206547757 +# MD5 Fingerprint: 42:65:ca:be:01:9a:9a:4c:a9:8c:41:49:cd:c0:d5:7f +# SHA1 Fingerprint: 29:36:21:02:8b:20:ed:02:f5:66:c5:32:d1:d6:ed:90:9f:45:00:2f +# SHA256 Fingerprint: 0a:81:ec:5a:92:97:77:f1:45:90:4a:f3:8d:5d:50:9f:66:b5:e2:c5:8f:cd:b5:31:05:8b:0e:17:f3:f0:b4:1b +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIIfE8EORzUmS0wDQYJKoZIhvcNAQEFBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBOZXR3b3JraW5nMB4XDTEwMDEyOTE0MDgyNFoXDTMwMTIzMTE0MDgyNFowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBOZXR3b3JraW5nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAtITMMxcua5Rsa2FSoOujz3mUTOWUgJnLVWREZY9nZOIG41w3SfYvm4SEHi3y +YJ0wTsyEheIszx6e/jarM3c1RNg1lho9Nuh6DtjVR6FqaYvZ/Ls6rnla1fTWcbua +kCNrmreIdIcMHl+5ni36q1Mr3Lt2PpNMCAiMHqIjHNRqrSK6mQEubWXLviRmVSRL +QESxG9fhwoXA3hA/Pe24/PHxI1Pcv2WXb9n5QHGNfb2V1M6+oF4nI979ptAmDgAp +6zxG8D1gvz9Q0twmQVGeFDdCBKNwV6gbh+0t+nvujArjqWaJGctB+d1ENmHP4ndG +yH329JKBNv3bNPFyfvMMFr20FQIDAQABo0IwQDAdBgNVHQ4EFgQUBx/S55zawm6i +QLSwelAQUHTEyL0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQEFBQADggEBAIlXshZ6qML91tmbmzTCnLQyFE2npN/svqe++EPbkTfO +tDIuUFUaNU52Q3Eg75N3ThVwLofDwR1t3Mu1J9QsVtFSUzpE0nPIxBsFZVpikpzu +QY0x2+c06lkh1QF612S4ZDnNye2v7UsDSKegmQGA3GWjNq5lWUhPgkvIZfFXHeVZ +Lgo/bNjR9eUJtGxUAArgFU2HdW23WJZa3W3SAKD0m0i+wzekujbgfIeFlxoVot4u +olu9rxj5kFDNcFn4J2dHy8egBzp90SxdbBk6ZrV9/ZFvgrG+CJPbFEfxojfHRZ48 +x3evZKiT3/Zpg4Jg8klCNO1aAFSFHBY2kgxc+qatv9s= +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Premium O=AffirmTrust +# Subject: CN=AffirmTrust Premium O=AffirmTrust +# Label: "AffirmTrust Premium" +# Serial: 7893706540734352110 +# MD5 Fingerprint: c4:5d:0e:48:b6:ac:28:30:4e:0a:bc:f9:38:16:87:57 +# SHA1 Fingerprint: d8:a6:33:2c:e0:03:6f:b1:85:f6:63:4f:7d:6a:06:65:26:32:28:27 +# SHA256 Fingerprint: 70:a7:3f:7f:37:6b:60:07:42:48:90:45:34:b1:14:82:d5:bf:0e:69:8e:cc:49:8d:f5:25:77:eb:f2:e9:3b:9a +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIIbYwURrGmCu4wDQYJKoZIhvcNAQEMBQAwQTELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1UcnVz +dCBQcmVtaXVtMB4XDTEwMDEyOTE0MTAzNloXDTQwMTIzMTE0MTAzNlowQTELMAkG +A1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1U +cnVzdCBQcmVtaXVtMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxBLf +qV/+Qd3d9Z+K4/as4Tx4mrzY8H96oDMq3I0gW64tb+eT2TZwamjPjlGjhVtnBKAQ +JG9dKILBl1fYSCkTtuG+kU3fhQxTGJoeJKJPj/CihQvL9Cl/0qRY7iZNyaqoe5rZ ++jjeRFcV5fiMyNlI4g0WJx0eyIOFJbe6qlVBzAMiSy2RjYvmia9mx+n/K+k8rNrS +s8PhaJyJ+HoAVt70VZVs+7pk3WKL3wt3MutizCaam7uqYoNMtAZ6MMgpv+0GTZe5 +HMQxK9VfvFMSF5yZVylmd2EhMQcuJUmdGPLu8ytxjLW6OQdJd/zvLpKQBY0tL3d7 +70O/Nbua2Plzpyzy0FfuKE4mX4+QaAkvuPjcBukumj5Rp9EixAqnOEhss/n/fauG +V+O61oV4d7pD6kh/9ti+I20ev9E2bFhc8e6kGVQa9QPSdubhjL08s9NIS+LI+H+S +qHZGnEJlPqQewQcDWkYtuJfzt9WyVSHvutxMAJf7FJUnM7/oQ0dG0giZFmA7mn7S +5u046uwBHjxIVkkJx0w3AJ6IDsBz4W9m6XJHMD4Q5QsDyZpCAGzFlH5hxIrff4Ia +C1nEWTJ3s7xgaVY5/bQGeyzWZDbZvUjthB9+pSKPKrhC9IK31FOQeE4tGv2Bb0TX +OwF0lkLgAOIua+rF7nKsu7/+6qqo+Nz2snmKtmcCAwEAAaNCMEAwHQYDVR0OBBYE +FJ3AZ6YMItkm9UWrpmVSESfYRaxjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgEGMA0GCSqGSIb3DQEBDAUAA4ICAQCzV00QYk465KzquByvMiPIs0laUZx2 +KI15qldGF9X1Uva3ROgIRL8YhNILgM3FEv0AVQVhh0HctSSePMTYyPtwni94loMg +Nt58D2kTiKV1NpgIpsbfrM7jWNa3Pt668+s0QNiigfV4Py/VpfzZotReBA4Xrf5B +8OWycvpEgjNC6C1Y91aMYj+6QrCcDFx+LmUmXFNPALJ4fqENmS2NuB2OosSw/WDQ +MKSOyARiqcTtNd56l+0OOF6SL5Nwpamcb6d9Ex1+xghIsV5n61EIJenmJWtSKZGc +0jlzCFfemQa0W50QBuHCAKi4HEoCChTQwUHK+4w1IX2COPKpVJEZNZOUbWo6xbLQ +u4mGk+ibyQ86p3q4ofB4Rvr8Ny/lioTz3/4E2aFooC8k4gmVBtWVyuEklut89pMF +u+1z6S3RdTnX5yTb2E5fQ4+e0BQ5v1VwSJlXMbSc7kqYA5YwH2AG7hsj/oFgIxpH +YoWlzBk0gG+zrBrjn/B7SK3VAdlntqlyk+otZrWyuOQ9PLLvTIzq6we/qzWaVYa8 +GKa1qF60g2xraUDTn9zxw2lrueFtCfTxqlB2Cnp9ehehVZZCmTEJ3WARjQUwfuaO +RtGdFNrHF+QFlozEJLUbzxQHskD4o55BhrwE0GuWyCqANP2/7waj3VjFhT0+j/6e +KeC2uAloGRwYQw== +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Premium ECC O=AffirmTrust +# Subject: CN=AffirmTrust Premium ECC O=AffirmTrust +# Label: "AffirmTrust Premium ECC" +# Serial: 8401224907861490260 +# MD5 Fingerprint: 64:b0:09:55:cf:b1:d5:99:e2:be:13:ab:a6:5d:ea:4d +# SHA1 Fingerprint: b8:23:6b:00:2f:1d:16:86:53:01:55:6c:11:a4:37:ca:eb:ff:c3:bb +# SHA256 Fingerprint: bd:71:fd:f6:da:97:e4:cf:62:d1:64:7a:dd:25:81:b0:7d:79:ad:f8:39:7e:b4:ec:ba:9c:5e:84:88:82:14:23 +-----BEGIN CERTIFICATE----- +MIIB/jCCAYWgAwIBAgIIdJclisc/elQwCgYIKoZIzj0EAwMwRTELMAkGA1UEBhMC +VVMxFDASBgNVBAoMC0FmZmlybVRydXN0MSAwHgYDVQQDDBdBZmZpcm1UcnVzdCBQ +cmVtaXVtIEVDQzAeFw0xMDAxMjkxNDIwMjRaFw00MDEyMzExNDIwMjRaMEUxCzAJ +BgNVBAYTAlVTMRQwEgYDVQQKDAtBZmZpcm1UcnVzdDEgMB4GA1UEAwwXQWZmaXJt +VHJ1c3QgUHJlbWl1bSBFQ0MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQNMF4bFZ0D +0KF5Nbc6PJJ6yhUczWLznCZcBz3lVPqj1swS6vQUX+iOGasvLkjmrBhDeKzQN8O9 +ss0s5kfiGuZjuD0uL3jET9v0D6RoTFVya5UdThhClXjMNzyR4ptlKymjQjBAMB0G +A1UdDgQWBBSaryl6wBE1NSZRMADDav5A1a7WPDAPBgNVHRMBAf8EBTADAQH/MA4G +A1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNnADBkAjAXCfOHiFBar8jAQr9HX/Vs +aobgxCd05DhT1wV/GzTjxi+zygk8N53X57hG8f2h4nECMEJZh0PUUd+60wkyWs6I +flc9nF9Ca/UHLbXwgpP5WW+uZPpY5Yse42O+tYHNbwKMeQ== +-----END CERTIFICATE----- + +# Issuer: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Subject: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Label: "Certum Trusted Network CA" +# Serial: 279744 +# MD5 Fingerprint: d5:e9:81:40:c5:18:69:fc:46:2c:89:75:62:0f:aa:78 +# SHA1 Fingerprint: 07:e0:32:e0:20:b7:2c:3f:19:2f:06:28:a2:59:3a:19:a7:0f:06:9e +# SHA256 Fingerprint: 5c:58:46:8d:55:f5:8e:49:7e:74:39:82:d2:b5:00:10:b6:d1:65:37:4a:cf:83:a7:d4:a3:2d:b7:68:c4:40:8e +-----BEGIN CERTIFICATE----- +MIIDuzCCAqOgAwIBAgIDBETAMA0GCSqGSIb3DQEBBQUAMH4xCzAJBgNVBAYTAlBM +MSIwIAYDVQQKExlVbml6ZXRvIFRlY2hub2xvZ2llcyBTLkEuMScwJQYDVQQLEx5D +ZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxIjAgBgNVBAMTGUNlcnR1bSBU +cnVzdGVkIE5ldHdvcmsgQ0EwHhcNMDgxMDIyMTIwNzM3WhcNMjkxMjMxMTIwNzM3 +WjB+MQswCQYDVQQGEwJQTDEiMCAGA1UEChMZVW5pemV0byBUZWNobm9sb2dpZXMg +Uy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MSIw +IAYDVQQDExlDZXJ0dW0gVHJ1c3RlZCBOZXR3b3JrIENBMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA4/t9o3K6wvDJFIf1awFO4W5AB7ptJ11/91sts1rH +UV+rpDKmYYe2bg+G0jACl/jXaVehGDldamR5xgFZrDwxSjh80gTSSyjoIF87B6LM +TXPb865Px1bVWqeWifrzq2jUI4ZZJ88JJ7ysbnKDHDBy3+Ci6dLhdHUZvSqeexVU +BBvXQzmtVSjF4hq79MDkrjhJM8x2hZ85RdKknvISjFH4fOQtf/WsX+sWn7Et0brM +kUJ3TCXJkDhv2/DM+44el1k+1WBO5gUo7Ul5E0u6SNsv+XLTOcr+H9g0cvW0QM8x +AcPs3hEtF10fuFDRXhmnad4HMyjKUJX5p1TLVIZQRan5SQIDAQABo0IwQDAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBQIds3LB/8k9sXN7buQvOKEN0Z19zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBAKaorSLOAT2mo/9i0Eidi15y +sHhE49wcrwn9I0j6vSrEuVUEtRCjjSfeC4Jj0O7eDDd5QVsisrCaQVymcODU0HfL +I9MA4GxWL+FpDQ3Zqr8hgVDZBqWo/5U30Kr+4rP1mS1FhIrlQgnXdAIv94nYmem8 +J9RHjboNRhx3zxSkHLmkMcScKHQDNP8zGSal6Q10tz6XxnboJ5ajZt3hrvJBW8qY +VoNzcOSGGtIxQbovvi0TWnZvTuhOgQ4/WwMioBK+ZlgRSssDxLQqKi2WF+A5VLxI +03YnnZotBqbJ7DnSq9ufmgsnAjUpsUCV5/nonFWIGUbWtzT1fs45mtk48VH3Tyw= +-----END CERTIFICATE----- + +# Issuer: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA +# Label: "TWCA Root Certification Authority" +# Serial: 1 +# MD5 Fingerprint: aa:08:8f:f6:f9:7b:b7:f2:b1:a7:1e:9b:ea:ea:bd:79 +# SHA1 Fingerprint: cf:9e:87:6d:d3:eb:fc:42:26:97:a3:b5:a3:7a:a0:76:a9:06:23:48 +# SHA256 Fingerprint: bf:d8:8f:e1:10:1c:41:ae:3e:80:1b:f8:be:56:35:0e:e9:ba:d1:a6:b9:bd:51:5e:dc:5c:6d:5b:87:11:ac:44 +-----BEGIN CERTIFICATE----- +MIIDezCCAmOgAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJUVzES +MBAGA1UECgwJVEFJV0FOLUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFU +V0NBIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwODI4MDcyNDMz +WhcNMzAxMjMxMTU1OTU5WjBfMQswCQYDVQQGEwJUVzESMBAGA1UECgwJVEFJV0FO +LUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFUV0NBIFJvb3QgQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCwfnK4pAOU5qfeCTiRShFAh6d8WWQUe7UREN3+v9XAu1bihSX0NXIP+FPQQeFE +AcK0HMMxQhZHhTMidrIKbw/lJVBPhYa+v5guEGcevhEFhgWQxFnQfHgQsIBct+HH +K3XLfJ+utdGdIzdjp9xCoi2SBBtQwXu4PhvJVgSLL1KbralW6cH/ralYhzC2gfeX +RfwZVzsrb+RH9JlF/h3x+JejiB03HFyP4HYlmlD4oFT/RJB2I9IyxsOrBr/8+7/z +rX2SYgJbKdM1o5OaQ2RgXbL6Mv87BK9NQGr5x+PvI/1ry+UPizgN7gr8/g+YnzAx +3WxSZfmLgb4i4RxYA7qRG4kHAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqOFsmjd6LWvJPelSDGRjjCDWmujANBgkq +hkiG9w0BAQUFAAOCAQEAPNV3PdrfibqHDAhUaiBQkr6wQT25JmSDCi/oQMCXKCeC +MErJk/9q56YAf4lCmtYR5VPOL8zy2gXE/uJQxDqGfczafhAJO5I1KlOy/usrBdls +XebQ79NqZp4VKIV66IIArB6nCWlWQtNoURi+VJq/REG6Sb4gumlc7rh3zc5sH62D +lhh9DrUUOYTxKOkto557HnpyWoOzeW/vtPzQCqVYT0bf+215WfKEIlKuD8z7fDvn +aspHYcN6+NOSBB+4IIThNlQWx0DeO4pz3N/GCUzf7Nr/1FNCocnyYh0igzyXxfkZ +YiesZSLX0zzG5Y6yU8xJzrww/nsOM5D77dIUkR8Hrw== +-----END CERTIFICATE----- + +# Issuer: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 +# Subject: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 +# Label: "Security Communication RootCA2" +# Serial: 0 +# MD5 Fingerprint: 6c:39:7d:a4:0e:55:59:b2:3f:d6:41:b1:12:50:de:43 +# SHA1 Fingerprint: 5f:3b:8c:f2:f8:10:b3:7d:78:b4:ce:ec:19:19:c3:73:34:b9:c7:74 +# SHA256 Fingerprint: 51:3b:2c:ec:b8:10:d4:cd:e5:dd:85:39:1a:df:c6:c2:dd:60:d8:7b:b7:36:d2:b5:21:48:4a:a4:7a:0e:be:f6 +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIBADANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJKUDEl +MCMGA1UEChMcU0VDT00gVHJ1c3QgU3lzdGVtcyBDTy4sTFRELjEnMCUGA1UECxMe +U2VjdXJpdHkgQ29tbXVuaWNhdGlvbiBSb290Q0EyMB4XDTA5MDUyOTA1MDAzOVoX +DTI5MDUyOTA1MDAzOVowXTELMAkGA1UEBhMCSlAxJTAjBgNVBAoTHFNFQ09NIFRy +dXN0IFN5c3RlbXMgQ08uLExURC4xJzAlBgNVBAsTHlNlY3VyaXR5IENvbW11bmlj +YXRpb24gUm9vdENBMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANAV +OVKxUrO6xVmCxF1SrjpDZYBLx/KWvNs2l9amZIyoXvDjChz335c9S672XewhtUGr +zbl+dp+++T42NKA7wfYxEUV0kz1XgMX5iZnK5atq1LXaQZAQwdbWQonCv/Q4EpVM +VAX3NuRFg3sUZdbcDE3R3n4MqzvEFb46VqZab3ZpUql6ucjrappdUtAtCms1FgkQ +hNBqyjoGADdH5H5XTz+L62e4iKrFvlNVspHEfbmwhRkGeC7bYRr6hfVKkaHnFtWO +ojnflLhwHyg/i/xAXmODPIMqGplrz95Zajv8bxbXH/1KEOtOghY6rCcMU/Gt1SSw +awNQwS08Ft1ENCcadfsCAwEAAaNCMEAwHQYDVR0OBBYEFAqFqXdlBZh8QIH4D5cs +OPEK7DzPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBMOqNErLlFsceTfsgLCkLfZOoc7llsCLqJX2rKSpWeeo8HxdpF +coJxDjrSzG+ntKEju/Ykn8sX/oymzsLS28yN/HH8AynBbF0zX2S2ZTuJbxh2ePXc +okgfGT+Ok+vx+hfuzU7jBBJV1uXk3fs+BXziHV7Gp7yXT2g69ekuCkO2r1dcYmh8 +t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy +1UkC9gLl9B/rfNmWVan/7Ir5mUf/NVoCqgTLiluHcSmRvaS0eg29mvVXIwAHIRc/ +SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 +-----END CERTIFICATE----- + +# Issuer: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority +# Subject: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority +# Label: "Hellenic Academic and Research Institutions RootCA 2011" +# Serial: 0 +# MD5 Fingerprint: 73:9f:4c:4b:73:5b:79:e9:fa:ba:1c:ef:6e:cb:d5:c9 +# SHA1 Fingerprint: fe:45:65:9b:79:03:5b:98:a1:61:b5:51:2e:ac:da:58:09:48:22:4d +# SHA256 Fingerprint: bc:10:4f:15:a4:8b:e7:09:dc:a5:42:a7:e1:d4:b9:df:6f:05:45:27:e8:02:ea:a9:2d:59:54:44:25:8a:fe:71 +-----BEGIN CERTIFICATE----- +MIIEMTCCAxmgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBlTELMAkGA1UEBhMCR1Ix +RDBCBgNVBAoTO0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1 +dGlvbnMgQ2VydC4gQXV0aG9yaXR5MUAwPgYDVQQDEzdIZWxsZW5pYyBBY2FkZW1p +YyBhbmQgUmVzZWFyY2ggSW5zdGl0dXRpb25zIFJvb3RDQSAyMDExMB4XDTExMTIw +NjEzNDk1MloXDTMxMTIwMTEzNDk1MlowgZUxCzAJBgNVBAYTAkdSMUQwQgYDVQQK +EztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0dXRpb25zIENl +cnQuIEF1dGhvcml0eTFAMD4GA1UEAxM3SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl +c2VhcmNoIEluc3RpdHV0aW9ucyBSb290Q0EgMjAxMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAKlTAOMupvaO+mDYLZU++CwqVE7NuYRhlFhPjz2L5EPz +dYmNUeTDN9KKiE15HrcS3UN4SoqS5tdI1Q+kOilENbgH9mgdVc04UfCMJDGFr4PJ +fel3r+0ae50X+bOdOFAPplp5kYCvN66m0zH7tSYJnTxa71HFK9+WXesyHgLacEns +bgzImjeN9/E2YEsmLIKe0HjzDQ9jpFEw4fkrJxIH2Oq9GGKYsFk3fb7u8yBRQlqD +75O6aRXxYp2fmTmCobd0LovUxQt7L/DICto9eQqakxylKHJzkUOap9FNhYS5qXSP +FEDH3N6sQWRstBmbAmNtJGSPRLIl6s5ddAxjMlyNh+UCAwEAAaOBiTCBhjAPBgNV +HRMBAf8EBTADAQH/MAsGA1UdDwQEAwIBBjAdBgNVHQ4EFgQUppFC/RNhSiOeCKQp +5dgTBCPuQSUwRwYDVR0eBEAwPqA8MAWCAy5ncjAFggMuZXUwBoIELmVkdTAGggQu +b3JnMAWBAy5ncjAFgQMuZXUwBoEELmVkdTAGgQQub3JnMA0GCSqGSIb3DQEBBQUA +A4IBAQAf73lB4XtuP7KMhjdCSk4cNx6NZrokgclPEg8hwAOXhiVtXdMiKahsog2p +6z0GW5k6x8zDmjR/qw7IThzh+uTczQ2+vyT+bOdrwg3IBp5OjWEopmr95fZi6hg8 +TqBTnbI6nOulnJEWtk2C4AwFSKls9cz4y51JtPACpf1wA+2KIaWuE4ZJwzNzvoc7 +dIsXRSZMFpGD/md9zU1jZ/rzAxKWeAaNsWftjj++n08C9bMJL/NMh98qy5V8Acys +Nnq/onN694/BtZqhFLKPM58N7yLcZnuEvUUXBj08yrl3NI/K6s8/MT7jiOOASSXI +l7WdmplNsDz4SgCbZN2fOUvRJ9e4 +-----END CERTIFICATE----- + +# Issuer: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 +# Subject: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 +# Label: "Actalis Authentication Root CA" +# Serial: 6271844772424770508 +# MD5 Fingerprint: 69:c1:0d:4f:07:a3:1b:c3:fe:56:3d:04:bc:11:f6:a6 +# SHA1 Fingerprint: f3:73:b3:87:06:5a:28:84:8a:f2:f3:4a:ce:19:2b:dd:c7:8e:9c:ac +# SHA256 Fingerprint: 55:92:60:84:ec:96:3a:64:b9:6e:2a:be:01:ce:0b:a8:6a:64:fb:fe:bc:c7:aa:b5:af:c1:55:b3:7f:d7:60:66 +-----BEGIN CERTIFICATE----- +MIIFuzCCA6OgAwIBAgIIVwoRl0LE48wwDQYJKoZIhvcNAQELBQAwazELMAkGA1UE +BhMCSVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8w +MzM1ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290 +IENBMB4XDTExMDkyMjExMjIwMloXDTMwMDkyMjExMjIwMlowazELMAkGA1UEBhMC +SVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8wMzM1 +ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290IENB +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAp8bEpSmkLO/lGMWwUKNv +UTufClrJwkg4CsIcoBh/kbWHuUA/3R1oHwiD1S0eiKD4j1aPbZkCkpAW1V8IbInX +4ay8IMKx4INRimlNAJZaby/ARH6jDuSRzVju3PvHHkVH3Se5CAGfpiEd9UEtL0z9 +KK3giq0itFZljoZUj5NDKd45RnijMCO6zfB9E1fAXdKDa0hMxKufgFpbOr3JpyI/ +gCczWw63igxdBzcIy2zSekciRDXFzMwujt0q7bd9Zg1fYVEiVRvjRuPjPdA1Yprb +rxTIW6HMiRvhMCb8oJsfgadHHwTrozmSBp+Z07/T6k9QnBn+locePGX2oxgkg4YQ +51Q+qDp2JE+BIcXjDwL4k5RHILv+1A7TaLndxHqEguNTVHnd25zS8gebLra8Pu2F +be8lEfKXGkJh90qX6IuxEAf6ZYGyojnP9zz/GPvG8VqLWeICrHuS0E4UT1lF9gxe +KF+w6D9Fz8+vm2/7hNN3WpVvrJSEnu68wEqPSpP4RCHiMUVhUE4Q2OM1fEwZtN4F +v6MGn8i1zeQf1xcGDXqVdFUNaBr8EBtiZJ1t4JWgw5QHVw0U5r0F+7if5t+L4sbn +fpb2U8WANFAoWPASUHEXMLrmeGO89LKtmyuy/uE5jF66CyCU3nuDuP/jVo23Eek7 +jPKxwV2dpAtMK9myGPW1n0sCAwEAAaNjMGEwHQYDVR0OBBYEFFLYiDrIn3hm7Ynz +ezhwlMkCAjbQMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUUtiIOsifeGbt +ifN7OHCUyQICNtAwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAL +e3KHwGCmSUyIWOYdiPcUZEim2FgKDk8TNd81HdTtBjHIgT5q1d07GjLukD0R0i70 +jsNjLiNmsGe+b7bAEzlgqqI0JZN1Ut6nna0Oh4lScWoWPBkdg/iaKWW+9D+a2fDz +WochcYBNy+A4mz+7+uAwTc+G02UQGRjRlwKxK3JCaKygvU5a2hi/a5iB0P2avl4V +SM0RFbnAKVy06Ij3Pjaut2L9HmLecHgQHEhb2rykOLpn7VU+Xlff1ANATIGk0k9j +pwlCCRT8AKnCgHNPLsBA2RF7SOp6AsDT6ygBJlh0wcBzIm2Tlf05fbsq4/aC4yyX +X04fkZT6/iyj2HYauE2yOE+b+h1IYHkm4vP9qdCa6HCPSXrW5b0KDtst842/6+Ok +fcvHlXHo2qN8xcL4dJIEG4aspCJTQLas/kx2z/uUMsA1n3Y/buWQbqCmJqK4LL7R +K4X9p2jIugErsWx0Hbhzlefut8cl8ABMALJ+tguLHPPAUJ4lueAI3jZm/zel0btU +ZCzJJ7VLkn5l/9Mt4blOvH+kQSGQQXemOR/qnuOf0GZvBeyqdn6/axag67XH/JJU +LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT +LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg== +-----END CERTIFICATE----- + +# Issuer: O=Trustis Limited OU=Trustis FPS Root CA +# Subject: O=Trustis Limited OU=Trustis FPS Root CA +# Label: "Trustis FPS Root CA" +# Serial: 36053640375399034304724988975563710553 +# MD5 Fingerprint: 30:c9:e7:1e:6b:e6:14:eb:65:b2:16:69:20:31:67:4d +# SHA1 Fingerprint: 3b:c0:38:0b:33:c3:f6:a6:0c:86:15:22:93:d9:df:f5:4b:81:c0:04 +# SHA256 Fingerprint: c1:b4:82:99:ab:a5:20:8f:e9:63:0a:ce:55:ca:68:a0:3e:da:5a:51:9c:88:02:a0:d3:a6:73:be:8f:8e:55:7d +-----BEGIN CERTIFICATE----- +MIIDZzCCAk+gAwIBAgIQGx+ttiD5JNM2a/fH8YygWTANBgkqhkiG9w0BAQUFADBF +MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPVHJ1c3RpcyBMaW1pdGVkMRwwGgYDVQQL +ExNUcnVzdGlzIEZQUyBSb290IENBMB4XDTAzMTIyMzEyMTQwNloXDTI0MDEyMTEx +MzY1NFowRTELMAkGA1UEBhMCR0IxGDAWBgNVBAoTD1RydXN0aXMgTGltaXRlZDEc +MBoGA1UECxMTVHJ1c3RpcyBGUFMgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAMVQe547NdDfxIzNjpvto8A2mfRC6qc+gIMPpqdZh8mQRUN+ +AOqGeSoDvT03mYlmt+WKVoaTnGhLaASMk5MCPjDSNzoiYYkchU59j9WvezX2fihH +iTHcDnlkH5nSW7r+f2C/revnPDgpai/lkQtV/+xvWNUtyd5MZnGPDNcE2gfmHhjj +vSkCqPoc4Vu5g6hBSLwacY3nYuUtsuvffM/bq1rKMfFMIvMFE/eC+XN5DL7XSxzA +0RU8k0Fk0ea+IxciAIleH2ulrG6nS4zto3Lmr2NNL4XSFDWaLk6M6jKYKIahkQlB +OrTh4/L68MkKokHdqeMDx4gVOxzUGpTXn2RZEm0CAwEAAaNTMFEwDwYDVR0TAQH/ +BAUwAwEB/zAfBgNVHSMEGDAWgBS6+nEleYtXQSUhhgtx67JkDoshZzAdBgNVHQ4E +FgQUuvpxJXmLV0ElIYYLceuyZA6LIWcwDQYJKoZIhvcNAQEFBQADggEBAH5Y//01 +GX2cGE+esCu8jowU/yyg2kdbw++BLa8F6nRIW/M+TgfHbcWzk88iNVy2P3UnXwmW +zaD+vkAMXBJV+JOCyinpXj9WV4s4NvdFGkwozZ5BuO1WTISkQMi4sKUraXAEasP4 +1BIy+Q7DsdwyhEQsb8tGD+pmQQ9P8Vilpg0ND2HepZ5dfWWhPBfnqFVO76DH7cZE +f1T1o+CP8HxVIo8ptoGj4W1OLBuAZ+ytIJ8MYmHVl/9D7S3B2l0pKoU/rGXuhg8F +jZBf3+6f9L/uHfuY5H+QK4R4EA5sSVPvFVtlRkpdr7r7OnIdzfYliB6XzCGcKQEN +ZetX2fNXlrtIzYE= +-----END CERTIFICATE----- + +# Issuer: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 +# Subject: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 +# Label: "Buypass Class 2 Root CA" +# Serial: 2 +# MD5 Fingerprint: 46:a7:d2:fe:45:fb:64:5a:a8:59:90:9b:78:44:9b:29 +# SHA1 Fingerprint: 49:0a:75:74:de:87:0a:47:fe:58:ee:f6:c7:6b:eb:c6:0b:12:40:99 +# SHA256 Fingerprint: 9a:11:40:25:19:7c:5b:b9:5d:94:e6:3d:55:cd:43:79:08:47:b6:46:b2:3c:df:11:ad:a4:a0:0e:ff:15:fb:48 +-----BEGIN CERTIFICATE----- +MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd +MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg +Q2xhc3MgMiBSb290IENBMB4XDTEwMTAyNjA4MzgwM1oXDTQwMTAyNjA4MzgwM1ow +TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw +HgYDVQQDDBdCdXlwYXNzIENsYXNzIDIgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB +BQADggIPADCCAgoCggIBANfHXvfBB9R3+0Mh9PT1aeTuMgHbo4Yf5FkNuud1g1Lr +6hxhFUi7HQfKjK6w3Jad6sNgkoaCKHOcVgb/S2TwDCo3SbXlzwx87vFKu3MwZfPV +L4O2fuPn9Z6rYPnT8Z2SdIrkHJasW4DptfQxh6NR/Md+oW+OU3fUl8FVM5I+GC91 +1K2GScuVr1QGbNgGE41b/+EmGVnAJLqBcXmQRFBoJJRfuLMR8SlBYaNByyM21cHx +MlAQTn/0hpPshNOOvEu/XAFOBz3cFIqUCqTqc/sLUegTBxj6DvEr0VQVfTzh97QZ +QmdiXnfgolXsttlpF9U6r0TtSsWe5HonfOV116rLJeffawrbD02TTqigzXsu8lkB +arcNuAeBfos4GzjmCleZPe4h6KP1DBbdi+w0jpwqHAAVF41og9JwnxgIzRFo1clr +Us3ERo/ctfPYV3Me6ZQ5BL/T3jjetFPsaRyifsSP5BtwrfKi+fv3FmRmaZ9JUaLi +FRhnBkp/1Wy1TbMz4GHrXb7pmA8y1x1LPC5aAVKRCfLf6o3YBkBjqhHk/sM3nhRS +P/TizPJhk9H9Z2vXUq6/aKtAQ6BXNVN48FP4YUIHZMbXb5tMOA1jrGKvNouicwoN +9SG9dKpN6nIDSdvHXx1iY8f93ZHsM+71bbRuMGjeyNYmsHVee7QHIJihdjK4TWxP +AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMmAd+BikoL1Rpzz +uvdMw964o605MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAU18h +9bqwOlI5LJKwbADJ784g7wbylp7ppHR/ehb8t/W2+xUbP6umwHJdELFx7rxP462s +A20ucS6vxOOto70MEae0/0qyexAQH6dXQbLArvQsWdZHEIjzIVEpMMpghq9Gqx3t +OluwlN5E40EIosHsHdb9T7bWR9AUC8rmyrV7d35BH16Dx7aMOZawP5aBQW9gkOLo ++fsicdl9sz1Gv7SEr5AcD48Saq/v7h56rgJKihcrdv6sVIkkLE8/trKnToyokZf7 +KcZ7XC25y2a2t6hbElGFtQl+Ynhw/qlqYLYdDnkM/crqJIByw5c/8nerQyIKx+u2 +DISCLIBrQYoIwOula9+ZEsuK1V6ADJHgJgg2SMX6OBE1/yWDLfJ6v9r9jv6ly0Us +H8SIU653DtmadsWOLB2jutXsMq7Aqqz30XpN69QH4kj3Io6wpJ9qzo6ysmD0oyLQ +I+uUWnpp3Q+/QFesa1lQ2aOZ4W7+jQF5JyMV3pKdewlNWudLSDBaGOYKbeaP4NK7 +5t98biGCwWg5TbSYWGZizEqQXsP6JwSxeRV0mcy+rSDeJmAc61ZRpqPq5KM/p/9h +3PFaTWwyI0PurKju7koSCTxdccK+efrCh2gdC/1cacwG0Jp9VJkqyTkaGa9LKkPz +Y11aWOIv4x3kqdbQCtCev9eBCfHJxyYNrJgWVqA= +-----END CERTIFICATE----- + +# Issuer: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 +# Subject: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 +# Label: "Buypass Class 3 Root CA" +# Serial: 2 +# MD5 Fingerprint: 3d:3b:18:9e:2c:64:5a:e8:d5:88:ce:0e:f9:37:c2:ec +# SHA1 Fingerprint: da:fa:f7:fa:66:84:ec:06:8f:14:50:bd:c7:c2:81:a5:bc:a9:64:57 +# SHA256 Fingerprint: ed:f7:eb:bc:a2:7a:2a:38:4d:38:7b:7d:40:10:c6:66:e2:ed:b4:84:3e:4c:29:b4:ae:1d:5b:93:32:e6:b2:4d +-----BEGIN CERTIFICATE----- +MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd +MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg +Q2xhc3MgMyBSb290IENBMB4XDTEwMTAyNjA4Mjg1OFoXDTQwMTAyNjA4Mjg1OFow +TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw +HgYDVQQDDBdCdXlwYXNzIENsYXNzIDMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB +BQADggIPADCCAgoCggIBAKXaCpUWUOOV8l6ddjEGMnqb8RB2uACatVI2zSRHsJ8Y +ZLya9vrVediQYkwiL944PdbgqOkcLNt4EemOaFEVcsfzM4fkoF0LXOBXByow9c3E +N3coTRiR5r/VUv1xLXA+58bEiuPwKAv0dpihi4dVsjoT/Lc+JzeOIuOoTyrvYLs9 +tznDDgFHmV0ST9tD+leh7fmdvhFHJlsTmKtdFoqwNxxXnUX/iJY2v7vKB3tvh2PX +0DJq1l1sDPGzbjniazEuOQAnFN44wOwZZoYS6J1yFhNkUsepNxz9gjDthBgd9K5c +/3ATAOux9TN6S9ZV+AWNS2mw9bMoNlwUxFFzTWsL8TQH2xc519woe2v1n/MuwU8X +KhDzzMro6/1rqy6any2CbgTUUgGTLT2G/H783+9CHaZr77kgxve9oKeV/afmiSTY +zIw0bOIjL9kSGiG5VZFvC5F5GQytQIgLcOJ60g7YaEi7ghM5EFjp2CoHxhLbWNvS +O1UQRwUVZ2J+GGOmRj8JDlQyXr8NYnon74Do29lLBlo3WiXQCBJ31G8JUJc9yB3D +34xFMFbG02SrZvPAXpacw8Tvw3xrizp5f7NJzz3iiZ+gMEuFuZyUJHmPfWupRWgP +K9Dx2hzLabjKSWJtyNBjYt1gD1iqj6G8BaVmos8bdrKEZLFMOVLAMLrwjEsCsLa3 +AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEe4zf/lb+74suwv +Tg75JbCOPGvDMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAACAj +QTUEkMJAYmDv4jVM1z+s4jSQuKFvdvoWFqRINyzpkMLyPPgKn9iB5btb2iUspKdV +cSQy9sgL8rxq+JOssgfCX5/bzMiKqr5qb+FJEMwx14C7u8jYog5kV+qi9cKpMRXS +IGrs/CIBKM+GuIAeqcwRpTzyFrNHnfzSgCHEy9BHcEGhyoMZCCxt8l13nIoUE9Q2 +HJLw5QY33KbmkJs4j1xrG0aGQ0JfPgEHU1RdZX33inOhmlRaHylDFCfChQ+1iHsa +O5S3HWCntZznKWlXWpuTekMwGwPXYshApqr8ZORK15FTAaggiG6cX0S5y2CBNOxv +033aSF/rtJC8LakcC6wc1aJoIIAE1vyxjy+7SjENSoYc6+I2KSb12tjE8nVhz36u +dmNKekBlk4f4HoCMhuWG1o8O/FMsYOgWYRqiPkN7zTlgVGr18okmAWiDSKIz6MkE +kbIRNBE+6tBDGR8Dk5AM/1E9V/RBbuHLoL7ryWPNbczk+DaqaJ3tvV2XcEQNtg41 +3OEMXbugUZTLfhbrES+jkkXITHHZvMmZUldGL1DPvTVp9D0VzgalLA8+9oG6lLvD +u79leNKGef9JOxqDDPDeeOzI8k1MGt6CKfjBWtrt7uYnXuhF0J0cUahoq0Tj0Itq +4/g7u9xN12TyUb7mqqta6THuBrxzvxNiCp/HuZc= +-----END CERTIFICATE----- + +# Issuer: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Subject: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Label: "T-TeleSec GlobalRoot Class 3" +# Serial: 1 +# MD5 Fingerprint: ca:fb:40:a8:4e:39:92:8a:1d:fe:8e:2f:c4:27:ea:ef +# SHA1 Fingerprint: 55:a6:72:3e:cb:f2:ec:cd:c3:23:74:70:19:9d:2a:be:11:e3:81:d1 +# SHA256 Fingerprint: fd:73:da:d3:1c:64:4f:f1:b4:3b:ef:0c:cd:da:96:71:0b:9c:d9:87:5e:ca:7e:31:70:7a:f3:e9:6d:52:2b:bd +-----BEGIN CERTIFICATE----- +MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx +KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd +BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl +YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 +OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy +aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 +ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN +8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ +RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 +hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 +ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM +EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 +A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy +WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ +1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 +6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT +91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml +e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p +TpPDpFQUWw== +-----END CERTIFICATE----- + +# Issuer: CN=EE Certification Centre Root CA O=AS Sertifitseerimiskeskus +# Subject: CN=EE Certification Centre Root CA O=AS Sertifitseerimiskeskus +# Label: "EE Certification Centre Root CA" +# Serial: 112324828676200291871926431888494945866 +# MD5 Fingerprint: 43:5e:88:d4:7d:1a:4a:7e:fd:84:2e:52:eb:01:d4:6f +# SHA1 Fingerprint: c9:a8:b9:e7:55:80:5e:58:e3:53:77:a7:25:eb:af:c3:7b:27:cc:d7 +# SHA256 Fingerprint: 3e:84:ba:43:42:90:85:16:e7:75:73:c0:99:2f:09:79:ca:08:4e:46:85:68:1f:f1:95:cc:ba:8a:22:9b:8a:76 +-----BEGIN CERTIFICATE----- +MIIEAzCCAuugAwIBAgIQVID5oHPtPwBMyonY43HmSjANBgkqhkiG9w0BAQUFADB1 +MQswCQYDVQQGEwJFRTEiMCAGA1UECgwZQVMgU2VydGlmaXRzZWVyaW1pc2tlc2t1 +czEoMCYGA1UEAwwfRUUgQ2VydGlmaWNhdGlvbiBDZW50cmUgUm9vdCBDQTEYMBYG +CSqGSIb3DQEJARYJcGtpQHNrLmVlMCIYDzIwMTAxMDMwMTAxMDMwWhgPMjAzMDEy +MTcyMzU5NTlaMHUxCzAJBgNVBAYTAkVFMSIwIAYDVQQKDBlBUyBTZXJ0aWZpdHNl +ZXJpbWlza2Vza3VzMSgwJgYDVQQDDB9FRSBDZXJ0aWZpY2F0aW9uIENlbnRyZSBS +b290IENBMRgwFgYJKoZIhvcNAQkBFglwa2lAc2suZWUwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDIIMDs4MVLqwd4lfNE7vsLDP90jmG7sWLqI9iroWUy +euuOF0+W2Ap7kaJjbMeMTC55v6kF/GlclY1i+blw7cNRfdCT5mzrMEvhvH2/UpvO +bntl8jixwKIy72KyaOBhU8E2lf/slLo2rpwcpzIP5Xy0xm90/XsY6KxX7QYgSzIw +WFv9zajmofxwvI6Sc9uXp3whrj3B9UiHbCe9nyV0gVWw93X2PaRka9ZP585ArQ/d +MtO8ihJTmMmJ+xAdTX7Nfh9WDSFwhfYggx/2uh8Ej+p3iDXE/+pOoYtNP2MbRMNE +1CV2yreN1x5KZmTNXMWcg+HCCIia7E6j8T4cLNlsHaFLAgMBAAGjgYowgYcwDwYD +VR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBLyWj7qVhy/ +zQas8fElyalL1BSZMEUGA1UdJQQ+MDwGCCsGAQUFBwMCBggrBgEFBQcDAQYIKwYB +BQUHAwMGCCsGAQUFBwMEBggrBgEFBQcDCAYIKwYBBQUHAwkwDQYJKoZIhvcNAQEF +BQADggEBAHv25MANqhlHt01Xo/6tu7Fq1Q+e2+RjxY6hUFaTlrg4wCQiZrxTFGGV +v9DHKpY5P30osxBAIWrEr7BSdxjhlthWXePdNl4dp1BUoMUq5KqMlIpPnTX/dqQG +E5Gion0ARD9V04I8GtVbvFZMIi5GQ4okQC3zErg7cBqklrkar4dBGmoYDQZPxz5u +uSlNDUmJEYcyW+ZLBMjkXOZ0c5RdFpgTlf7727FE5TpwrDdr5rMzcijJs1eg9gIW +iAYLtqZLICjU3j2LrTcFU3T+bsy8QxdxXvnFzBqpYe73dgzzcvRyrc9yAjYHR8/v +GVCJYMzpJJUPwssd8m92kMfMdcGWxZ0= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH +# Subject: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH +# Label: "D-TRUST Root Class 3 CA 2 2009" +# Serial: 623603 +# MD5 Fingerprint: cd:e0:25:69:8d:47:ac:9c:89:35:90:f7:fd:51:3d:2f +# SHA1 Fingerprint: 58:e8:ab:b0:36:15:33:fb:80:f7:9b:1b:6d:29:d3:ff:8d:5f:00:f0 +# SHA256 Fingerprint: 49:e7:a4:42:ac:f0:ea:62:87:05:00:54:b5:25:64:b6:50:e4:f4:9e:42:e3:48:d6:aa:38:e0:39:e9:57:b1:c1 +-----BEGIN CERTIFICATE----- +MIIEMzCCAxugAwIBAgIDCYPzMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNVBAYTAkRF +MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMMHkQtVFJVU1QgUm9vdCBD +bGFzcyAzIENBIDIgMjAwOTAeFw0wOTExMDUwODM1NThaFw0yOTExMDUwODM1NTha +ME0xCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMM +HkQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgMjAwOTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBANOySs96R+91myP6Oi/WUEWJNTrGa9v+2wBoqOADER03 +UAifTUpolDWzU9GUY6cgVq/eUXjsKj3zSEhQPgrfRlWLJ23DEE0NkVJD2IfgXU42 +tSHKXzlABF9bfsyjxiupQB7ZNoTWSPOSHjRGICTBpFGOShrvUD9pXRl/RcPHAY9R +ySPocq60vFYJfxLLHLGvKZAKyVXMD9O0Gu1HNVpK7ZxzBCHQqr0ME7UAyiZsxGsM +lFqVlNpQmvH/pStmMaTJOKDfHR+4CS7zp+hnUquVH+BGPtikw8paxTGA6Eian5Rp +/hnd2HN8gcqW3o7tszIFZYQ05ub9VxC1X3a/L7AQDcUCAwEAAaOCARowggEWMA8G +A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFP3aFMSfMN4hvR5COfyrYyNJ4PGEMA4G +A1UdDwEB/wQEAwIBBjCB0wYDVR0fBIHLMIHIMIGAoH6gfIZ6bGRhcDovL2RpcmVj +dG9yeS5kLXRydXN0Lm5ldC9DTj1ELVRSVVNUJTIwUm9vdCUyMENsYXNzJTIwMyUy +MENBJTIwMiUyMDIwMDksTz1ELVRydXN0JTIwR21iSCxDPURFP2NlcnRpZmljYXRl +cmV2b2NhdGlvbmxpc3QwQ6BBoD+GPWh0dHA6Ly93d3cuZC10cnVzdC5uZXQvY3Js +L2QtdHJ1c3Rfcm9vdF9jbGFzc18zX2NhXzJfMjAwOS5jcmwwDQYJKoZIhvcNAQEL +BQADggEBAH+X2zDI36ScfSF6gHDOFBJpiBSVYEQBrLLpME+bUMJm2H6NMLVwMeni +acfzcNsgFYbQDfC+rAF1hM5+n02/t2A7nPPKHeJeaNijnZflQGDSNiH+0LS4F9p0 +o3/U37CYAqxva2ssJSRyoWXuJVrl5jLn8t+rSfrzkGkj2wTZ51xY/GXUl77M/C4K +zCUqNQT4YJEVdT1B/yMfGchs64JTBKbkTCJNjYy6zltz7GRUUG3RnFX7acM2w4y8 +PIWmawomDeCTmGCufsYkl4phX5GOZpIJhzbNi5stPvZR1FDUWSi9g/LMKHtThm3Y +Johw1+qRzT65ysCQblrGXnRl11z+o+I= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH +# Subject: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH +# Label: "D-TRUST Root Class 3 CA 2 EV 2009" +# Serial: 623604 +# MD5 Fingerprint: aa:c6:43:2c:5e:2d:cd:c4:34:c0:50:4f:11:02:4f:b6 +# SHA1 Fingerprint: 96:c9:1b:0b:95:b4:10:98:42:fa:d0:d8:22:79:fe:60:fa:b9:16:83 +# SHA256 Fingerprint: ee:c5:49:6b:98:8c:e9:86:25:b9:34:09:2e:ec:29:08:be:d0:b0:f3:16:c2:d4:73:0c:84:ea:f1:f3:d3:48:81 +-----BEGIN CERTIFICATE----- +MIIEQzCCAyugAwIBAgIDCYP0MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNVBAYTAkRF +MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNVBAMMIUQtVFJVU1QgUm9vdCBD +bGFzcyAzIENBIDIgRVYgMjAwOTAeFw0wOTExMDUwODUwNDZaFw0yOTExMDUwODUw +NDZaMFAxCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNV +BAMMIUQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgRVYgMjAwOTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAJnxhDRwui+3MKCOvXwEz75ivJn9gpfSegpn +ljgJ9hBOlSJzmY3aFS3nBfwZcyK3jpgAvDw9rKFs+9Z5JUut8Mxk2og+KbgPCdM0 +3TP1YtHhzRnp7hhPTFiu4h7WDFsVWtg6uMQYZB7jM7K1iXdODL/ZlGsTl28So/6Z +qQTMFexgaDbtCHu39b+T7WYxg4zGcTSHThfqr4uRjRxWQa4iN1438h3Z0S0NL2lR +p75mpoo6Kr3HGrHhFPC+Oh25z1uxav60sUYgovseO3Dvk5h9jHOW8sXvhXCtKSb8 +HgQ+HKDYD8tSg2J87otTlZCpV6LqYQXY+U3EJ/pure3511H3a6UCAwEAAaOCASQw +ggEgMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNOUikxiEyoZLsyvcop9Ntea +HNxnMA4GA1UdDwEB/wQEAwIBBjCB3QYDVR0fBIHVMIHSMIGHoIGEoIGBhn9sZGFw +Oi8vZGlyZWN0b3J5LmQtdHJ1c3QubmV0L0NOPUQtVFJVU1QlMjBSb290JTIwQ2xh +c3MlMjAzJTIwQ0ElMjAyJTIwRVYlMjAyMDA5LE89RC1UcnVzdCUyMEdtYkgsQz1E +RT9jZXJ0aWZpY2F0ZXJldm9jYXRpb25saXN0MEagRKBChkBodHRwOi8vd3d3LmQt +dHJ1c3QubmV0L2NybC9kLXRydXN0X3Jvb3RfY2xhc3NfM19jYV8yX2V2XzIwMDku +Y3JsMA0GCSqGSIb3DQEBCwUAA4IBAQA07XtaPKSUiO8aEXUHL7P+PPoeUSbrh/Yp +3uDx1MYkCenBz1UbtDDZzhr+BlGmFaQt77JLvyAoJUnRpjZ3NOhk31KxEcdzes05 +nsKtjHEh8lprr988TlWvsoRlFIm5d8sqMb7Po23Pb0iUMkZv53GMoKaEGTcH8gNF +CSuGdXzfX2lXANtu2KZyIktQ1HWYVt+3GP9DQ1CuekR78HlR10M9p9OB0/DJT7na +xpeG0ILD5EJt/rDiZE4OJudANCa1CInXCGNjOCd1HjPqbqjdn5lPdE2BiYBL3ZqX +KVwvvoFBuYz/6n1gBp7N1z3TLqMVvKjmJuVvw9y4AyHqnxbxLFS1 +-----END CERTIFICATE----- + +# Issuer: CN=CA Disig Root R2 O=Disig a.s. +# Subject: CN=CA Disig Root R2 O=Disig a.s. +# Label: "CA Disig Root R2" +# Serial: 10572350602393338211 +# MD5 Fingerprint: 26:01:fb:d8:27:a7:17:9a:45:54:38:1a:43:01:3b:03 +# SHA1 Fingerprint: b5:61:eb:ea:a4:de:e4:25:4b:69:1a:98:a5:57:47:c2:34:c7:d9:71 +# SHA256 Fingerprint: e2:3d:4a:03:6d:7b:70:e9:f5:95:b1:42:20:79:d2:b9:1e:df:bb:1f:b6:51:a0:63:3e:aa:8a:9d:c5:f8:07:03 +-----BEGIN CERTIFICATE----- +MIIFaTCCA1GgAwIBAgIJAJK4iNuwisFjMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV +BAYTAlNLMRMwEQYDVQQHEwpCcmF0aXNsYXZhMRMwEQYDVQQKEwpEaXNpZyBhLnMu +MRkwFwYDVQQDExBDQSBEaXNpZyBSb290IFIyMB4XDTEyMDcxOTA5MTUzMFoXDTQy +MDcxOTA5MTUzMFowUjELMAkGA1UEBhMCU0sxEzARBgNVBAcTCkJyYXRpc2xhdmEx +EzARBgNVBAoTCkRpc2lnIGEucy4xGTAXBgNVBAMTEENBIERpc2lnIFJvb3QgUjIw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCio8QACdaFXS1tFPbCw3Oe +NcJxVX6B+6tGUODBfEl45qt5WDza/3wcn9iXAng+a0EE6UG9vgMsRfYvZNSrXaNH +PWSb6WiaxswbP7q+sos0Ai6YVRn8jG+qX9pMzk0DIaPY0jSTVpbLTAwAFjxfGs3I +x2ymrdMxp7zo5eFm1tL7A7RBZckQrg4FY8aAamkw/dLukO8NJ9+flXP04SXabBbe +QTg06ov80egEFGEtQX6sx3dOy1FU+16SGBsEWmjGycT6txOgmLcRK7fWV8x8nhfR +yyX+hk4kLlYMeE2eARKmK6cBZW58Yh2EhN/qwGu1pSqVg8NTEQxzHQuyRpDRQjrO +QG6Vrf/GlK1ul4SOfW+eioANSW1z4nuSHsPzwfPrLgVv2RvPN3YEyLRa5Beny912 +H9AZdugsBbPWnDTYltxhh5EF5EQIM8HauQhl1K6yNg3ruji6DOWbnuuNZt2Zz9aJ +QfYEkoopKW1rOhzndX0CcQ7zwOe9yxndnWCywmZgtrEE7snmhrmaZkCo5xHtgUUD +i/ZnWejBBhG93c+AAk9lQHhcR1DIm+YfgXvkRKhbhZri3lrVx/k6RGZL5DJUfORs +nLMOPReisjQS1n6yqEm70XooQL6iFh/f5DcfEXP7kAplQ6INfPgGAVUzfbANuPT1 +rqVCV3w2EYx7XsQDnYx5nQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud +DwEB/wQEAwIBBjAdBgNVHQ4EFgQUtZn4r7CU9eMg1gqtzk5WpC5uQu0wDQYJKoZI +hvcNAQELBQADggIBACYGXnDnZTPIgm7ZnBc6G3pmsgH2eDtpXi/q/075KMOYKmFM +tCQSin1tERT3nLXK5ryeJ45MGcipvXrA1zYObYVybqjGom32+nNjf7xueQgcnYqf +GopTpti72TVVsRHFqQOzVju5hJMiXn7B9hJSi+osZ7z+Nkz1uM/Rs0mSO9MpDpkb +lvdhuDvEK7Z4bLQjb/D907JedR+Zlais9trhxTF7+9FGs9K8Z7RiVLoJ92Owk6Ka ++elSLotgEqv89WBW7xBci8QaQtyDW2QOy7W81k/BfDxujRNt+3vrMNDcTa/F1bal +TFtxyegxvug4BkihGuLq0t4SOVga/4AOgnXmt8kHbA7v/zjxmHHEt38OFdAlab0i +nSvtBfZGR6ztwPDUO+Ls7pZbkBNOHlY667DvlruWIxG68kOGdGSVyCh13x01utI3 +gzhTODY7z2zp+WsO0PsE6E9312UBeIYMej4hYvF/Y3EMyZ9E26gnonW+boE+18Dr +G5gPcFw0sorMwIUY6256s/daoQe/qUKS82Ail+QUoQebTnbAjn39pCXHR+3/H3Os +zMOl6W8KjptlwlCFtaOgUxLMVYdh84GuEEZhvUQhuMI9dM9+JDX6HAcOmz0iyu8x +L4ysEr3vQCj8KWefshNPZiTEUxnpHikV7+ZtsH8tZ/3zbBt1RqPlShfppNcL +-----END CERTIFICATE----- + +# Issuer: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV +# Subject: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV +# Label: "ACCVRAIZ1" +# Serial: 6828503384748696800 +# MD5 Fingerprint: d0:a0:5a:ee:05:b6:09:94:21:a1:7d:f1:b2:29:82:02 +# SHA1 Fingerprint: 93:05:7a:88:15:c6:4f:ce:88:2f:fa:91:16:52:28:78:bc:53:64:17 +# SHA256 Fingerprint: 9a:6e:c0:12:e1:a7:da:9d:be:34:19:4d:47:8a:d7:c0:db:18:22:fb:07:1d:f1:29:81:49:6e:d1:04:38:41:13 +-----BEGIN CERTIFICATE----- +MIIH0zCCBbugAwIBAgIIXsO3pkN/pOAwDQYJKoZIhvcNAQEFBQAwQjESMBAGA1UE +AwwJQUNDVlJBSVoxMRAwDgYDVQQLDAdQS0lBQ0NWMQ0wCwYDVQQKDARBQ0NWMQsw +CQYDVQQGEwJFUzAeFw0xMTA1MDUwOTM3MzdaFw0zMDEyMzEwOTM3MzdaMEIxEjAQ +BgNVBAMMCUFDQ1ZSQUlaMTEQMA4GA1UECwwHUEtJQUNDVjENMAsGA1UECgwEQUND +VjELMAkGA1UEBhMCRVMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCb +qau/YUqXry+XZpp0X9DZlv3P4uRm7x8fRzPCRKPfmt4ftVTdFXxpNRFvu8gMjmoY +HtiP2Ra8EEg2XPBjs5BaXCQ316PWywlxufEBcoSwfdtNgM3802/J+Nq2DoLSRYWo +G2ioPej0RGy9ocLLA76MPhMAhN9KSMDjIgro6TenGEyxCQ0jVn8ETdkXhBilyNpA +lHPrzg5XPAOBOp0KoVdDaaxXbXmQeOW1tDvYvEyNKKGno6e6Ak4l0Squ7a4DIrhr +IA8wKFSVf+DuzgpmndFALW4ir50awQUZ0m/A8p/4e7MCQvtQqR0tkw8jq8bBD5L/ +0KIV9VMJcRz/RROE5iZe+OCIHAr8Fraocwa48GOEAqDGWuzndN9wrqODJerWx5eH +k6fGioozl2A3ED6XPm4pFdahD9GILBKfb6qkxkLrQaLjlUPTAYVtjrs78yM2x/47 +4KElB0iryYl0/wiPgL/AlmXz7uxLaL2diMMxs0Dx6M/2OLuc5NF/1OVYm3z61PMO +m3WR5LpSLhl+0fXNWhn8ugb2+1KoS5kE3fj5tItQo05iifCHJPqDQsGH+tUtKSpa +cXpkatcnYGMN285J9Y0fkIkyF/hzQ7jSWpOGYdbhdQrqeWZ2iE9x6wQl1gpaepPl +uUsXQA+xtrn13k/c4LOsOxFwYIRKQ26ZIMApcQrAZQIDAQABo4ICyzCCAscwfQYI +KwYBBQUHAQEEcTBvMEwGCCsGAQUFBzAChkBodHRwOi8vd3d3LmFjY3YuZXMvZmls +ZWFkbWluL0FyY2hpdm9zL2NlcnRpZmljYWRvcy9yYWl6YWNjdjEuY3J0MB8GCCsG +AQUFBzABhhNodHRwOi8vb2NzcC5hY2N2LmVzMB0GA1UdDgQWBBTSh7Tj3zcnk1X2 +VuqB5TbMjB4/vTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNKHtOPfNyeT +VfZW6oHlNsyMHj+9MIIBcwYDVR0gBIIBajCCAWYwggFiBgRVHSAAMIIBWDCCASIG +CCsGAQUFBwICMIIBFB6CARAAQQB1AHQAbwByAGkAZABhAGQAIABkAGUAIABDAGUA +cgB0AGkAZgBpAGMAYQBjAGkA8wBuACAAUgBhAO0AegAgAGQAZQAgAGwAYQAgAEEA +QwBDAFYAIAAoAEEAZwBlAG4AYwBpAGEAIABkAGUAIABUAGUAYwBuAG8AbABvAGcA +7QBhACAAeQAgAEMAZQByAHQAaQBmAGkAYwBhAGMAaQDzAG4AIABFAGwAZQBjAHQA +cgDzAG4AaQBjAGEALAAgAEMASQBGACAAUQA0ADYAMAAxADEANQA2AEUAKQAuACAA +QwBQAFMAIABlAG4AIABoAHQAdABwADoALwAvAHcAdwB3AC4AYQBjAGMAdgAuAGUA +czAwBggrBgEFBQcCARYkaHR0cDovL3d3dy5hY2N2LmVzL2xlZ2lzbGFjaW9uX2Mu +aHRtMFUGA1UdHwROMEwwSqBIoEaGRGh0dHA6Ly93d3cuYWNjdi5lcy9maWxlYWRt +aW4vQXJjaGl2b3MvY2VydGlmaWNhZG9zL3JhaXphY2N2MV9kZXIuY3JsMA4GA1Ud +DwEB/wQEAwIBBjAXBgNVHREEEDAOgQxhY2N2QGFjY3YuZXMwDQYJKoZIhvcNAQEF +BQADggIBAJcxAp/n/UNnSEQU5CmH7UwoZtCPNdpNYbdKl02125DgBS4OxnnQ8pdp +D70ER9m+27Up2pvZrqmZ1dM8MJP1jaGo/AaNRPTKFpV8M9xii6g3+CfYCS0b78gU +JyCpZET/LtZ1qmxNYEAZSUNUY9rizLpm5U9EelvZaoErQNV/+QEnWCzI7UiRfD+m +AM/EKXMRNt6GGT6d7hmKG9Ww7Y49nCrADdg9ZuM8Db3VlFzi4qc1GwQA9j9ajepD +vV+JHanBsMyZ4k0ACtrJJ1vnE5Bc5PUzolVt3OAJTS+xJlsndQAJxGJ3KQhfnlms +tn6tn1QwIgPBHnFk/vk4CpYY3QIUrCPLBhwepH2NDd4nQeit2hW3sCPdK6jT2iWH +7ehVRE2I9DZ+hJp4rPcOVkkO1jMl1oRQQmwgEh0q1b688nCBpHBgvgW1m54ERL5h +I6zppSSMEYCUWqKiuUnSwdzRp+0xESyeGabu4VXhwOrPDYTkF7eifKXeVSUG7szA +h1xA2syVP1XgNce4hL60Xc16gwFy7ofmXx2utYXGJt/mwZrpHgJHnyqobalbz+xF +d3+YJ5oyXSrjhO7FmGYvliAd3djDJ9ew+f7Zfc3Qn48LFFhRny+Lwzgt3uiP1o2H +pPVWQxaZLPSkVrQ0uGE3ycJYgBugl6H8WY3pEfbRD0tVNEYqi4Y7 +-----END CERTIFICATE----- + +# Issuer: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA +# Label: "TWCA Global Root CA" +# Serial: 3262 +# MD5 Fingerprint: f9:03:7e:cf:e6:9e:3c:73:7a:2a:90:07:69:ff:2b:96 +# SHA1 Fingerprint: 9c:bb:48:53:f6:a4:f6:d3:52:a4:e8:32:52:55:60:13:f5:ad:af:65 +# SHA256 Fingerprint: 59:76:90:07:f7:68:5d:0f:cd:50:87:2f:9f:95:d5:75:5a:5b:2b:45:7d:81:f3:69:2b:61:0a:98:67:2f:0e:1b +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgICDL4wDQYJKoZIhvcNAQELBQAwUTELMAkGA1UEBhMCVFcx +EjAQBgNVBAoTCVRBSVdBTi1DQTEQMA4GA1UECxMHUm9vdCBDQTEcMBoGA1UEAxMT +VFdDQSBHbG9iYWwgUm9vdCBDQTAeFw0xMjA2MjcwNjI4MzNaFw0zMDEyMzExNTU5 +NTlaMFExCzAJBgNVBAYTAlRXMRIwEAYDVQQKEwlUQUlXQU4tQ0ExEDAOBgNVBAsT +B1Jvb3QgQ0ExHDAaBgNVBAMTE1RXQ0EgR2xvYmFsIFJvb3QgQ0EwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCwBdvI64zEbooh745NnHEKH1Jw7W2CnJfF +10xORUnLQEK1EjRsGcJ0pDFfhQKX7EMzClPSnIyOt7h52yvVavKOZsTuKwEHktSz +0ALfUPZVr2YOy+BHYC8rMjk1Ujoog/h7FsYYuGLWRyWRzvAZEk2tY/XTP3VfKfCh +MBwqoJimFb3u/Rk28OKRQ4/6ytYQJ0lM793B8YVwm8rqqFpD/G2Gb3PpN0Wp8DbH +zIh1HrtsBv+baz4X7GGqcXzGHaL3SekVtTzWoWH1EfcFbx39Eb7QMAfCKbAJTibc +46KokWofwpFFiFzlmLhxpRUZyXx1EcxwdE8tmx2RRP1WKKD+u4ZqyPpcC1jcxkt2 +yKsi2XMPpfRaAok/T54igu6idFMqPVMnaR1sjjIsZAAmY2E2TqNGtz99sy2sbZCi +laLOz9qC5wc0GZbpuCGqKX6mOL6OKUohZnkfs8O1CWfe1tQHRvMq2uYiN2DLgbYP +oA/pyJV/v1WRBXrPPRXAb94JlAGD1zQbzECl8LibZ9WYkTunhHiVJqRaCPgrdLQA +BDzfuBSO6N+pjWxnkjMdwLfS7JLIvgm/LCkFbwJrnu+8vyq8W8BQj0FwcYeyTbcE +qYSjMq+u7msXi7Kx/mzhkIyIqJdIzshNy/MGz19qCkKxHh53L46g5pIOBvwFItIm +4TFRfTLcDwIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAgEAXzSBdu+WHdXltdkCY4QWwa6gcFGn90xHNcgL +1yg9iXHZqjNB6hQbbCEAwGxCGX6faVsgQt+i0trEfJdLjbDorMjupWkEmQqSpqsn +LhpNgb+E1HAerUf+/UqdM+DyucRFCCEK2mlpc3INvjT+lIutwx4116KD7+U4x6WF +H6vPNOw/KP4M8VeGTslV9xzU2KV9Bnpv1d8Q34FOIWWxtuEXeZVFBs5fzNxGiWNo +RI2T9GRwoD2dKAXDOXC4Ynsg/eTb6QihuJ49CcdP+yz4k3ZB3lLg4VfSnQO8d57+ +nile98FRYB/e2guyLXW3Q0iT5/Z5xoRdgFlglPx4mI88k1HtQJAH32RjJMtOcQWh +15QaiDLxInQirqWm2BJpTGCjAu4r7NRjkgtevi92a6O2JryPA9gK8kxkRr05YuWW +6zRjESjMlfGt7+/cgFhI6Uu46mWs6fyAtbXIRfmswZ/ZuepiiI7E8UuDEq3mi4TW +nsLrgxifarsbJGAzcMzs9zLzXNl5fe+epP7JI8Mk7hWSsT2RTyaGvWZzJBPqpK5j +wa19hAM8EHiGG3njxPPyBJUgriOCxLM6AGK/5jYk4Ve6xx6QddVfP5VhK8E7zeWz +aGHQRiapIVJpLesux+t3zqY6tQMzT3bR51xUAV3LePTJDL/PEo4XLSNolOer/qmy +KwbQBM0= +-----END CERTIFICATE----- + +# Issuer: CN=TeliaSonera Root CA v1 O=TeliaSonera +# Subject: CN=TeliaSonera Root CA v1 O=TeliaSonera +# Label: "TeliaSonera Root CA v1" +# Serial: 199041966741090107964904287217786801558 +# MD5 Fingerprint: 37:41:49:1b:18:56:9a:26:f5:ad:c2:66:fb:40:a5:4c +# SHA1 Fingerprint: 43:13:bb:96:f1:d5:86:9b:c1:4e:6a:92:f6:cf:f6:34:69:87:82:37 +# SHA256 Fingerprint: dd:69:36:fe:21:f8:f0:77:c1:23:a1:a5:21:c1:22:24:f7:22:55:b7:3e:03:a7:26:06:93:e8:a2:4b:0f:a3:89 +-----BEGIN CERTIFICATE----- +MIIFODCCAyCgAwIBAgIRAJW+FqD3LkbxezmCcvqLzZYwDQYJKoZIhvcNAQEFBQAw +NzEUMBIGA1UECgwLVGVsaWFTb25lcmExHzAdBgNVBAMMFlRlbGlhU29uZXJhIFJv +b3QgQ0EgdjEwHhcNMDcxMDE4MTIwMDUwWhcNMzIxMDE4MTIwMDUwWjA3MRQwEgYD +VQQKDAtUZWxpYVNvbmVyYTEfMB0GA1UEAwwWVGVsaWFTb25lcmEgUm9vdCBDQSB2 +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMK+6yfwIaPzaSZVfp3F +VRaRXP3vIb9TgHot0pGMYzHw7CTww6XScnwQbfQ3t+XmfHnqjLWCi65ItqwA3GV1 +7CpNX8GH9SBlK4GoRz6JI5UwFpB/6FcHSOcZrr9FZ7E3GwYq/t75rH2D+1665I+X +Z75Ljo1kB1c4VWk0Nj0TSO9P4tNmHqTPGrdeNjPUtAa9GAH9d4RQAEX1jF3oI7x+ +/jXh7VB7qTCNGdMJjmhnXb88lxhTuylixcpecsHHltTbLaC0H2kD7OriUPEMPPCs +81Mt8Bz17Ww5OXOAFshSsCPN4D7c3TxHoLs1iuKYaIu+5b9y7tL6pe0S7fyYGKkm +dtwoSxAgHNN/Fnct7W+A90m7UwW7XWjH1Mh1Fj+JWov3F0fUTPHSiXk+TT2YqGHe +Oh7S+F4D4MHJHIzTjU3TlTazN19jY5szFPAtJmtTfImMMsJu7D0hADnJoWjiUIMu +sDor8zagrC/kb2HCUQk5PotTubtn2txTuXZZNp1D5SDgPTJghSJRt8czu90VL6R4 +pgd7gUY2BIbdeTXHlSw7sKMXNeVzH7RcWe/a6hBle3rQf5+ztCo3O3CLm1u5K7fs +slESl1MpWtTwEhDcTwK7EpIvYtQ/aUN8Ddb8WHUBiJ1YFkveupD/RwGJBmr2X7KQ +arMCpgKIv7NHfirZ1fpoeDVNAgMBAAGjPzA9MA8GA1UdEwEB/wQFMAMBAf8wCwYD +VR0PBAQDAgEGMB0GA1UdDgQWBBTwj1k4ALP1j5qWDNXr+nuqF+gTEjANBgkqhkiG +9w0BAQUFAAOCAgEAvuRcYk4k9AwI//DTDGjkk0kiP0Qnb7tt3oNmzqjMDfz1mgbl +dxSR651Be5kqhOX//CHBXfDkH1e3damhXwIm/9fH907eT/j3HEbAek9ALCI18Bmx +0GtnLLCo4MBANzX2hFxc469CeP6nyQ1Q6g2EdvZR74NTxnr/DlZJLo961gzmJ1Tj +TQpgcmLNkQfWpb/ImWvtxBnmq0wROMVvMeJuScg/doAmAyYp4Db29iBT4xdwNBed +Y2gea+zDTYa4EzAvXUYNR0PVG6pZDrlcjQZIrXSHX8f8MVRBE+LHIQ6e4B4N4cB7 +Q4WQxYpYxmUKeFfyxiMPAdkgS94P+5KFdSpcc41teyWRyu5FrgZLAMzTsVlQ2jqI +OylDRl6XK1TOU2+NSueW+r9xDkKLfP0ooNBIytrEgUy7onOTJsjrDNYmiLbAJM+7 +vVvrdX3pCI6GMyx5dwlppYn8s3CQh3aP0yK7Qs69cwsgJirQmz1wHiRszYd2qReW +t88NkvuOGKmYSdGe/mBEciG5Ge3C9THxOUiIkCR1VBatzvT4aRRkOfujuLpwQMcn +HL/EVlP6Y2XQ8xwOFvVrhlhNGNTkDY6lnVuR3HYkUD/GKvvZt5y11ubQ2egZixVx +SK236thZiNSQvxaz2emsWWFUyBy6ysHK4bkgTI86k4mloMy/0/Z1pHWWbVY= +-----END CERTIFICATE----- + +# Issuer: CN=E-Tugra Certification Authority O=E-Tuğra EBG Bilişim Teknolojileri ve Hizmetleri A.Ş. OU=E-Tugra Sertifikasyon Merkezi +# Subject: CN=E-Tugra Certification Authority O=E-Tuğra EBG Bilişim Teknolojileri ve Hizmetleri A.Ş. OU=E-Tugra Sertifikasyon Merkezi +# Label: "E-Tugra Certification Authority" +# Serial: 7667447206703254355 +# MD5 Fingerprint: b8:a1:03:63:b0:bd:21:71:70:8a:6f:13:3a:bb:79:49 +# SHA1 Fingerprint: 51:c6:e7:08:49:06:6e:f3:92:d4:5c:a0:0d:6d:a3:62:8f:c3:52:39 +# SHA256 Fingerprint: b0:bf:d5:2b:b0:d7:d9:bd:92:bf:5d:4d:c1:3d:a2:55:c0:2c:54:2f:37:83:65:ea:89:39:11:f5:5e:55:f2:3c +-----BEGIN CERTIFICATE----- +MIIGSzCCBDOgAwIBAgIIamg+nFGby1MwDQYJKoZIhvcNAQELBQAwgbIxCzAJBgNV +BAYTAlRSMQ8wDQYDVQQHDAZBbmthcmExQDA+BgNVBAoMN0UtVHXEn3JhIEVCRyBC +aWxpxZ9pbSBUZWtub2xvamlsZXJpIHZlIEhpem1ldGxlcmkgQS7Fni4xJjAkBgNV +BAsMHUUtVHVncmEgU2VydGlmaWthc3lvbiBNZXJrZXppMSgwJgYDVQQDDB9FLVR1 +Z3JhIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTEzMDMwNTEyMDk0OFoXDTIz +MDMwMzEyMDk0OFowgbIxCzAJBgNVBAYTAlRSMQ8wDQYDVQQHDAZBbmthcmExQDA+ +BgNVBAoMN0UtVHXEn3JhIEVCRyBCaWxpxZ9pbSBUZWtub2xvamlsZXJpIHZlIEhp +em1ldGxlcmkgQS7Fni4xJjAkBgNVBAsMHUUtVHVncmEgU2VydGlmaWthc3lvbiBN +ZXJrZXppMSgwJgYDVQQDDB9FLVR1Z3JhIENlcnRpZmljYXRpb24gQXV0aG9yaXR5 +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4vU/kwVRHoViVF56C/UY +B4Oufq9899SKa6VjQzm5S/fDxmSJPZQuVIBSOTkHS0vdhQd2h8y/L5VMzH2nPbxH +D5hw+IyFHnSOkm0bQNGZDbt1bsipa5rAhDGvykPL6ys06I+XawGb1Q5KCKpbknSF +Q9OArqGIW66z6l7LFpp3RMih9lRozt6Plyu6W0ACDGQXwLWTzeHxE2bODHnv0ZEo +q1+gElIwcxmOj+GMB6LDu0rw6h8VqO4lzKRG+Bsi77MOQ7osJLjFLFzUHPhdZL3D +k14opz8n8Y4e0ypQBaNV2cvnOVPAmJ6MVGKLJrD3fY185MaeZkJVgkfnsliNZvcH +fC425lAcP9tDJMW/hkd5s3kc91r0E+xs+D/iWR+V7kI+ua2oMoVJl0b+SzGPWsut +dEcf6ZG33ygEIqDUD13ieU/qbIWGvaimzuT6w+Gzrt48Ue7LE3wBf4QOXVGUnhMM +ti6lTPk5cDZvlsouDERVxcr6XQKj39ZkjFqzAQqptQpHF//vkUAqjqFGOjGY5RH8 +zLtJVor8udBhmm9lbObDyz51Sf6Pp+KJxWfXnUYTTjF2OySznhFlhqt/7x3U+Lzn +rFpct1pHXFXOVbQicVtbC/DP3KBhZOqp12gKY6fgDT+gr9Oq0n7vUaDmUStVkhUX +U8u3Zg5mTPj5dUyQ5xJwx0UCAwEAAaNjMGEwHQYDVR0OBBYEFC7j27JJ0JxUeVz6 +Jyr+zE7S6E5UMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAULuPbsknQnFR5 +XPonKv7MTtLoTlQwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAF +Nzr0TbdF4kV1JI+2d1LoHNgQk2Xz8lkGpD4eKexd0dCrfOAKkEh47U6YA5n+KGCR +HTAduGN8qOY1tfrTYXbm1gdLymmasoR6d5NFFxWfJNCYExL/u6Au/U5Mh/jOXKqY +GwXgAEZKgoClM4so3O0409/lPun++1ndYYRP0lSWE2ETPo+Aab6TR7U1Q9Jauz1c +77NCR807VRMGsAnb/WP2OogKmW9+4c4bU2pEZiNRCHu8W1Ki/QY3OEBhj0qWuJA3 ++GbHeJAAFS6LrVE1Uweoa2iu+U48BybNCAVwzDk/dr2l02cmAYamU9JgO3xDf1WK +vJUawSg5TB9D0pH0clmKuVb8P7Sd2nCcdlqMQ1DujjByTd//SffGqWfZbawCEeI6 +FiWnWAjLb1NBnEg4R2gz0dfHj9R0IdTDBZB6/86WiLEVKV0jq9BgoRJP3vQXzTLl +yb/IQ639Lo7xr+L0mPoSHyDYwKcMhcWQ9DstliaxLL5Mq+ux0orJ23gTDx4JnW2P +AJ8C2sH6H3p6CcRK5ogql5+Ji/03X186zjhZhkuvcQu02PJwT58yE+Owp1fl2tpD +y4Q08ijE6m30Ku/Ba3ba+367hTzSU8JNvnHhRdH9I2cNE3X7z2VnIp2usAnRCf8d +NL/+I5c30jn6PQ0GC7TbO6Orb1wdtn7os4I07QZcJA== +-----END CERTIFICATE----- + +# Issuer: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Subject: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Label: "T-TeleSec GlobalRoot Class 2" +# Serial: 1 +# MD5 Fingerprint: 2b:9b:9e:e4:7b:6c:1f:00:72:1a:cc:c1:77:79:df:6a +# SHA1 Fingerprint: 59:0d:2d:7d:88:4f:40:2e:61:7e:a5:62:32:17:65:cf:17:d8:94:e9 +# SHA256 Fingerprint: 91:e2:f5:78:8d:58:10:eb:a7:ba:58:73:7d:e1:54:8a:8e:ca:cd:01:45:98:bc:0b:14:3e:04:1b:17:05:25:52 +-----BEGIN CERTIFICATE----- +MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx +KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd +BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl +YyBHbG9iYWxSb290IENsYXNzIDIwHhcNMDgxMDAxMTA0MDE0WhcNMzMxMDAxMjM1 +OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy +aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 +ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDIwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqX9obX+hzkeXaXPSi5kfl82hVYAUd +AqSzm1nzHoqvNK38DcLZSBnuaY/JIPwhqgcZ7bBcrGXHX+0CfHt8LRvWurmAwhiC +FoT6ZrAIxlQjgeTNuUk/9k9uN0goOA/FvudocP05l03Sx5iRUKrERLMjfTlH6VJi +1hKTXrcxlkIF+3anHqP1wvzpesVsqXFP6st4vGCvx9702cu+fjOlbpSD8DT6Iavq +jnKgP6TeMFvvhk1qlVtDRKgQFRzlAVfFmPHmBiiRqiDFt1MmUUOyCxGVWOHAD3bZ +wI18gfNycJ5v/hqO2V81xrJvNHy+SE/iWjnX2J14np+GPgNeGYtEotXHAgMBAAGj +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS/ +WSA2AHmgoCJrjNXyYdK4LMuCSjANBgkqhkiG9w0BAQsFAAOCAQEAMQOiYQsfdOhy +NsZt+U2e+iKo4YFWz827n+qrkRk4r6p8FU3ztqONpfSO9kSpp+ghla0+AGIWiPAC +uvxhI+YzmzB6azZie60EI4RYZeLbK4rnJVM3YlNfvNoBYimipidx5joifsFvHZVw +IEoHNN/q/xWA5brXethbdXwFeilHfkCoMRN3zUA7tFFHei4R40cR3p1m0IvVVGb6 +g1XqfMIpiRvpb7PO4gWEyS8+eIVibslfwXhjdFjASBgMmTnrpMwatXlajRWc2BQN +9noHV8cigwUtPJslJj0Ys6lDfMjIq2SPDqO/nBudMNva0Bkuqjzx+zOAduTNrRlP +BSeOE6Fuwg== +-----END CERTIFICATE----- + +# Issuer: CN=Atos TrustedRoot 2011 O=Atos +# Subject: CN=Atos TrustedRoot 2011 O=Atos +# Label: "Atos TrustedRoot 2011" +# Serial: 6643877497813316402 +# MD5 Fingerprint: ae:b9:c4:32:4b:ac:7f:5d:66:cc:77:94:bb:2a:77:56 +# SHA1 Fingerprint: 2b:b1:f5:3e:55:0c:1d:c5:f1:d4:e6:b7:6a:46:4b:55:06:02:ac:21 +# SHA256 Fingerprint: f3:56:be:a2:44:b7:a9:1e:b3:5d:53:ca:9a:d7:86:4a:ce:01:8e:2d:35:d5:f8:f9:6d:df:68:a6:f4:1a:a4:74 +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 1 G3" +# Serial: 687049649626669250736271037606554624078720034195 +# MD5 Fingerprint: a4:bc:5b:3f:fe:37:9a:fa:64:f0:e2:fa:05:3d:0b:ab +# SHA1 Fingerprint: 1b:8e:ea:57:96:29:1a:c9:39:ea:b8:0a:81:1a:73:73:c0:93:79:67 +# SHA256 Fingerprint: 8a:86:6f:d1:b2:76:b5:7e:57:8e:92:1c:65:82:8a:2b:ed:58:e9:f2:f2:88:05:41:34:b7:f1:f4:bf:c9:cc:74 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIUeFhfLq0sGUvjNwc1NBMotZbUZZMwDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMSBHMzAeFw0xMjAxMTIxNzI3NDRaFw00 +MjAxMTIxNzI3NDRaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDEgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCgvlAQjunybEC0BJyFuTHK3C3kEakEPBtV +wedYMB0ktMPvhd6MLOHBPd+C5k+tR4ds7FtJwUrVu4/sh6x/gpqG7D0DmVIB0jWe +rNrwU8lmPNSsAgHaJNM7qAJGr6Qc4/hzWHa39g6QDbXwz8z6+cZM5cOGMAqNF341 +68Xfuw6cwI2H44g4hWf6Pser4BOcBRiYz5P1sZK0/CPTz9XEJ0ngnjybCKOLXSoh +4Pw5qlPafX7PGglTvF0FBM+hSo+LdoINofjSxxR3W5A2B4GbPgb6Ul5jxaYA/qXp +UhtStZI5cgMJYr2wYBZupt0lwgNm3fME0UDiTouG9G/lg6AnhF4EwfWQvTA9xO+o +abw4m6SkltFi2mnAAZauy8RRNOoMqv8hjlmPSlzkYZqn0ukqeI1RPToV7qJZjqlc +3sX5kCLliEVx3ZGZbHqfPT2YfF72vhZooF6uCyP8Wg+qInYtyaEQHeTTRCOQiJ/G +KubX9ZqzWB4vMIkIG1SitZgj7Ah3HJVdYdHLiZxfokqRmu8hqkkWCKi9YSgxyXSt +hfbZxbGL0eUQMk1fiyA6PEkfM4VZDdvLCXVDaXP7a3F98N/ETH3Goy7IlXnLc6KO +Tk0k+17kBL5yG6YnLUlamXrXXAkgt3+UuU/xDRxeiEIbEbfnkduebPRq34wGmAOt +zCjvpUfzUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQUo5fW816iEOGrRZ88F2Q87gFwnMwwDQYJKoZIhvcNAQELBQAD +ggIBABj6W3X8PnrHX3fHyt/PX8MSxEBd1DKquGrX1RUVRpgjpeaQWxiZTOOtQqOC +MTaIzen7xASWSIsBx40Bz1szBpZGZnQdT+3Btrm0DWHMY37XLneMlhwqI2hrhVd2 +cDMT/uFPpiN3GPoajOi9ZcnPP/TJF9zrx7zABC4tRi9pZsMbj/7sPtPKlL92CiUN +qXsCHKnQO18LwIE6PWThv6ctTr1NxNgpxiIY0MWscgKCP6o6ojoilzHdCGPDdRS5 +YCgtW2jgFqlmgiNR9etT2DGbe+m3nUvriBbP+V04ikkwj+3x6xn0dxoxGE1nVGwv +b2X52z3sIexe9PSLymBlVNFxZPT5pqOBMzYzcfCkeF9OrYMh3jRJjehZrJ3ydlo2 +8hP0r+AJx2EqbPfgna67hkooby7utHnNkDPDs3b69fBsnQGQ+p6Q9pxyz0fawx/k +NSBT8lTR32GDpgLiJTjehTItXnOQUl1CxM49S+H5GYQd1aJQzEH7QRTDvdbJWqNj +ZgKAvQU6O0ec7AAmTPWIUb+oI38YB7AL7YsmoWTTYUrrXJ/es69nA7Mf3W1daWhp +q1467HxpvMc7hU6eFbm0FU/DlXpY18ls6Wy58yljXrQs8C097Vpl4KlbQMJImYFt +nh8GKjwStIsPm6Ik8KaN1nrgS7ZklmOVhMJKzRwuJIczYOXD +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 2 G3" +# Serial: 390156079458959257446133169266079962026824725800 +# MD5 Fingerprint: af:0c:86:6e:bf:40:2d:7f:0b:3e:12:50:ba:12:3d:06 +# SHA1 Fingerprint: 09:3c:61:f3:8b:8b:dc:7d:55:df:75:38:02:05:00:e1:25:f5:c8:36 +# SHA256 Fingerprint: 8f:e4:fb:0a:f9:3a:4d:0d:67:db:0b:eb:b2:3e:37:c7:1b:f3:25:dc:bc:dd:24:0e:a0:4d:af:58:b4:7e:18:40 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIURFc0JFuBiZs18s64KztbpybwdSgwDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMiBHMzAeFw0xMjAxMTIxODU5MzJaFw00 +MjAxMTIxODU5MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDIgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQChriWyARjcV4g/Ruv5r+LrI3HimtFhZiFf +qq8nUeVuGxbULX1QsFN3vXg6YOJkApt8hpvWGo6t/x8Vf9WVHhLL5hSEBMHfNrMW +n4rjyduYNM7YMxcoRvynyfDStNVNCXJJ+fKH46nafaF9a7I6JaltUkSs+L5u+9ym +c5GQYaYDFCDy54ejiK2toIz/pgslUiXnFgHVy7g1gQyjO/Dh4fxaXc6AcW34Sas+ +O7q414AB+6XrW7PFXmAqMaCvN+ggOp+oMiwMzAkd056OXbxMmO7FGmh77FOm6RQ1 +o9/NgJ8MSPsc9PG/Srj61YxxSscfrf5BmrODXfKEVu+lV0POKa2Mq1W/xPtbAd0j +IaFYAI7D0GoT7RPjEiuA3GfmlbLNHiJuKvhB1PLKFAeNilUSxmn1uIZoL1NesNKq +IcGY5jDjZ1XHm26sGahVpkUG0CM62+tlXSoREfA7T8pt9DTEceT/AFr2XK4jYIVz +8eQQsSWu1ZK7E8EM4DnatDlXtas1qnIhO4M15zHfeiFuuDIIfR0ykRVKYnLP43eh +vNURG3YBZwjgQQvD6xVu+KQZ2aKrr+InUlYrAoosFCT5v0ICvybIxo/gbjh9Uy3l +7ZizlWNof/k19N+IxWA1ksB8aRxhlRbQ694Lrz4EEEVlWFA4r0jyWbYW8jwNkALG +cC4BrTwV1wIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQU7edvdlq/YOxJW8ald7tyFnGbxD0wDQYJKoZIhvcNAQELBQAD +ggIBAJHfgD9DCX5xwvfrs4iP4VGyvD11+ShdyLyZm3tdquXK4Qr36LLTn91nMX66 +AarHakE7kNQIXLJgapDwyM4DYvmL7ftuKtwGTTwpD4kWilhMSA/ohGHqPHKmd+RC +roijQ1h5fq7KpVMNqT1wvSAZYaRsOPxDMuHBR//47PERIjKWnML2W2mWeyAMQ0Ga +W/ZZGYjeVYg3UQt4XAoeo0L9x52ID8DyeAIkVJOviYeIyUqAHerQbj5hLja7NQ4n +lv1mNDthcnPxFlxHBlRJAHpYErAK74X9sbgzdWqTHBLmYF5vHX/JHyPLhGGfHoJE ++V+tYlUkmlKY7VHnoX6XOuYvHxHaU4AshZ6rNRDbIl9qxV6XU/IyAgkwo1jwDQHV +csaxfGl7w/U2Rcxhbl5MlMVerugOXou/983g7aEOGzPuVBj+D77vfoRrQ+NwmNtd +dbINWQeFFSM51vHfqSYP1kjHs6Yi9TM3WpVHn3u6GBVv/9YUZINJ0gpnIdsPNWNg +KCLjsZWDzYWm3S8P52dSbrsvhXz1SnPnxT7AvSESBT/8twNJAlvIJebiVDj1eYeM +HVOyToV7BjjHLPj4sHKNJeV3UvQDHEimUF+IIDBu8oJDqz2XhOdT+yHBTw8imoa4 +WSr2Rz0ZiC3oheGe7IUIarFsNMkd7EgrO3jtZsSOeWmD3n+M +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 3 G3" +# Serial: 268090761170461462463995952157327242137089239581 +# MD5 Fingerprint: df:7d:b9:ad:54:6f:68:a1:df:89:57:03:97:43:b0:d7 +# SHA1 Fingerprint: 48:12:bd:92:3c:a8:c4:39:06:e7:30:6d:27:96:e6:a4:cf:22:2e:7d +# SHA256 Fingerprint: 88:ef:81:de:20:2e:b0:18:45:2e:43:f8:64:72:5c:ea:5f:bd:1f:c2:d9:d2:05:73:07:09:c5:d8:b8:69:0f:46 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIULvWbAiin23r/1aOp7r0DoM8Sah0wDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMyBHMzAeFw0xMjAxMTIyMDI2MzJaFw00 +MjAxMTIyMDI2MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDMgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCzyw4QZ47qFJenMioKVjZ/aEzHs286IxSR +/xl/pcqs7rN2nXrpixurazHb+gtTTK/FpRp5PIpM/6zfJd5O2YIyC0TeytuMrKNu +FoM7pmRLMon7FhY4futD4tN0SsJiCnMK3UmzV9KwCoWdcTzeo8vAMvMBOSBDGzXR +U7Ox7sWTaYI+FrUoRqHe6okJ7UO4BUaKhvVZR74bbwEhELn9qdIoyhA5CcoTNs+c +ra1AdHkrAj80//ogaX3T7mH1urPnMNA3I4ZyYUUpSFlob3emLoG+B01vr87ERROR +FHAGjx+f+IdpsQ7vw4kZ6+ocYfx6bIrc1gMLnia6Et3UVDmrJqMz6nWB2i3ND0/k +A9HvFZcba5DFApCTZgIhsUfei5pKgLlVj7WiL8DWM2fafsSntARE60f75li59wzw +eyuxwHApw0BiLTtIadwjPEjrewl5qW3aqDCYz4ByA4imW0aucnl8CAMhZa634Ryl +sSqiMd5mBPfAdOhx3v89WcyWJhKLhZVXGqtrdQtEPREoPHtht+KPZ0/l7DxMYIBp +VzgeAVuNVejH38DMdyM0SXV89pgR6y3e7UEuFAUCf+D+IOs15xGsIs5XPd7JMG0Q +A4XN8f+MFrXBsj6IbGB/kE+V9/YtrQE5BwT6dYB9v0lQ7e/JxHwc64B+27bQ3RP+ +ydOc17KXqQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQUxhfQvKjqAkPyGwaZXSuQILnXnOQwDQYJKoZIhvcNAQELBQAD +ggIBADRh2Va1EodVTd2jNTFGu6QHcrxfYWLopfsLN7E8trP6KZ1/AvWkyaiTt3px +KGmPc+FSkNrVvjrlt3ZqVoAh313m6Tqe5T72omnHKgqwGEfcIHB9UqM+WXzBusnI +FUBhynLWcKzSt/Ac5IYp8M7vaGPQtSCKFWGafoaYtMnCdvvMujAWzKNhxnQT5Wvv +oxXqA/4Ti2Tk08HS6IT7SdEQTXlm66r99I0xHnAUrdzeZxNMgRVhvLfZkXdxGYFg +u/BYpbWcC/ePIlUnwEsBbTuZDdQdm2NnL9DuDcpmvJRPpq3t/O5jrFc/ZSXPsoaP +0Aj/uHYUbt7lJ+yreLVTubY/6CD50qi+YUbKh4yE8/nxoGibIh6BJpsQBJFxwAYf +3KDTuVan45gtf4Od34wrnDKOMpTwATwiKp9Dwi7DmDkHOHv8XgBCH/MyJnmDhPbl +8MFREsALHgQjDFSlTC9JxUrRtm5gDWv8a4uFJGS3iQ6rJUdbPM9+Sb3H6QrG2vd+ +DhcI00iX0HGS8A85PjRqHH3Y8iKuu2n0M7SmSFXRDw4m6Oy2Cy2nhTXN/VnIn9HN +PlopNLk9hM6xZdRZkZFWdSHBd575euFgndOtBBj0fOtek49TSiIp+EgrPk2GrFt/ +ywaZWWDYWGWVjUTR939+J399roD1B0y2PpxxVJkES/1Y+Zj0 +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G2" +# Serial: 15385348160840213938643033620894905419 +# MD5 Fingerprint: 92:38:b9:f8:63:24:82:65:2c:57:33:e6:fe:81:8f:9d +# SHA1 Fingerprint: a1:4b:48:d9:43:ee:0a:0e:40:90:4f:3c:e0:a4:c0:91:93:51:5d:3f +# SHA256 Fingerprint: 7d:05:eb:b6:82:33:9f:8c:94:51:ee:09:4e:eb:fe:fa:79:53:a1:14:ed:b2:f4:49:49:45:2f:ab:7d:2f:c1:85 +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA +n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc +biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp +EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA +bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu +YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB +AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW +BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI +QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I +0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni +lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 +B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv +ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo +IhNzbM8m9Yop5w== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G3" +# Serial: 15459312981008553731928384953135426796 +# MD5 Fingerprint: 7c:7f:65:31:0c:81:df:8d:ba:3e:99:e2:5c:ad:6e:fb +# SHA1 Fingerprint: f5:17:a2:4f:9a:48:c6:c9:f8:a2:00:26:9f:dc:0f:48:2c:ab:30:89 +# SHA256 Fingerprint: 7e:37:cb:8b:4c:47:09:0c:ab:36:55:1b:a6:f4:5d:b8:40:68:0f:ba:16:6a:95:2d:b1:00:71:7f:43:05:3f:c2 +-----BEGIN CERTIFICATE----- +MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg +RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf +Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q +RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD +AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY +JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv +6pZjamVFkpUBtA== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G2" +# Serial: 4293743540046975378534879503202253541 +# MD5 Fingerprint: e4:a6:8a:c8:54:ac:52:42:46:0a:fd:72:48:1b:2a:44 +# SHA1 Fingerprint: df:3c:24:f9:bf:d6:66:76:1b:26:80:73:fe:06:d1:cc:8d:4f:82:a4 +# SHA256 Fingerprint: cb:3c:cb:b7:60:31:e5:e0:13:8f:8d:d3:9a:23:f9:de:47:ff:c3:5e:43:c1:14:4c:ea:27:d4:6a:5a:b1:cb:5f +-----BEGIN CERTIFICATE----- +MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH +MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI +2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx +1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ +q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz +tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ +vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV +5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY +1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 +NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG +Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 +8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe +pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl +MrY= +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G3" +# Serial: 7089244469030293291760083333884364146 +# MD5 Fingerprint: f5:5d:a4:50:a5:fb:28:7e:1e:0f:0d:cc:96:57:56:ca +# SHA1 Fingerprint: 7e:04:de:89:6a:3e:66:6d:00:e6:87:d3:3f:fa:d9:3b:e8:3d:34:9e +# SHA256 Fingerprint: 31:ad:66:48:f8:10:41:38:c7:38:f3:9e:a4:32:01:33:39:3e:3a:18:cc:02:29:6e:f9:7c:2a:c9:ef:67:31:d0 +-----BEGIN CERTIFICATE----- +MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe +Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw +EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x +IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF +K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG +fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO +Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd +BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx +AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ +oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 +sycX +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Trusted Root G4" +# Serial: 7451500558977370777930084869016614236 +# MD5 Fingerprint: 78:f2:fc:aa:60:1f:2f:b4:eb:c9:37:ba:53:2e:75:49 +# SHA1 Fingerprint: dd:fb:16:cd:49:31:c9:73:a2:03:7d:3f:c8:3a:4d:7d:77:5d:05:e4 +# SHA256 Fingerprint: 55:2f:7b:dc:f1:a7:af:9e:6c:e6:72:01:7f:4f:12:ab:f7:72:40:c7:8e:76:1a:c2:03:d1:d9:d2:0a:c8:99:88 +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ +-----END CERTIFICATE----- + +# Issuer: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Label: "COMODO RSA Certification Authority" +# Serial: 101909084537582093308941363524873193117 +# MD5 Fingerprint: 1b:31:b0:71:40:36:cc:14:36:91:ad:c4:3e:fd:ec:18 +# SHA1 Fingerprint: af:e5:d2:44:a8:d1:19:42:30:ff:47:9f:e2:f8:97:bb:cd:7a:8c:b4 +# SHA256 Fingerprint: 52:f0:e1:c4:e5:8e:c6:29:29:1b:60:31:7f:07:46:71:b8:5d:7e:a8:0d:5b:07:27:34:63:53:4b:32:b4:02:34 +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR +6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X +pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC +9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV +/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf +Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z ++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w +qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah +SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC +u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf +Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq +crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl +wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM +4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV +2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna +FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ +CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK +boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke +jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL +S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb +QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl +0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB +NVOFBkpdn627G190 +-----END CERTIFICATE----- + +# Issuer: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Label: "USERTrust RSA Certification Authority" +# Serial: 2645093764781058787591871645665788717 +# MD5 Fingerprint: 1b:fe:69:d1:91:b7:19:33:a3:72:a8:0f:e1:55:e5:b5 +# SHA1 Fingerprint: 2b:8f:1b:57:33:0d:bb:a2:d0:7a:6c:51:f7:0e:e9:0d:da:b9:ad:8e +# SHA256 Fingerprint: e7:93:c9:b0:2f:d8:aa:13:e2:1c:31:22:8a:cc:b0:81:19:64:3b:74:9c:89:89:64:b1:74:6d:46:c3:d4:cb:d2 +-----BEGIN CERTIFICATE----- +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 +-----END CERTIFICATE----- + +# Issuer: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Label: "USERTrust ECC Certification Authority" +# Serial: 123013823720199481456569720443997572134 +# MD5 Fingerprint: fa:68:bc:d9:b5:7f:ad:fd:c9:1d:06:83:28:cc:24:c1 +# SHA1 Fingerprint: d1:cb:ca:5d:b2:d5:2a:7f:69:3b:67:4d:e5:f0:5a:1d:0c:95:7d:f0 +# SHA256 Fingerprint: 4f:f4:60:d5:4b:9c:86:da:bf:bc:fc:57:12:e0:40:0d:2b:ed:3f:bc:4d:4f:bd:aa:86:e0:6a:dc:d2:a9:ad:7a +-----BEGIN CERTIFICATE----- +MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT +Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg +VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo +I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng +o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G +A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB +zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW +RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Label: "GlobalSign ECC Root CA - R4" +# Serial: 14367148294922964480859022125800977897474 +# MD5 Fingerprint: 20:f0:27:68:d1:7e:a0:9d:0e:e6:2a:ca:df:5c:89:8e +# SHA1 Fingerprint: 69:69:56:2e:40:80:f4:24:a1:e7:19:9f:14:ba:f3:ee:58:ab:6a:bb +# SHA256 Fingerprint: be:c9:49:11:c2:95:56:76:db:6c:0a:55:09:86:d7:6e:3b:a0:05:66:7c:44:2c:97:62:b4:fb:b7:73:de:22:8c +-----BEGIN CERTIFICATE----- +MIIB4TCCAYegAwIBAgIRKjikHJYKBN5CsiilC+g0mAIwCgYIKoZIzj0EAwIwUDEk +MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpH +bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX +DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD +QSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuMZ5049sJQ6fLjkZHAOkrprlOQcJ +FspjsbmG+IpXwVfOQvpzofdlQv8ewQCybnMO/8ch5RikqtlxP6jUuc6MHaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFSwe61F +uOJAf/sKbvu+M8k8o4TVMAoGCCqGSM49BAMCA0gAMEUCIQDckqGgE6bPA7DmxCGX +kPoUVy0D7O48027KqGx2vKLeuwIgJ6iFJzWbVsaj8kfSt24bAgAXqmemFZHe+pTs +ewv4n4Q= +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Label: "GlobalSign ECC Root CA - R5" +# Serial: 32785792099990507226680698011560947931244 +# MD5 Fingerprint: 9f:ad:3b:1c:02:1e:8a:ba:17:74:38:81:0c:a2:bc:08 +# SHA1 Fingerprint: 1f:24:c6:30:cd:a4:18:ef:20:69:ff:ad:4f:dd:5f:46:3a:1b:69:aa +# SHA256 Fingerprint: 17:9f:bc:14:8a:3d:d0:0f:d2:4e:a1:34:58:cc:43:bf:a7:f5:9c:81:82:d7:83:a5:13:f6:eb:ec:10:0c:89:24 +-----BEGIN CERTIFICATE----- +MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk +MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH +bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX +DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD +QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu +MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc +8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke +hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI +KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg +515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO +xwy8p2Fp8fc74SrL+SvzZpA3 +-----END CERTIFICATE----- + +# Issuer: CN=Staat der Nederlanden Root CA - G3 O=Staat der Nederlanden +# Subject: CN=Staat der Nederlanden Root CA - G3 O=Staat der Nederlanden +# Label: "Staat der Nederlanden Root CA - G3" +# Serial: 10003001 +# MD5 Fingerprint: 0b:46:67:07:db:10:2f:19:8c:35:50:60:d1:0b:f4:37 +# SHA1 Fingerprint: d8:eb:6b:41:51:92:59:e0:f3:e7:85:00:c0:3d:b6:88:97:c9:ee:fc +# SHA256 Fingerprint: 3c:4f:b0:b9:5a:b8:b3:00:32:f4:32:b8:6f:53:5f:e1:72:c1:85:d0:fd:39:86:58:37:cf:36:18:7f:a6:f4:28 +-----BEGIN CERTIFICATE----- +MIIFdDCCA1ygAwIBAgIEAJiiOTANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJO +TDEeMBwGA1UECgwVU3RhYXQgZGVyIE5lZGVybGFuZGVuMSswKQYDVQQDDCJTdGFh +dCBkZXIgTmVkZXJsYW5kZW4gUm9vdCBDQSAtIEczMB4XDTEzMTExNDExMjg0MloX +DTI4MTExMzIzMDAwMFowWjELMAkGA1UEBhMCTkwxHjAcBgNVBAoMFVN0YWF0IGRl +ciBOZWRlcmxhbmRlbjErMCkGA1UEAwwiU3RhYXQgZGVyIE5lZGVybGFuZGVuIFJv +b3QgQ0EgLSBHMzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL4yolQP +cPssXFnrbMSkUeiFKrPMSjTysF/zDsccPVMeiAho2G89rcKezIJnByeHaHE6n3WW +IkYFsO2tx1ueKt6c/DrGlaf1F2cY5y9JCAxcz+bMNO14+1Cx3Gsy8KL+tjzk7FqX +xz8ecAgwoNzFs21v0IJyEavSgWhZghe3eJJg+szeP4TrjTgzkApyI/o1zCZxMdFy +KJLZWyNtZrVtB0LrpjPOktvA9mxjeM3KTj215VKb8b475lRgsGYeCasH/lSJEULR +9yS6YHgamPfJEf0WwTUaVHXvQ9Plrk7O53vDxk5hUUurmkVLoR9BvUhTFXFkC4az +5S6+zqQbwSmEorXLCCN2QyIkHxcE1G6cxvx/K2Ya7Irl1s9N9WMJtxU51nus6+N8 +6U78dULI7ViVDAZCopz35HCz33JvWjdAidiFpNfxC95DGdRKWCyMijmev4SH8RY7 +Ngzp07TKbBlBUgmhHbBqv4LvcFEhMtwFdozL92TkA1CvjJFnq8Xy7ljY3r735zHP +bMk7ccHViLVlvMDoFxcHErVc0qsgk7TmgoNwNsXNo42ti+yjwUOH5kPiNL6VizXt +BznaqB16nzaeErAMZRKQFWDZJkBE41ZgpRDUajz9QdwOWke275dhdU/Z/seyHdTt +XUmzqWrLZoQT1Vyg3N9udwbRcXXIV2+vD3dbAgMBAAGjQjBAMA8GA1UdEwEB/wQF +MAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRUrfrHkleuyjWcLhL75Lpd +INyUVzANBgkqhkiG9w0BAQsFAAOCAgEAMJmdBTLIXg47mAE6iqTnB/d6+Oea31BD +U5cqPco8R5gu4RV78ZLzYdqQJRZlwJ9UXQ4DO1t3ApyEtg2YXzTdO2PCwyiBwpwp +LiniyMMB8jPqKqrMCQj3ZWfGzd/TtiunvczRDnBfuCPRy5FOCvTIeuXZYzbB1N/8 +Ipf3YF3qKS9Ysr1YvY2WTxB1v0h7PVGHoTx0IsL8B3+A3MSs/mrBcDCw6Y5p4ixp +gZQJut3+TcCDjJRYwEYgr5wfAvg1VUkvRtTA8KCWAg8zxXHzniN9lLf9OtMJgwYh +/WA9rjLA0u6NpvDntIJ8CsxwyXmA+P5M9zWEGYox+wrZ13+b8KKaa8MFSu1BYBQw +0aoRQm7TIwIEC8Zl3d1Sd9qBa7Ko+gE4uZbqKmxnl4mUnrzhVNXkanjvSr0rmj1A +fsbAddJu+2gw7OyLnflJNZoaLNmzlTnVHpL3prllL+U9bTpITAjc5CgSKL59NVzq +4BZ+Extq1z7XnvwtdbLBFNUjA9tbbws+eC8N3jONFrdI54OagQ97wUNNVQQXOEpR +1VmiiXTTn74eS9fGbbeIJG9gkaSChVtWQbzQRKtqE77RLFi3EjNYsjdj3BP1lB0/ +QFH1T/U67cjF68IeHRaVesd+QnGTbksVtzDfqu1XhUisHWrdOWnk4Xl4vs4Fv6EM +94B7IWcnMFk= +-----END CERTIFICATE----- + +# Issuer: CN=Staat der Nederlanden EV Root CA O=Staat der Nederlanden +# Subject: CN=Staat der Nederlanden EV Root CA O=Staat der Nederlanden +# Label: "Staat der Nederlanden EV Root CA" +# Serial: 10000013 +# MD5 Fingerprint: fc:06:af:7b:e8:1a:f1:9a:b4:e8:d2:70:1f:c0:f5:ba +# SHA1 Fingerprint: 76:e2:7e:c1:4f:db:82:c1:c0:a6:75:b5:05:be:3d:29:b4:ed:db:bb +# SHA256 Fingerprint: 4d:24:91:41:4c:fe:95:67:46:ec:4c:ef:a6:cf:6f:72:e2:8a:13:29:43:2f:9d:8a:90:7a:c4:cb:5d:ad:c1:5a +-----BEGIN CERTIFICATE----- +MIIFcDCCA1igAwIBAgIEAJiWjTANBgkqhkiG9w0BAQsFADBYMQswCQYDVQQGEwJO +TDEeMBwGA1UECgwVU3RhYXQgZGVyIE5lZGVybGFuZGVuMSkwJwYDVQQDDCBTdGFh +dCBkZXIgTmVkZXJsYW5kZW4gRVYgUm9vdCBDQTAeFw0xMDEyMDgxMTE5MjlaFw0y +MjEyMDgxMTEwMjhaMFgxCzAJBgNVBAYTAk5MMR4wHAYDVQQKDBVTdGFhdCBkZXIg +TmVkZXJsYW5kZW4xKTAnBgNVBAMMIFN0YWF0IGRlciBOZWRlcmxhbmRlbiBFViBS +b290IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA48d+ifkkSzrS +M4M1LGns3Amk41GoJSt5uAg94JG6hIXGhaTK5skuU6TJJB79VWZxXSzFYGgEt9nC +UiY4iKTWO0Cmws0/zZiTs1QUWJZV1VD+hq2kY39ch/aO5ieSZxeSAgMs3NZmdO3d +Z//BYY1jTw+bbRcwJu+r0h8QoPnFfxZpgQNH7R5ojXKhTbImxrpsX23Wr9GxE46p +rfNeaXUmGD5BKyF/7otdBwadQ8QpCiv8Kj6GyzyDOvnJDdrFmeK8eEEzduG/L13l +pJhQDBXd4Pqcfzho0LKmeqfRMb1+ilgnQ7O6M5HTp5gVXJrm0w912fxBmJc+qiXb +j5IusHsMX/FjqTf5m3VpTCgmJdrV8hJwRVXj33NeN/UhbJCONVrJ0yPr08C+eKxC +KFhmpUZtcALXEPlLVPxdhkqHz3/KRawRWrUgUY0viEeXOcDPusBCAUCZSCELa6fS +/ZbV0b5GnUngC6agIk440ME8MLxwjyx1zNDFjFE7PZQIZCZhfbnDZY8UnCHQqv0X +cgOPvZuM5l5Tnrmd74K74bzickFbIZTTRTeU0d8JOV3nI6qaHcptqAqGhYqCvkIH +1vI4gnPah1vlPNOePqc7nvQDs/nxfRN0Av+7oeX6AHkcpmZBiFxgV6YuCcS6/ZrP +px9Aw7vMWgpVSzs4dlG4Y4uElBbmVvMCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP6rAJCYniT8qcwaivsnuL8wbqg7 +MA0GCSqGSIb3DQEBCwUAA4ICAQDPdyxuVr5Os7aEAJSrR8kN0nbHhp8dB9O2tLsI +eK9p0gtJ3jPFrK3CiAJ9Brc1AsFgyb/E6JTe1NOpEyVa/m6irn0F3H3zbPB+po3u +2dfOWBfoqSmuc0iH55vKbimhZF8ZE/euBhD/UcabTVUlT5OZEAFTdfETzsemQUHS +v4ilf0X8rLiltTMMgsT7B/Zq5SWEXwbKwYY5EdtYzXc7LMJMD16a4/CrPmEbUCTC +wPTxGfARKbalGAKb12NMcIxHowNDXLldRqANb/9Zjr7dn3LDWyvfjFvO5QxGbJKy +CqNMVEIYFRIYvdr8unRu/8G2oGTYqV9Vrp9canaW2HNnh/tNf1zuacpzEPuKqf2e +vTY4SUmH9A4U8OmHuD+nT3pajnnUk+S7aFKErGzp85hwVXIy+TSrK0m1zSBi5Dp6 +Z2Orltxtrpfs/J92VoguZs9btsmksNcFuuEnL5O7Jiqik7Ab846+HUCjuTaPPoIa +Gl6I6lD4WeKDRikL40Rc4ZW2aZCaFG+XroHPaO+Zmr615+F/+PoTRxZMzG0IQOeL +eG9QgkRQP2YGiqtDhFZKDyAthg710tvSeopLzaXoTvFeJiUBWSOgftL2fiFX1ye8 +FVdMpEbB4IMeDExNH08GGeL5qPQ6gqGyeUN51q1veieQA6TqJIc/2b3Z6fJfUEkc +7uzXLg== +-----END CERTIFICATE----- + +# Issuer: CN=IdenTrust Commercial Root CA 1 O=IdenTrust +# Subject: CN=IdenTrust Commercial Root CA 1 O=IdenTrust +# Label: "IdenTrust Commercial Root CA 1" +# Serial: 13298821034946342390520003877796839426 +# MD5 Fingerprint: b3:3e:77:73:75:ee:a0:d3:e3:7e:49:63:49:59:bb:c7 +# SHA1 Fingerprint: df:71:7e:aa:4a:d9:4e:c9:55:84:99:60:2d:48:de:5f:bc:f0:3a:25 +# SHA256 Fingerprint: 5d:56:49:9b:e4:d2:e0:8b:cf:ca:d0:8a:3e:38:72:3d:50:50:3b:de:70:69:48:e4:2f:55:60:30:19:e5:28:ae +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK +MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu +VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw +MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw +JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT +3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU ++ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp +S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 +bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi +T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL +vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK +Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK +dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT +c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv +l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N +iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD +ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH +6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt +LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 +nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 ++wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK +W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT +AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq +l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG +4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ +mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A +7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H +-----END CERTIFICATE----- + +# Issuer: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust +# Subject: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust +# Label: "IdenTrust Public Sector Root CA 1" +# Serial: 13298821034946342390521976156843933698 +# MD5 Fingerprint: 37:06:a5:b0:fc:89:9d:ba:f4:6b:8c:1a:64:cd:d5:ba +# SHA1 Fingerprint: ba:29:41:60:77:98:3f:f4:f3:ef:f2:31:05:3b:2e:ea:6d:4d:45:fd +# SHA256 Fingerprint: 30:d0:89:5a:9a:44:8a:26:20:91:63:55:22:d1:f5:20:10:b5:86:7a:ca:e1:2c:78:ef:95:8f:d4:f4:38:9f:2f +-----BEGIN CERTIFICATE----- +MIIFZjCCA06gAwIBAgIQCgFCgAAAAUUjz0Z8AAAAAjANBgkqhkiG9w0BAQsFADBN +MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVu +VHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwHhcNMTQwMTE2MTc1MzMyWhcN +MzQwMTE2MTc1MzMyWjBNMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0 +MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2IpT8pEiv6EdrCvsnduTyP4o7 +ekosMSqMjbCpwzFrqHd2hCa2rIFCDQjrVVi7evi8ZX3yoG2LqEfpYnYeEe4IFNGy +RBb06tD6Hi9e28tzQa68ALBKK0CyrOE7S8ItneShm+waOh7wCLPQ5CQ1B5+ctMlS +bdsHyo+1W/CD80/HLaXIrcuVIKQxKFdYWuSNG5qrng0M8gozOSI5Cpcu81N3uURF +/YTLNiCBWS2ab21ISGHKTN9T0a9SvESfqy9rg3LvdYDaBjMbXcjaY8ZNzaxmMc3R +3j6HEDbhuaR672BQssvKplbgN6+rNBM5Jeg5ZuSYeqoSmJxZZoY+rfGwyj4GD3vw +EUs3oERte8uojHH01bWRNszwFcYr3lEXsZdMUD2xlVl8BX0tIdUAvwFnol57plzy +9yLxkA2T26pEUWbMfXYD62qoKjgZl3YNa4ph+bz27nb9cCvdKTz4Ch5bQhyLVi9V +GxyhLrXHFub4qjySjmm2AcG1hp2JDws4lFTo6tyePSW8Uybt1as5qsVATFSrsrTZ +2fjXctscvG29ZV/viDUqZi/u9rNl8DONfJhBaUYPQxxp+pu10GFqzcpL2UyQRqsV +WaFHVCkugyhfHMKiq3IXAAaOReyL4jM9f9oZRORicsPfIsbyVtTdX5Vy7W1f90gD +W/3FKqD2cyOEEBsB5wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAdBgNVHQ4EFgQU43HgntinQtnbcZFrlJPrw6PRFKMwDQYJKoZIhvcN +AQELBQADggIBAEf63QqwEZE4rU1d9+UOl1QZgkiHVIyqZJnYWv6IAcVYpZmxI1Qj +t2odIFflAWJBF9MJ23XLblSQdf4an4EKwt3X9wnQW3IV5B4Jaj0z8yGa5hV+rVHV +DRDtfULAj+7AmgjVQdZcDiFpboBhDhXAuM/FSRJSzL46zNQuOAXeNf0fb7iAaJg9 +TaDKQGXSc3z1i9kKlT/YPyNtGtEqJBnZhbMX73huqVjRI9PHE+1yJX9dsXNw0H8G +lwmEKYBhHfpe/3OsoOOJuBxxFcbeMX8S3OFtm6/n6J91eEyrRjuazr8FGF1NFTwW +mhlQBJqymm9li1JfPFgEKCXAZmExfrngdbkaqIHWchezxQMxNRF4eKLg6TCMf4Df +WN88uieW4oA0beOY02QnrEh+KHdcxiVhJfiFDGX6xDIvpZgF5PgLZxYWxoK4Mhn5 ++bl53B/N66+rDt0b20XkeucC4pVd/GnwU2lhlXV5C15V5jgclKlZM57IcXR5f1GJ +tshquDDIajjDbp7hNxbqBWJMWxJH7ae0s1hWx0nzfxJoCTFx8G34Tkf71oXuxVhA +GaQdp/lLQzfcaFpPz+vCZHTetBXZ9FRUGi8c15dxVJCO2SCdUyt/q4/i6jC8UDfv +8Ue1fXwsBOxonbRJRBD0ckscZOf85muQ3Wl9af0AVqW3rLatt8o+Ae+c +-----END CERTIFICATE----- + +# Issuer: CN=Entrust Root Certification Authority - G2 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2009 Entrust, Inc. - for authorized use only +# Subject: CN=Entrust Root Certification Authority - G2 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2009 Entrust, Inc. - for authorized use only +# Label: "Entrust Root Certification Authority - G2" +# Serial: 1246989352 +# MD5 Fingerprint: 4b:e2:c9:91:96:65:0c:f4:0e:5a:93:92:a0:0a:fe:b2 +# SHA1 Fingerprint: 8c:f4:27:fd:79:0c:3a:d1:66:06:8d:e8:1e:57:ef:bb:93:22:72:d4 +# SHA256 Fingerprint: 43:df:57:74:b0:3e:7f:ef:5f:e4:0d:93:1a:7b:ed:f1:bb:2e:6b:42:73:8c:4e:6d:38:41:10:3d:3a:a7:f3:39 +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 +cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs +IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz +dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy +NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu +dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt +dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 +aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T +RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN +cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW +wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 +U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 +jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN +BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ +jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v +1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R +nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH +VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== +-----END CERTIFICATE----- + +# Issuer: CN=Entrust Root Certification Authority - EC1 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2012 Entrust, Inc. - for authorized use only +# Subject: CN=Entrust Root Certification Authority - EC1 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2012 Entrust, Inc. - for authorized use only +# Label: "Entrust Root Certification Authority - EC1" +# Serial: 51543124481930649114116133369 +# MD5 Fingerprint: b6:7e:1d:f0:58:c5:49:6c:24:3b:3d:ed:98:18:ed:bc +# SHA1 Fingerprint: 20:d8:06:40:df:9b:25:f5:12:25:3a:11:ea:f7:59:8a:eb:14:b5:47 +# SHA256 Fingerprint: 02:ed:0e:b2:8c:14:da:45:16:5c:56:67:91:70:0d:64:51:d7:fb:56:f0:b2:ab:1d:3b:8e:b0:70:e5:6e:df:f5 +-----BEGIN CERTIFICATE----- +MIIC+TCCAoCgAwIBAgINAKaLeSkAAAAAUNCR+TAKBggqhkjOPQQDAzCBvzELMAkG +A1UEBhMCVVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3 +d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDEyIEVu +dHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEzMDEGA1UEAxMq +RW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRUMxMB4XDTEy +MTIxODE1MjUzNloXDTM3MTIxODE1NTUzNlowgb8xCzAJBgNVBAYTAlVTMRYwFAYD +VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0 +L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxMiBFbnRydXN0LCBJbmMuIC0g +Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMzAxBgNVBAMTKkVudHJ1c3QgUm9vdCBD +ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEVDMTB2MBAGByqGSM49AgEGBSuBBAAi +A2IABIQTydC6bUF74mzQ61VfZgIaJPRbiWlH47jCffHyAsWfoPZb1YsGGYZPUxBt +ByQnoaD41UcZYUx9ypMn6nQM72+WCf5j7HBdNq1nd67JnXxVRDqiY1Ef9eNi1KlH +Bz7MIKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O +BBYEFLdj5xrdjekIplWDpOBqUEFlEUJJMAoGCCqGSM49BAMDA2cAMGQCMGF52OVC +R98crlOZF7ZvHH3hvxGU0QOIdeSNiaSKd0bebWHvAvX7td/M/k7//qnmpwIwW5nX +hTcGtXsI/esni0qU+eH6p44mCOh8kmhtc9hvJqwhAriZtyZBWyVgrtBIGu4G +-----END CERTIFICATE----- + +# Issuer: CN=CFCA EV ROOT O=China Financial Certification Authority +# Subject: CN=CFCA EV ROOT O=China Financial Certification Authority +# Label: "CFCA EV ROOT" +# Serial: 407555286 +# MD5 Fingerprint: 74:e1:b6:ed:26:7a:7a:44:30:33:94:ab:7b:27:81:30 +# SHA1 Fingerprint: e2:b8:29:4b:55:84:ab:6b:58:c2:90:46:6c:ac:3f:b8:39:8f:84:83 +# SHA256 Fingerprint: 5c:c3:d7:8e:4e:1d:5e:45:54:7a:04:e6:87:3e:64:f9:0c:f9:53:6d:1c:cc:2e:f8:00:f3:55:c4:c5:fd:70:fd +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIEGErM1jANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJD +TjEwMC4GA1UECgwnQ2hpbmEgRmluYW5jaWFsIENlcnRpZmljYXRpb24gQXV0aG9y +aXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJPT1QwHhcNMTIwODA4MDMwNzAxWhcNMjkx +MjMxMDMwNzAxWjBWMQswCQYDVQQGEwJDTjEwMC4GA1UECgwnQ2hpbmEgRmluYW5j +aWFsIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJP +T1QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDXXWvNED8fBVnVBU03 +sQ7smCuOFR36k0sXgiFxEFLXUWRwFsJVaU2OFW2fvwwbwuCjZ9YMrM8irq93VCpL +TIpTUnrD7i7es3ElweldPe6hL6P3KjzJIx1qqx2hp/Hz7KDVRM8Vz3IvHWOX6Jn5 +/ZOkVIBMUtRSqy5J35DNuF++P96hyk0g1CXohClTt7GIH//62pCfCqktQT+x8Rgp +7hZZLDRJGqgG16iI0gNyejLi6mhNbiyWZXvKWfry4t3uMCz7zEasxGPrb382KzRz +EpR/38wmnvFyXVBlWY9ps4deMm/DGIq1lY+wejfeWkU7xzbh72fROdOXW3NiGUgt +hxwG+3SYIElz8AXSG7Ggo7cbcNOIabla1jj0Ytwli3i/+Oh+uFzJlU9fpy25IGvP +a931DfSCt/SyZi4QKPaXWnuWFo8BGS1sbn85WAZkgwGDg8NNkt0yxoekN+kWzqot +aK8KgWU6cMGbrU1tVMoqLUuFG7OA5nBFDWteNfB/O7ic5ARwiRIlk9oKmSJgamNg +TnYGmE69g60dWIolhdLHZR4tjsbftsbhf4oEIRUpdPA+nJCdDC7xij5aqgwJHsfV +PKPtl8MeNPo4+QgO48BdK4PRVmrJtqhUUy54Mmc9gn900PvhtgVguXDbjgv5E1hv +cWAQUhC5wUEJ73IfZzF4/5YFjQIDAQABo2MwYTAfBgNVHSMEGDAWgBTj/i39KNAL +tbq2osS/BqoFjJP7LzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAd +BgNVHQ4EFgQU4/4t/SjQC7W6tqLEvwaqBYyT+y8wDQYJKoZIhvcNAQELBQADggIB +ACXGumvrh8vegjmWPfBEp2uEcwPenStPuiB/vHiyz5ewG5zz13ku9Ui20vsXiObT +ej/tUxPQ4i9qecsAIyjmHjdXNYmEwnZPNDatZ8POQQaIxffu2Bq41gt/UP+TqhdL +jOztUmCypAbqTuv0axn96/Ua4CUqmtzHQTb3yHQFhDmVOdYLO6Qn+gjYXB74BGBS +ESgoA//vU2YApUo0FmZ8/Qmkrp5nGm9BC2sGE5uPhnEFtC+NiWYzKXZUmhH4J/qy +P5Hgzg0b8zAarb8iXRvTvyUFTeGSGn+ZnzxEk8rUQElsgIfXBDrDMlI1Dlb4pd19 +xIsNER9Tyx6yF7Zod1rg1MvIB671Oi6ON7fQAUtDKXeMOZePglr4UeWJoBjnaH9d +Ci77o0cOPaYjesYBx4/IXr9tgFa+iiS6M+qf4TIRnvHST4D2G0CvOJ4RUHlzEhLN +5mydLIhyPDCBBpEi6lmt2hkuIsKNuYyH4Ga8cyNfIWRjgEj1oDwYPZTISEEdQLpe +/v5WOaHIz16eGWRGENoXkbcFgKyLmZJ956LYBws2J+dIeWCKw9cTXPhyQN9Ky8+Z +AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ +5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GB CA" +# Serial: 157768595616588414422159278966750757568 +# MD5 Fingerprint: a4:eb:b9:61:28:2e:b7:2f:98:b0:35:26:90:99:51:1d +# SHA1 Fingerprint: 0f:f9:40:76:18:d3:d7:6a:4b:98:f0:a8:35:9e:0c:fd:27:ac:cc:ed +# SHA256 Fingerprint: 6b:9c:08:e8:6e:b0:f7:67:cf:ad:65:cd:98:b6:21:49:e5:49:4a:67:f5:84:5e:7b:d1:ed:01:9f:27:b8:6b:d6 +-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIQdrEgUnTwhYdGs/gjGvbCwDANBgkqhkiG9w0BAQsFADBt +MQswCQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUg +Rm91bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9i +YWwgUm9vdCBHQiBDQTAeFw0xNDEyMDExNTAwMzJaFw0zOTEyMDExNTEwMzFaMG0x +CzAJBgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBG +b3VuZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2Jh +bCBSb290IEdCIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2Be3 +HEokKtaXscriHvt9OO+Y9bI5mE4nuBFde9IllIiCFSZqGzG7qFshISvYD06fWvGx +WuR51jIjK+FTzJlFXHtPrby/h0oLS5daqPZI7H17Dc0hBt+eFf1Biki3IPShehtX +1F1Q/7pn2COZH8g/497/b1t3sWtuuMlk9+HKQUYOKXHQuSP8yYFfTvdv37+ErXNk +u7dCjmn21HYdfp2nuFeKUWdy19SouJVUQHMD9ur06/4oQnc/nSMbsrY9gBQHTC5P +99UKFg29ZkM3fiNDecNAhvVMKdqOmq0NpQSHiB6F4+lT1ZvIiwNjeOvgGUpuuy9r +M2RYk61pv48b74JIxwIDAQABo1EwTzALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUNQ/INmNe4qPs+TtmFc5RUuORmj0wEAYJKwYBBAGCNxUB +BAMCAQAwDQYJKoZIhvcNAQELBQADggEBAEBM+4eymYGQfp3FsLAmzYh7KzKNbrgh +cViXfa43FK8+5/ea4n32cZiZBKpDdHij40lhPnOMTZTg+XHEthYOU3gf1qKHLwI5 +gSk8rxWYITD+KJAAjNHhy/peyP34EEY7onhCkRd0VQreUGdNZtGn//3ZwLWoo4rO +ZvUPQ82nK1d7Y0Zqqi5S2PTt4W2tKZB4SLrhI6qjiey1q5bAtEuiHZeeevJuQHHf +aPFlTc58Bd9TZaml8LGXBHAVRgOY1NK/VLSgWH1Sb9pWJmLU2NuJMW8c8CLC02Ic +Nc1MaRVUGpCY3useX8p3x8uOPUNpnJpY0CQ73xtAln41rYHHTnG6iBM= +-----END CERTIFICATE----- + +# Issuer: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. +# Subject: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. +# Label: "SZAFIR ROOT CA2" +# Serial: 357043034767186914217277344587386743377558296292 +# MD5 Fingerprint: 11:64:c1:89:b0:24:b1:8c:b1:07:7e:89:9e:51:9e:99 +# SHA1 Fingerprint: e2:52:fa:95:3f:ed:db:24:60:bd:6e:28:f3:9c:cc:cf:5e:b3:3f:de +# SHA256 Fingerprint: a1:33:9d:33:28:1a:0b:56:e5:57:d3:d3:2b:1c:e7:f9:36:7e:b0:94:bd:5f:a7:2a:7e:50:04:c8:de:d7:ca:fe +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUPopdB+xV0jLVt+O2XwHrLdzk1uQwDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCUEwxKDAmBgNVBAoMH0tyYWpvd2EgSXpiYSBSb3psaWN6 +ZW5pb3dhIFMuQS4xGDAWBgNVBAMMD1NaQUZJUiBST09UIENBMjAeFw0xNTEwMTkw +NzQzMzBaFw0zNTEwMTkwNzQzMzBaMFExCzAJBgNVBAYTAlBMMSgwJgYDVQQKDB9L +cmFqb3dhIEl6YmEgUm96bGljemVuaW93YSBTLkEuMRgwFgYDVQQDDA9TWkFGSVIg +Uk9PVCBDQTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3vD5QqEvN +QLXOYeeWyrSh2gwisPq1e3YAd4wLz32ohswmUeQgPYUM1ljj5/QqGJ3a0a4m7utT +3PSQ1hNKDJA8w/Ta0o4NkjrcsbH/ON7Dui1fgLkCvUqdGw+0w8LBZwPd3BucPbOw +3gAeqDRHu5rr/gsUvTaE2g0gv/pby6kWIK05YO4vdbbnl5z5Pv1+TW9NL++IDWr6 +3fE9biCloBK0TXC5ztdyO4mTp4CEHCdJckm1/zuVnsHMyAHs6A6KCpbns6aH5db5 +BSsNl0BwPLqsdVqc1U2dAgrSS5tmS0YHF2Wtn2yIANwiieDhZNRnvDF5YTy7ykHN +XGoAyDw4jlivAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD +AgEGMB0GA1UdDgQWBBQuFqlKGLXLzPVvUPMjX/hd56zwyDANBgkqhkiG9w0BAQsF +AAOCAQEAtXP4A9xZWx126aMqe5Aosk3AM0+qmrHUuOQn/6mWmc5G4G18TKI4pAZw +8PRBEew/R40/cof5O/2kbytTAOD/OblqBw7rHRz2onKQy4I9EYKL0rufKq8h5mOG +nXkZ7/e7DDWQw4rtTw/1zBLZpD67oPwglV9PJi8RI4NOdQcPv5vRtB3pEAT+ymCP +oky4rc/hkA/NrgrHXXu3UNLUYfrVFdvXn4dRVOul4+vJhaAlIDf7js4MNIThPIGy +d05DpYhfhmehPea0XGG2Ptv+tyjFogeutcrKjSoS75ftwjCkySp6+/NNIxuZMzSg +LvWpCz/UXeHPhJ/iGcJfitYgHuNztw== +-----END CERTIFICATE----- + +# Issuer: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Subject: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Label: "Certum Trusted Network CA 2" +# Serial: 44979900017204383099463764357512596969 +# MD5 Fingerprint: 6d:46:9e:d9:25:6d:08:23:5b:5e:74:7d:1e:27:db:f2 +# SHA1 Fingerprint: d3:dd:48:3e:2b:bf:4c:05:e8:af:10:f5:fa:76:26:cf:d3:dc:30:92 +# SHA256 Fingerprint: b6:76:f2:ed:da:e8:77:5c:d3:6c:b0:f6:3c:d1:d4:60:39:61:f4:9e:62:65:ba:01:3a:2f:03:07:b6:d0:b8:04 +-----BEGIN CERTIFICATE----- +MIIF0jCCA7qgAwIBAgIQIdbQSk8lD8kyN/yqXhKN6TANBgkqhkiG9w0BAQ0FADCB +gDELMAkGA1UEBhMCUEwxIjAgBgNVBAoTGVVuaXpldG8gVGVjaG5vbG9naWVzIFMu +QS4xJzAlBgNVBAsTHkNlcnR1bSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTEkMCIG +A1UEAxMbQ2VydHVtIFRydXN0ZWQgTmV0d29yayBDQSAyMCIYDzIwMTExMDA2MDgz +OTU2WhgPMjA0NjEwMDYwODM5NTZaMIGAMQswCQYDVQQGEwJQTDEiMCAGA1UEChMZ +VW5pemV0byBUZWNobm9sb2dpZXMgUy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5MSQwIgYDVQQDExtDZXJ0dW0gVHJ1c3RlZCBOZXR3 +b3JrIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9+Xj45tWA +DGSdhhuWZGc/IjoedQF97/tcZ4zJzFxrqZHmuULlIEub2pt7uZld2ZuAS9eEQCsn +0+i6MLs+CRqnSZXvK0AkwpfHp+6bJe+oCgCXhVqqndwpyeI1B+twTUrWwbNWuKFB +OJvR+zF/j+Bf4bE/D44WSWDXBo0Y+aomEKsq09DRZ40bRr5HMNUuctHFY9rnY3lE +fktjJImGLjQ/KUxSiyqnwOKRKIm5wFv5HdnnJ63/mgKXwcZQkpsCLL2puTRZCr+E +Sv/f/rOf69me4Jgj7KZrdxYq28ytOxykh9xGc14ZYmhFV+SQgkK7QtbwYeDBoz1m +o130GO6IyY0XRSmZMnUCMe4pJshrAua1YkV/NxVaI2iJ1D7eTiew8EAMvE0Xy02i +sx7QBlrd9pPPV3WZ9fqGGmd4s7+W/jTcvedSVuWz5XV710GRBdxdaeOVDUO5/IOW +OZV7bIBaTxNyxtd9KXpEulKkKtVBRgkg/iKgtlswjbyJDNXXcPiHUv3a76xRLgez +Tv7QCdpw75j6VuZt27VXS9zlLCUVyJ4ueE742pyehizKV/Ma5ciSixqClnrDvFAS +adgOWkaLOusm+iPJtrCBvkIApPjW/jAux9JG9uWOdf3yzLnQh1vMBhBgu4M1t15n +3kfsmUjxpKEV/q2MYo45VU85FrmxY53/twIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBS2oVQ5AsOgP46KvPrU+Bym0ToO/TAOBgNVHQ8BAf8EBAMC +AQYwDQYJKoZIhvcNAQENBQADggIBAHGlDs7k6b8/ONWJWsQCYftMxRQXLYtPU2sQ +F/xlhMcQSZDe28cmk4gmb3DWAl45oPePq5a1pRNcgRRtDoGCERuKTsZPpd1iHkTf +CVn0W3cLN+mLIMb4Ck4uWBzrM9DPhmDJ2vuAL55MYIR4PSFk1vtBHxgP58l1cb29 +XN40hz5BsA72udY/CROWFC/emh1auVbONTqwX3BNXuMp8SMoclm2q8KMZiYcdywm +djWLKKdpoPk79SPdhRB0yZADVpHnr7pH1BKXESLjokmUbOe3lEu6LaTaM4tMpkT/ +WjzGHWTYtTHkpjx6qFcL2+1hGsvxznN3Y6SHb0xRONbkX8eftoEq5IVIeVheO/jb +AoJnwTnbw3RLPTYe+SmTiGhbqEQZIfCn6IENLOiTNrQ3ssqwGyZ6miUfmpqAnksq +P/ujmv5zMnHCnsZy4YpoJ/HkD7TETKVhk/iXEAcqMCWpuchxuO9ozC1+9eB+D4Ko +b7a6bINDd82Kkhehnlt4Fj1F4jNy3eFmypnTycUm/Q1oBEauttmbjL4ZvrHG8hnj +XALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLXis7VmFxWlgPF7ncGNf/P +5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7zAYspsbi +DrW5viSP +-----END CERTIFICATE----- + +# Issuer: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Subject: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Label: "Hellenic Academic and Research Institutions RootCA 2015" +# Serial: 0 +# MD5 Fingerprint: ca:ff:e2:db:03:d9:cb:4b:e9:0f:ad:84:fd:7b:18:ce +# SHA1 Fingerprint: 01:0c:06:95:a6:98:19:14:ff:bf:5f:c6:b0:b6:95:ea:29:e9:12:a6 +# SHA256 Fingerprint: a0:40:92:9a:02:ce:53:b4:ac:f4:f2:ff:c6:98:1c:e4:49:6f:75:5e:6d:45:fe:0b:2a:69:2b:cd:52:52:3f:36 +-----BEGIN CERTIFICATE----- +MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1Ix +DzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5k +IFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMT +N0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9v +dENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAxMTIxWjCBpjELMAkG +A1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNh +ZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkx +QDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1 +dGlvbnMgUm9vdENBIDIwMTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQDC+Kk/G4n8PDwEXT2QNrCROnk8ZlrvbTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA +4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+ehiGsxr/CL0BgzuNtFajT0 +AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+6PAQZe10 +4S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06C +ojXdFPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV +9Cz82XBST3i4vTwri5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrD +gfgXy5I2XdGj2HUb4Ysn6npIQf1FGQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6 +Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2fu/Z8VFRfS0myGlZYeCsargq +NhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9muiNX6hME6wGko +LfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc +Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNV +HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVd +ctA4GGqd83EkVAswDQYJKoZIhvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0I +XtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+D1hYc2Ryx+hFjtyp8iY/xnmMsVMI +M4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrMd/K4kPFox/la/vot +9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+yd+2V +Z5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/ea +j8GsGsVn82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnh +X9izjFk0WaSrT2y7HxjbdavYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQ +l033DlZdwJVqwjbDG2jJ9SrcR5q+ss7FJej6A7na+RZukYT1HCjI/CbM1xyQVqdf +bzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVtJ94Cj8rDtSvK6evIIVM4 +pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGaJI7ZjnHK +e7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0 +vm9qp/UsQu0yrbYhnr68 +-----END CERTIFICATE----- + +# Issuer: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Subject: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Label: "Hellenic Academic and Research Institutions ECC RootCA 2015" +# Serial: 0 +# MD5 Fingerprint: 81:e5:b4:17:eb:c2:f5:e1:4b:0d:41:7b:49:92:fe:ef +# SHA1 Fingerprint: 9f:f1:71:8d:92:d5:9a:f3:7d:74:97:b4:bc:6f:84:68:0b:ba:b6:66 +# SHA256 Fingerprint: 44:b5:45:aa:8a:25:e6:5a:73:ca:15:dc:27:fc:36:d2:4c:1c:b9:95:3a:06:65:39:b1:15:82:dc:48:7b:48:33 +-----BEGIN CERTIFICATE----- +MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzAN +BgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl +c2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hl +bGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgRUNDIFJv +b3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEwMzcxMlowgaoxCzAJ +BgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmljIEFj +YWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5 +MUQwQgYDVQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0 +dXRpb25zIEVDQyBSb290Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKg +QehLgoRc4vgxEZmGZE4JJS+dQS8KrjVPdJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJa +jq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoKVlp8aQuqgAkkbH7BRqNC +MEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFLQi +C4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaep +lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof +TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR +-----END CERTIFICATE----- + +# Issuer: CN=ISRG Root X1 O=Internet Security Research Group +# Subject: CN=ISRG Root X1 O=Internet Security Research Group +# Label: "ISRG Root X1" +# Serial: 172886928669790476064670243504169061120 +# MD5 Fingerprint: 0c:d2:f9:e0:da:17:73:e9:ed:86:4d:a5:e3:70:e7:4e +# SHA1 Fingerprint: ca:bd:2a:79:a1:07:6a:31:f2:1d:25:36:35:cb:03:9d:43:29:a5:e8 +# SHA256 Fingerprint: 96:bc:ec:06:26:49:76:f3:74:60:77:9a:cf:28:c5:a7:cf:e8:a3:c0:aa:e1:1a:8f:fc:ee:05:c0:bd:df:08:c6 +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- + +# Issuer: O=FNMT-RCM OU=AC RAIZ FNMT-RCM +# Subject: O=FNMT-RCM OU=AC RAIZ FNMT-RCM +# Label: "AC RAIZ FNMT-RCM" +# Serial: 485876308206448804701554682760554759 +# MD5 Fingerprint: e2:09:04:b4:d3:bd:d1:a0:14:fd:1a:d2:47:c4:57:1d +# SHA1 Fingerprint: ec:50:35:07:b2:15:c4:95:62:19:e2:a8:9a:5b:42:99:2c:4c:2c:20 +# SHA256 Fingerprint: eb:c5:57:0c:29:01:8c:4d:67:b1:aa:12:7b:af:12:f7:03:b4:61:1e:bc:17:b7:da:b5:57:38:94:17:9b:93:fa +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIPXZONMGc2yAYdGsdUhGkHMA0GCSqGSIb3DQEBCwUAMDsx +CzAJBgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJ +WiBGTk1ULVJDTTAeFw0wODEwMjkxNTU5NTZaFw0zMDAxMDEwMDAwMDBaMDsxCzAJ +BgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJWiBG +Tk1ULVJDTTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALpxgHpMhm5/ +yBNtwMZ9HACXjywMI7sQmkCpGreHiPibVmr75nuOi5KOpyVdWRHbNi63URcfqQgf +BBckWKo3Shjf5TnUV/3XwSyRAZHiItQDwFj8d0fsjz50Q7qsNI1NOHZnjrDIbzAz +WHFctPVrbtQBULgTfmxKo0nRIBnuvMApGGWn3v7v3QqQIecaZ5JCEJhfTzC8PhxF +tBDXaEAUwED653cXeuYLj2VbPNmaUtu1vZ5Gzz3rkQUCwJaydkxNEJY7kvqcfw+Z +374jNUUeAlz+taibmSXaXvMiwzn15Cou08YfxGyqxRxqAQVKL9LFwag0Jl1mpdIC +IfkYtwb1TplvqKtMUejPUBjFd8g5CSxJkjKZqLsXF3mwWsXmo8RZZUc1g16p6DUL +mbvkzSDGm0oGObVo/CK67lWMK07q87Hj/LaZmtVC+nFNCM+HHmpxffnTtOmlcYF7 +wk5HlqX2doWjKI/pgG6BU6VtX7hI+cL5NqYuSf+4lsKMB7ObiFj86xsc3i1w4peS +MKGJ47xVqCfWS+2QrYv6YyVZLag13cqXM7zlzced0ezvXg5KkAYmY6252TUtB7p2 +ZSysV4999AeU14ECll2jB0nVetBX+RvnU0Z1qrB5QstocQjpYL05ac70r8NWQMet +UqIJ5G+GR4of6ygnXYMgrwTJbFaai0b1AgMBAAGjgYMwgYAwDwYDVR0TAQH/BAUw +AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFPd9xf3E6Jobd2Sn9R2gzL+H +YJptMD4GA1UdIAQ3MDUwMwYEVR0gADArMCkGCCsGAQUFBwIBFh1odHRwOi8vd3d3 +LmNlcnQuZm5tdC5lcy9kcGNzLzANBgkqhkiG9w0BAQsFAAOCAgEAB5BK3/MjTvDD +nFFlm5wioooMhfNzKWtN/gHiqQxjAb8EZ6WdmF/9ARP67Jpi6Yb+tmLSbkyU+8B1 +RXxlDPiyN8+sD8+Nb/kZ94/sHvJwnvDKuO+3/3Y3dlv2bojzr2IyIpMNOmqOFGYM +LVN0V2Ue1bLdI4E7pWYjJ2cJj+F3qkPNZVEI7VFY/uY5+ctHhKQV8Xa7pO6kO8Rf +77IzlhEYt8llvhjho6Tc+hj507wTmzl6NLrTQfv6MooqtyuGC2mDOL7Nii4LcK2N +JpLuHvUBKwrZ1pebbuCoGRw6IYsMHkCtA+fdZn71uSANA+iW+YJF1DngoABd15jm +fZ5nc8OaKveri6E6FO80vFIOiZiaBECEHX5FaZNXzuvO+FB8TxxuBEOb+dY7Ixjp +6o7RTUaN8Tvkasq6+yO3m/qZASlaWFot4/nUbQ4mrcFuNLwy+AwF+mWj2zs3gyLp +1txyM/1d8iC9djwj2ij3+RvrWWTV3F9yfiD8zYm1kGdNYno/Tq0dwzn+evQoFt9B +9kiABdcPUXmsEKvU7ANm5mqwujGSQkBqvjrTcuFqN1W8rB2Vt2lh8kORdOag0wok +RqEIr9baRRmW1FMdW4R58MD3R++Lj8UGrp1MYp3/RgT408m2ECVAdf4WqslKYIYv +uu8wd+RU4riEmViAqhOLUTpPSPaLtrM= +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 1 O=Amazon +# Subject: CN=Amazon Root CA 1 O=Amazon +# Label: "Amazon Root CA 1" +# Serial: 143266978916655856878034712317230054538369994 +# MD5 Fingerprint: 43:c6:bf:ae:ec:fe:ad:2f:18:c6:88:68:30:fc:c8:e6 +# SHA1 Fingerprint: 8d:a7:f9:65:ec:5e:fc:37:91:0f:1c:6e:59:fd:c1:cc:6a:6e:de:16 +# SHA256 Fingerprint: 8e:cd:e6:88:4f:3d:87:b1:12:5b:a3:1a:c3:fc:b1:3d:70:16:de:7f:57:cc:90:4f:e1:cb:97:c6:ae:98:19:6e +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 2 O=Amazon +# Subject: CN=Amazon Root CA 2 O=Amazon +# Label: "Amazon Root CA 2" +# Serial: 143266982885963551818349160658925006970653239 +# MD5 Fingerprint: c8:e5:8d:ce:a8:42:e2:7a:c0:2a:5c:7c:9e:26:bf:66 +# SHA1 Fingerprint: 5a:8c:ef:45:d7:a6:98:59:76:7a:8c:8b:44:96:b5:78:cf:47:4b:1a +# SHA256 Fingerprint: 1b:a5:b2:aa:8c:65:40:1a:82:96:01:18:f8:0b:ec:4f:62:30:4d:83:ce:c4:71:3a:19:c3:9c:01:1e:a4:6d:b4 +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK +gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ +W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg +1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K +8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r +2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me +z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR +8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj +mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz +7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 ++XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI +0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm +UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 +LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY ++gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS +k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl +7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm +btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl +urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ +fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 +n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE +76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H +9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT +4PsJYGw= +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 3 O=Amazon +# Subject: CN=Amazon Root CA 3 O=Amazon +# Label: "Amazon Root CA 3" +# Serial: 143266986699090766294700635381230934788665930 +# MD5 Fingerprint: a0:d4:ef:0b:f7:b5:d8:49:95:2a:ec:f5:c4:fc:81:87 +# SHA1 Fingerprint: 0d:44:dd:8c:3c:8c:1a:1a:58:75:64:81:e9:0f:2e:2a:ff:b3:d2:6e +# SHA256 Fingerprint: 18:ce:6c:fe:7b:f1:4e:60:b2:e3:47:b8:df:e8:68:cb:31:d0:2e:bb:3a:da:27:15:69:f5:03:43:b4:6d:b3:a4 +-----BEGIN CERTIFICATE----- +MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl +ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr +ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr +BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM +YyRIHN8wfdVoOw== +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 4 O=Amazon +# Subject: CN=Amazon Root CA 4 O=Amazon +# Label: "Amazon Root CA 4" +# Serial: 143266989758080763974105200630763877849284878 +# MD5 Fingerprint: 89:bc:27:d5:eb:17:8d:06:6a:69:d5:fd:89:47:b4:cd +# SHA1 Fingerprint: f6:10:84:07:d6:f8:bb:67:98:0c:c2:e2:44:c2:eb:ae:1c:ef:63:be +# SHA256 Fingerprint: e3:5d:28:41:9e:d0:20:25:cf:a6:90:38:cd:62:39:62:45:8d:a5:c6:95:fb:de:a3:c2:2b:0b:fb:25:89:70:92 +-----BEGIN CERTIFICATE----- +MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi +9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk +M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB +MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw +CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW +1KyLa2tJElMzrdfkviT8tQp21KW8EA== +-----END CERTIFICATE----- + +# Issuer: CN=LuxTrust Global Root 2 O=LuxTrust S.A. +# Subject: CN=LuxTrust Global Root 2 O=LuxTrust S.A. +# Label: "LuxTrust Global Root 2" +# Serial: 59914338225734147123941058376788110305822489521 +# MD5 Fingerprint: b2:e1:09:00:61:af:f7:f1:91:6f:c4:ad:8d:5e:3b:7c +# SHA1 Fingerprint: 1e:0e:56:19:0a:d1:8b:25:98:b2:04:44:ff:66:8a:04:17:99:5f:3f +# SHA256 Fingerprint: 54:45:5f:71:29:c2:0b:14:47:c4:18:f9:97:16:8f:24:c5:8f:c5:02:3b:f5:da:5b:e2:eb:6e:1d:d8:90:2e:d5 +-----BEGIN CERTIFICATE----- +MIIFwzCCA6ugAwIBAgIUCn6m30tEntpqJIWe5rgV0xZ/u7EwDQYJKoZIhvcNAQEL +BQAwRjELMAkGA1UEBhMCTFUxFjAUBgNVBAoMDUx1eFRydXN0IFMuQS4xHzAdBgNV +BAMMFkx1eFRydXN0IEdsb2JhbCBSb290IDIwHhcNMTUwMzA1MTMyMTU3WhcNMzUw +MzA1MTMyMTU3WjBGMQswCQYDVQQGEwJMVTEWMBQGA1UECgwNTHV4VHJ1c3QgUy5B +LjEfMB0GA1UEAwwWTHV4VHJ1c3QgR2xvYmFsIFJvb3QgMjCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBANeFl78RmOnwYoNMPIf5U2o3C/IPPIfOb9wmKb3F +ibrJgz337spbxm1Jc7TJRqMbNBM/wYlFV/TZsfs2ZUv7COJIcRHIbjuend+JZTem +hfY7RBi2xjcwYkSSl2l9QjAk5A0MiWtj3sXh306pFGxT4GHO9hcvHTy95iJMHZP1 +EMShduxq3sVs35a0VkBCwGKSMKEtFZSg0iAGCW5qbeXrt77U8PEVfIvmTroTzEsn +Xpk8F12PgX8zPU/TPxvsXD/wPEx1bvKm1Z3aLQdjAsZy6ZS8TEmVT4hSyNvoaYL4 +zDRbIvCGp4m9SAptZoFtyMhk+wHh9OHe2Z7d21vUKpkmFRseTJIpgp7VkoGSQXAZ +96Tlk0u8d2cx3Rz9MXANF5kM+Qw5GSoXtTBxVdUPrljhPS80m8+f9niFwpN6cj5m +j5wWEWCPnolvZ77gR1o7DJpni89Gxq44o/KnvObWhWszJHAiS8sIm7vI+AIpHb4g +DEa/a4ebsypmQjVGbKq6rfmYe+lQVRQxv7HaLe2ArWgk+2mr2HETMOZns4dA/Yl+ +8kPREd8vZS9kzl8UubG/Mb2HeFpZZYiq/FkySIbWTLkpS5XTdvN3JW1CHDiDTf2j +X5t/Lax5Gw5CMZdjpPuKadUiDTSQMC6otOBttpSsvItO13D8xTiOZCXhTTmQzsmH +hFhxAgMBAAGjgagwgaUwDwYDVR0TAQH/BAUwAwEB/zBCBgNVHSAEOzA5MDcGByuB +KwEBAQowLDAqBggrBgEFBQcCARYeaHR0cHM6Ly9yZXBvc2l0b3J5Lmx1eHRydXN0 +Lmx1MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBT/GCh2+UgFLKGu8SsbK7JT ++Et8szAdBgNVHQ4EFgQU/xgodvlIBSyhrvErGyuyU/hLfLMwDQYJKoZIhvcNAQEL +BQADggIBAGoZFO1uecEsh9QNcH7X9njJCwROxLHOk3D+sFTAMs2ZMGQXvw/l4jP9 +BzZAcg4atmpZ1gDlaCDdLnINH2pkMSCEfUmmWjfrRcmF9dTHF5kH5ptV5AzoqbTO +jFu1EVzPig4N1qx3gf4ynCSecs5U89BvolbW7MM3LGVYvlcAGvI1+ut7MV3CwRI9 +loGIlonBWVx65n9wNOeD4rHh4bhY79SV5GCc8JaXcozrhAIuZY+kt9J/Z93I055c +qqmkoCUUBpvsT34tC38ddfEz2O3OuHVtPlu5mB0xDVbYQw8wkbIEa91WvpWAVWe+ +2M2D2RjuLg+GLZKecBPs3lHJQ3gCpU3I+V/EkVhGFndadKpAvAefMLmx9xIX3eP/ +JEAdemrRTxgKqpAd60Ae36EeRJIQmvKN4dFLRp7oRUKX6kWZ8+xm1QL68qZKJKre +zrnK+T+Tb/mjuuqlPpmt/f97mfVl7vBZKGfXkJWkE4SphMHozs51k2MavDzq1WQf +LSoSOcbDWjLtR5EWDrw4wVDej8oqkDQc7kGUnF4ZLvhFSZl0kbAEb+MEWrGrKqv+ +x9CWttrhSmQGbmBNvUJO/3jaJMobtNeWOWyu8Q6qp31IiyBMz2TWuJdGsE7RKlY6 +oJO9r4Ak4Ap+58rVyuiFVdw2KuGUaJPHZnJED4AhMmwlxyOAgwrr +-----END CERTIFICATE----- + +# Issuer: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM +# Subject: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM +# Label: "TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1" +# Serial: 1 +# MD5 Fingerprint: dc:00:81:dc:69:2f:3e:2f:b0:3b:f6:3d:5a:91:8e:49 +# SHA1 Fingerprint: 31:43:64:9b:ec:ce:27:ec:ed:3a:3f:0b:8f:0d:e4:e8:91:dd:ee:ca +# SHA256 Fingerprint: 46:ed:c3:68:90:46:d5:3a:45:3f:b3:10:4a:b8:0d:ca:ec:65:8b:26:60:ea:16:29:dd:7e:86:79:90:64:87:16 +-----BEGIN CERTIFICATE----- +MIIEYzCCA0ugAwIBAgIBATANBgkqhkiG9w0BAQsFADCB0jELMAkGA1UEBhMCVFIx +GDAWBgNVBAcTD0dlYnplIC0gS29jYWVsaTFCMEAGA1UEChM5VHVya2l5ZSBCaWxp +bXNlbCB2ZSBUZWtub2xvamlrIEFyYXN0aXJtYSBLdXJ1bXUgLSBUVUJJVEFLMS0w +KwYDVQQLEyRLYW11IFNlcnRpZmlrYXN5b24gTWVya2V6aSAtIEthbXUgU00xNjA0 +BgNVBAMTLVRVQklUQUsgS2FtdSBTTSBTU0wgS29rIFNlcnRpZmlrYXNpIC0gU3Vy +dW0gMTAeFw0xMzExMjUwODI1NTVaFw00MzEwMjUwODI1NTVaMIHSMQswCQYDVQQG +EwJUUjEYMBYGA1UEBxMPR2ViemUgLSBLb2NhZWxpMUIwQAYDVQQKEzlUdXJraXll +IEJpbGltc2VsIHZlIFRla25vbG9qaWsgQXJhc3Rpcm1hIEt1cnVtdSAtIFRVQklU +QUsxLTArBgNVBAsTJEthbXUgU2VydGlmaWthc3lvbiBNZXJrZXppIC0gS2FtdSBT +TTE2MDQGA1UEAxMtVFVCSVRBSyBLYW11IFNNIFNTTCBLb2sgU2VydGlmaWthc2kg +LSBTdXJ1bSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr3UwM6q7 +a9OZLBI3hNmNe5eA027n/5tQlT6QlVZC1xl8JoSNkvoBHToP4mQ4t4y86Ij5iySr +LqP1N+RAjhgleYN1Hzv/bKjFxlb4tO2KRKOrbEz8HdDc72i9z+SqzvBV96I01INr +N3wcwv61A+xXzry0tcXtAA9TNypN9E8Mg/uGz8v+jE69h/mniyFXnHrfA2eJLJ2X +YacQuFWQfw4tJzh03+f92k4S400VIgLI4OD8D62K18lUUMw7D8oWgITQUVbDjlZ/ +iSIzL+aFCr2lqBs23tPcLG07xxO9WSMs5uWk99gL7eqQQESolbuT1dCANLZGeA4f +AJNG4e7p+exPFwIDAQABo0IwQDAdBgNVHQ4EFgQUZT/HiobGPN08VFw1+DrtUgxH +V8gwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBACo/4fEyjq7hmFxLXs9rHmoJ0iKpEsdeV31zVmSAhHqT5Am5EM2fKifh +AHe+SMg1qIGf5LgsyX8OsNJLN13qudULXjS99HMpw+0mFZx+CFOKWI3QSyjfwbPf +IPP54+M638yclNhOT8NrF7f3cuitZjO1JVOr4PhMqZ398g26rrnZqsZr+ZO7rqu4 +lzwDGrpDxpa5RXI4s6ehlj2Re37AIVNMh+3yC1SVUZPVIqUNivGTDj5UDrDYyU7c +8jEyVupk+eq1nRZmQnLzf9OxMUP8pI4X8W0jq5Rm+K37DwhuJi1/FwcJsoz7UMCf +lo3Ptv0AnVoUmr8CRPXBwp8iXqIPoeM= +-----END CERTIFICATE----- + +# Issuer: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. +# Subject: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. +# Label: "GDCA TrustAUTH R5 ROOT" +# Serial: 9009899650740120186 +# MD5 Fingerprint: 63:cc:d9:3d:34:35:5c:6f:53:a3:e2:08:70:48:1f:b4 +# SHA1 Fingerprint: 0f:36:38:5b:81:1a:25:c3:9b:31:4e:83:ca:e9:34:66:70:cc:74:b4 +# SHA256 Fingerprint: bf:ff:8f:d0:44:33:48:7d:6a:8a:a6:0c:1a:29:76:7a:9f:c2:bb:b0:5e:42:0f:71:3a:13:b9:92:89:1d:38:93 +-----BEGIN CERTIFICATE----- +MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE +BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ +IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 +MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV +BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w +HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj +Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj +TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u +KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj +qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm +MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 +ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP +zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk +L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC +jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA +HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC +AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB +/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg +p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm +DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 +COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry +L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf +JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg +IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io +2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV +09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ +XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq +T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe +MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== +-----END CERTIFICATE----- + +# Issuer: CN=TrustCor RootCert CA-1 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Subject: CN=TrustCor RootCert CA-1 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Label: "TrustCor RootCert CA-1" +# Serial: 15752444095811006489 +# MD5 Fingerprint: 6e:85:f1:dc:1a:00:d3:22:d5:b2:b2:ac:6b:37:05:45 +# SHA1 Fingerprint: ff:bd:cd:e7:82:c8:43:5e:3c:6f:26:86:5c:ca:a8:3a:45:5b:c3:0a +# SHA256 Fingerprint: d4:0e:9c:86:cd:8f:e4:68:c1:77:69:59:f4:9e:a7:74:fa:54:86:84:b6:c4:06:f3:90:92:61:f4:dc:e2:57:5c +-----BEGIN CERTIFICATE----- +MIIEMDCCAxigAwIBAgIJANqb7HHzA7AZMA0GCSqGSIb3DQEBCwUAMIGkMQswCQYD +VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk +MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U +cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRydXN0Q29y +IFJvb3RDZXJ0IENBLTEwHhcNMTYwMjA0MTIzMjE2WhcNMjkxMjMxMTcyMzE2WjCB +pDELMAkGA1UEBhMCUEExDzANBgNVBAgMBlBhbmFtYTEUMBIGA1UEBwwLUGFuYW1h +IENpdHkxJDAiBgNVBAoMG1RydXN0Q29yIFN5c3RlbXMgUy4gZGUgUi5MLjEnMCUG +A1UECwweVHJ1c3RDb3IgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR8wHQYDVQQDDBZU +cnVzdENvciBSb290Q2VydCBDQS0xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAv463leLCJhJrMxnHQFgKq1mqjQCj/IDHUHuO1CAmujIS2CNUSSUQIpid +RtLByZ5OGy4sDjjzGiVoHKZaBeYei0i/mJZ0PmnK6bV4pQa81QBeCQryJ3pS/C3V +seq0iWEk8xoT26nPUu0MJLq5nux+AHT6k61sKZKuUbS701e/s/OojZz0JEsq1pme +9J7+wH5COucLlVPat2gOkEz7cD+PSiyU8ybdY2mplNgQTsVHCJCZGxdNuWxu72CV +EY4hgLW9oHPY0LJ3xEXqWib7ZnZ2+AYfYW0PVcWDtxBWcgYHpfOxGgMFZA6dWorW +hnAbJN7+KIor0Gqw/Hqi3LJ5DotlDwIDAQABo2MwYTAdBgNVHQ4EFgQU7mtJPHo/ +DeOxCbeKyKsZn3MzUOcwHwYDVR0jBBgwFoAU7mtJPHo/DeOxCbeKyKsZn3MzUOcw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggEBACUY1JGPE+6PHh0RU9otRCkZoB5rMZ5NDp6tPVxBb5UrJKF5mDo4Nvu7Zp5I +/5CQ7z3UuJu0h3U/IJvOcs+hVcFNZKIZBqEHMwwLKeXx6quj7LUKdJDHfXLy11yf +ke+Ri7fc7Waiz45mO7yfOgLgJ90WmMCV1Aqk5IGadZQ1nJBfiDcGrVmVCrDRZ9MZ +yonnMlo2HD6CqFqTvsbQZJG2z9m2GM/bftJlo6bEjhcxwft+dtvTheNYsnd6djts +L1Ac59v2Z3kf9YKVmgenFK+P3CghZwnS1k1aHBkcjndcw5QkPTJrS37UeJSDvjdN +zl/HHk484IkzlQsPpTLWPFp5LBk= +-----END CERTIFICATE----- + +# Issuer: CN=TrustCor RootCert CA-2 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Subject: CN=TrustCor RootCert CA-2 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Label: "TrustCor RootCert CA-2" +# Serial: 2711694510199101698 +# MD5 Fingerprint: a2:e1:f8:18:0b:ba:45:d5:c7:41:2a:bb:37:52:45:64 +# SHA1 Fingerprint: b8:be:6d:cb:56:f1:55:b9:63:d4:12:ca:4e:06:34:c7:94:b2:1c:c0 +# SHA256 Fingerprint: 07:53:e9:40:37:8c:1b:d5:e3:83:6e:39:5d:ae:a5:cb:83:9e:50:46:f1:bd:0e:ae:19:51:cf:10:fe:c7:c9:65 +-----BEGIN CERTIFICATE----- +MIIGLzCCBBegAwIBAgIIJaHfyjPLWQIwDQYJKoZIhvcNAQELBQAwgaQxCzAJBgNV +BAYTAlBBMQ8wDQYDVQQIDAZQYW5hbWExFDASBgNVBAcMC1BhbmFtYSBDaXR5MSQw +IgYDVQQKDBtUcnVzdENvciBTeXN0ZW1zIFMuIGRlIFIuTC4xJzAlBgNVBAsMHlRy +dXN0Q29yIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWVHJ1c3RDb3Ig +Um9vdENlcnQgQ0EtMjAeFw0xNjAyMDQxMjMyMjNaFw0zNDEyMzExNzI2MzlaMIGk +MQswCQYDVQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEg +Q2l0eTEkMCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYD +VQQLDB5UcnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRy +dXN0Q29yIFJvb3RDZXJ0IENBLTIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCnIG7CKqJiJJWQdsg4foDSq8GbZQWU9MEKENUCrO2fk8eHyLAnK0IMPQo+ +QVqedd2NyuCb7GgypGmSaIwLgQ5WoD4a3SwlFIIvl9NkRvRUqdw6VC0xK5mC8tkq +1+9xALgxpL56JAfDQiDyitSSBBtlVkxs1Pu2YVpHI7TYabS3OtB0PAx1oYxOdqHp +2yqlO/rOsP9+aij9JxzIsekp8VduZLTQwRVtDr4uDkbIXvRR/u8OYzo7cbrPb1nK +DOObXUm4TOJXsZiKQlecdu/vvdFoqNL0Cbt3Nb4lggjEFixEIFapRBF37120Hape +az6LMvYHL1cEksr1/p3C6eizjkxLAjHZ5DxIgif3GIJ2SDpxsROhOdUuxTTCHWKF +3wP+TfSvPd9cW436cOGlfifHhi5qjxLGhF5DUVCcGZt45vz27Ud+ez1m7xMTiF88 +oWP7+ayHNZ/zgp6kPwqcMWmLmaSISo5uZk3vFsQPeSghYA2FFn3XVDjxklb9tTNM +g9zXEJ9L/cb4Qr26fHMC4P99zVvh1Kxhe1fVSntb1IVYJ12/+CtgrKAmrhQhJ8Z3 +mjOAPF5GP/fDsaOGM8boXg25NSyqRsGFAnWAoOsk+xWq5Gd/bnc/9ASKL3x74xdh +8N0JqSDIvgmk0H5Ew7IwSjiqqewYmgeCK9u4nBit2uBGF6zPXQIDAQABo2MwYTAd +BgNVHQ4EFgQU2f4hQG6UnrybPZx9mCAZ5YwwYrIwHwYDVR0jBBgwFoAU2f4hQG6U +nrybPZx9mCAZ5YwwYrIwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYw +DQYJKoZIhvcNAQELBQADggIBAJ5Fngw7tu/hOsh80QA9z+LqBrWyOrsGS2h60COX +dKcs8AjYeVrXWoSK2BKaG9l9XE1wxaX5q+WjiYndAfrs3fnpkpfbsEZC89NiqpX+ +MWcUaViQCqoL7jcjx1BRtPV+nuN79+TMQjItSQzL/0kMmx40/W5ulop5A7Zv2wnL +/V9lFDfhOPXzYRZY5LVtDQsEGz9QLX+zx3oaFoBg+Iof6Rsqxvm6ARppv9JYx1RX +CI/hOWB3S6xZhBqI8d3LT3jX5+EzLfzuQfogsL7L9ziUwOHQhQ+77Sxzq+3+knYa +ZH9bDTMJBzN7Bj8RpFxwPIXAz+OQqIN3+tvmxYxoZxBnpVIt8MSZj3+/0WvitUfW +2dCFmU2Umw9Lje4AWkcdEQOsQRivh7dvDDqPys/cA8GiCcjl/YBeyGBCARsaU1q7 +N6a3vLqE6R5sGtRk2tRD/pOLS/IseRYQ1JMLiI+h2IYURpFHmygk71dSTlxCnKr3 +Sewn6EAes6aJInKc9Q0ztFijMDvd1GpUk74aTfOTlPf8hAs/hCBcNANExdqtvArB +As8e5ZTZ845b2EzwnexhF7sUMlQMAimTHpKG9n/v55IFDlndmQguLvqcAFLTxWYp +5KeXRKQOKIETNcX2b2TmQcTVL8w0RSXPQQCWPUouwpaYT05KnJe32x+SMsj/D1Fu +1uwJ +-----END CERTIFICATE----- + +# Issuer: CN=TrustCor ECA-1 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Subject: CN=TrustCor ECA-1 O=TrustCor Systems S. de R.L. OU=TrustCor Certificate Authority +# Label: "TrustCor ECA-1" +# Serial: 9548242946988625984 +# MD5 Fingerprint: 27:92:23:1d:0a:f5:40:7c:e9:e6:6b:9d:d8:f5:e7:6c +# SHA1 Fingerprint: 58:d1:df:95:95:67:6b:63:c0:f0:5b:1c:17:4d:8b:84:0b:c8:78:bd +# SHA256 Fingerprint: 5a:88:5d:b1:9c:01:d9:12:c5:75:93:88:93:8c:af:bb:df:03:1a:b2:d4:8e:91:ee:15:58:9b:42:97:1d:03:9c +-----BEGIN CERTIFICATE----- +MIIEIDCCAwigAwIBAgIJAISCLF8cYtBAMA0GCSqGSIb3DQEBCwUAMIGcMQswCQYD +VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk +MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U +cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxFzAVBgNVBAMMDlRydXN0Q29y +IEVDQS0xMB4XDTE2MDIwNDEyMzIzM1oXDTI5MTIzMTE3MjgwN1owgZwxCzAJBgNV +BAYTAlBBMQ8wDQYDVQQIDAZQYW5hbWExFDASBgNVBAcMC1BhbmFtYSBDaXR5MSQw +IgYDVQQKDBtUcnVzdENvciBTeXN0ZW1zIFMuIGRlIFIuTC4xJzAlBgNVBAsMHlRy +dXN0Q29yIENlcnRpZmljYXRlIEF1dGhvcml0eTEXMBUGA1UEAwwOVHJ1c3RDb3Ig +RUNBLTEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPj+ARtZ+odnbb +3w9U73NjKYKtR8aja+3+XzP4Q1HpGjORMRegdMTUpwHmspI+ap3tDvl0mEDTPwOA +BoJA6LHip1GnHYMma6ve+heRK9jGrB6xnhkB1Zem6g23xFUfJ3zSCNV2HykVh0A5 +3ThFEXXQmqc04L/NyFIduUd+Dbi7xgz2c1cWWn5DkR9VOsZtRASqnKmcp0yJF4Ou +owReUoCLHhIlERnXDH19MURB6tuvsBzvgdAsxZohmz3tQjtQJvLsznFhBmIhVE5/ +wZ0+fyCMgMsq2JdiyIMzkX2woloPV+g7zPIlstR8L+xNxqE6FXrntl019fZISjZF +ZtS6mFjBAgMBAAGjYzBhMB0GA1UdDgQWBBREnkj1zG1I1KBLf/5ZJC+Dl5mahjAf +BgNVHSMEGDAWgBREnkj1zG1I1KBLf/5ZJC+Dl5mahjAPBgNVHRMBAf8EBTADAQH/ +MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAQEABT41XBVwm8nHc2Fv +civUwo/yQ10CzsSUuZQRg2dd4mdsdXa/uwyqNsatR5Nj3B5+1t4u/ukZMjgDfxT2 +AHMsWbEhBuH7rBiVDKP/mZb3Kyeb1STMHd3BOuCYRLDE5D53sXOpZCz2HAF8P11F +hcCF5yWPldwX8zyfGm6wyuMdKulMY/okYWLW2n62HGz1Ah3UKt1VkOsqEUc8Ll50 +soIipX1TH0XsJ5F95yIW6MBoNtjG8U+ARDL54dHRHareqKucBK+tIA5kmE2la8BI +WJZpTdwHjFGTot+fDz2LYLSCjaoITmJF4PkL0uDgPFveXHEnJcLmA4GLEFPjx1Wi +tJ/X5g== +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com Root Certification Authority RSA O=SSL Corporation +# Subject: CN=SSL.com Root Certification Authority RSA O=SSL Corporation +# Label: "SSL.com Root Certification Authority RSA" +# Serial: 8875640296558310041 +# MD5 Fingerprint: 86:69:12:c0:70:f1:ec:ac:ac:c2:d5:bc:a5:5b:a1:29 +# SHA1 Fingerprint: b7:ab:33:08:d1:ea:44:77:ba:14:80:12:5a:6f:bd:a9:36:49:0c:bb +# SHA256 Fingerprint: 85:66:6a:56:2e:e0:be:5c:e9:25:c1:d8:89:0a:6f:76:a8:7e:c1:6d:4d:7d:5f:29:ea:74:19:cf:20:12:3b:69 +-----BEGIN CERTIFICATE----- +MIIF3TCCA8WgAwIBAgIIeyyb0xaAMpkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UE +BhMCVVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQK +DA9TU0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZp +Y2F0aW9uIEF1dGhvcml0eSBSU0EwHhcNMTYwMjEyMTczOTM5WhcNNDEwMjEyMTcz +OTM5WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv +dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNv +bSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFJTQTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAPkP3aMrfcvQKv7sZ4Wm5y4bunfh4/WvpOz6Sl2R +xFdHaxh3a3by/ZPkPQ/CFp4LZsNWlJ4Xg4XOVu/yFv0AYvUiCVToZRdOQbngT0aX +qhvIuG5iXmmxX9sqAn78bMrzQdjt0Oj8P2FI7bADFB0QDksZ4LtO7IZl/zbzXmcC +C52GVWH9ejjt/uIZALdvoVBidXQ8oPrIJZK0bnoix/geoeOy3ZExqysdBP+lSgQ3 +6YWkMyv94tZVNHwZpEpox7Ko07fKoZOI68GXvIz5HdkihCR0xwQ9aqkpk8zruFvh +/l8lqjRYyMEjVJ0bmBHDOJx+PYZspQ9AhnwC9FwCTyjLrnGfDzrIM/4RJTXq/LrF +YD3ZfBjVsqnTdXgDciLKOsMf7yzlLqn6niy2UUb9rwPW6mBo6oUWNmuF6R7As93E +JNyAKoFBbZQ+yODJgUEAnl6/f8UImKIYLEJAs/lvOCdLToD0PYFH4Ih86hzOtXVc +US4cK38acijnALXRdMbX5J+tB5O2UzU1/Dfkw/ZdFr4hc96SCvigY2q8lpJqPvi8 +ZVWb3vUNiSYE/CUapiVpy8JtynziWV+XrOvvLsi81xtZPCvM8hnIk2snYxnP/Okm ++Mpxm3+T/jRnhE6Z6/yzeAkzcLpmpnbtG3PrGqUNxCITIJRWCk4sbE6x/c+cCbqi +M+2HAgMBAAGjYzBhMB0GA1UdDgQWBBTdBAkHovV6fVJTEpKV7jiAJQ2mWTAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFN0ECQei9Xp9UlMSkpXuOIAlDaZZMA4G +A1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAIBgRlCn7Jp0cHh5wYfGV +cpNxJK1ok1iOMq8bs3AD/CUrdIWQPXhq9LmLpZc7tRiRux6n+UBbkflVma8eEdBc +Hadm47GUBwwyOabqG7B52B2ccETjit3E+ZUfijhDPwGFpUenPUayvOUiaPd7nNgs +PgohyC0zrL/FgZkxdMF1ccW+sfAjRfSda/wZY52jvATGGAslu1OJD7OAUN5F7kR/ +q5R4ZJjT9ijdh9hwZXT7DrkT66cPYakylszeu+1jTBi7qUD3oFRuIIhxdRjqerQ0 +cuAjJ3dctpDqhiVAq+8zD8ufgr6iIPv2tS0a5sKFsXQP+8hlAqRSAUfdSSLBv9jr +a6x+3uxjMxW3IwiPxg+NQVrdjsW5j+VFP3jbutIbQLH+cU0/4IGiul607BXgk90I +H37hVZkLId6Tngr75qNJvTYw/ud3sqB1l7UtgYgXZSD32pAAn8lSzDLKNXz1PQ/Y +K9f1JmzJBjSWFupwWRoyeXkLtoh/D1JIPb9s2KJELtFOt3JY04kTlf5Eq/jXixtu +nLwsoFvVagCvXzfh1foQC5ichucmj87w7G6KVwuA406ywKBjYZC6VWg3dGq2ktuf +oYYitmUnDuy2n0Jg5GfCtdpBC8TTi2EbvPofkSvXRAdeuims2cXp71NIWuuA8ShY +Ic2wBlX7Jz9TkHCpBB5XJ7k= +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com Root Certification Authority ECC O=SSL Corporation +# Subject: CN=SSL.com Root Certification Authority ECC O=SSL Corporation +# Label: "SSL.com Root Certification Authority ECC" +# Serial: 8495723813297216424 +# MD5 Fingerprint: 2e:da:e4:39:7f:9c:8f:37:d1:70:9f:26:17:51:3a:8e +# SHA1 Fingerprint: c3:19:7c:39:24:e6:54:af:1b:c4:ab:20:95:7a:e2:c3:0e:13:02:6a +# SHA256 Fingerprint: 34:17:bb:06:cc:60:07:da:1b:96:1c:92:0b:8a:b4:ce:3f:ad:82:0e:4a:a3:0b:9a:cb:c4:a7:4e:bd:ce:bc:65 +-----BEGIN CERTIFICATE----- +MIICjTCCAhSgAwIBAgIIdebfy8FoW6gwCgYIKoZIzj0EAwIwfDELMAkGA1UEBhMC +VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T +U0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZpY2F0 +aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNDAzWhcNNDEwMjEyMTgxNDAz +WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hvdXN0 +b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNvbSBS +b290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49AgEGBSuB +BAAiA2IABEVuqVDEpiM2nl8ojRfLliJkP9x6jh3MCLOicSS6jkm5BBtHllirLZXI +7Z4INcgn64mMU1jrYor+8FsPazFSY0E7ic3s7LaNGdM0B9y7xgZ/wkWV7Mt/qCPg +CemB+vNH06NjMGEwHQYDVR0OBBYEFILRhXMw5zUE044CkvvlpNHEIejNMA8GA1Ud +EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUgtGFczDnNQTTjgKS++Wk0cQh6M0wDgYD +VR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2cAMGQCMG/n61kRpGDPYbCWe+0F+S8T +kdzt5fxQaxFGRrMcIQBiu77D5+jNB5n5DQtdcj7EqgIwH7y6C+IwJPt8bYBVCpk+ +gA0z5Wajs6O7pdWLjwkspl1+4vAHCGht0nxpbl/f5Wpl +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation +# Subject: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation +# Label: "SSL.com EV Root Certification Authority RSA R2" +# Serial: 6248227494352943350 +# MD5 Fingerprint: e1:1e:31:58:1a:ae:54:53:02:f6:17:6a:11:7b:4d:95 +# SHA1 Fingerprint: 74:3a:f0:52:9b:d0:32:a0:f4:4a:83:cd:d4:ba:a9:7b:7c:2e:c4:9a +# SHA256 Fingerprint: 2e:7b:f1:6c:c2:24:85:a7:bb:e2:aa:86:96:75:07:61:b0:ae:39:be:3b:2f:e9:d0:cc:6d:4e:f7:34:91:42:5c +-----BEGIN CERTIFICATE----- +MIIF6zCCA9OgAwIBAgIIVrYpzTS8ePYwDQYJKoZIhvcNAQELBQAwgYIxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4GA1UEBwwHSG91c3RvbjEYMBYGA1UE +CgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQDDC5TU0wuY29tIEVWIFJvb3QgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIyMB4XDTE3MDUzMTE4MTQzN1oXDTQy +MDUzMDE4MTQzN1owgYIxCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4G +A1UEBwwHSG91c3RvbjEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQD +DC5TU0wuY29tIEVWIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIy +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAjzZlQOHWTcDXtOlG2mvq +M0fNTPl9fb69LT3w23jhhqXZuglXaO1XPqDQCEGD5yhBJB/jchXQARr7XnAjssuf +OePPxU7Gkm0mxnu7s9onnQqG6YE3Bf7wcXHswxzpY6IXFJ3vG2fThVUCAtZJycxa +4bH3bzKfydQ7iEGonL3Lq9ttewkfokxykNorCPzPPFTOZw+oz12WGQvE43LrrdF9 +HSfvkusQv1vrO6/PgN3B0pYEW3p+pKk8OHakYo6gOV7qd89dAFmPZiw+B6KjBSYR +aZfqhbcPlgtLyEDhULouisv3D5oi53+aNxPN8k0TayHRwMwi8qFG9kRpnMphNQcA +b9ZhCBHqurj26bNg5U257J8UZslXWNvNh2n4ioYSA0e/ZhN2rHd9NCSFg83XqpyQ +Gp8hLH94t2S42Oim9HizVcuE0jLEeK6jj2HdzghTreyI/BXkmg3mnxp3zkyPuBQV +PWKchjgGAGYS5Fl2WlPAApiiECtoRHuOec4zSnaqW4EWG7WK2NAAe15itAnWhmMO +pgWVSbooi4iTsjQc2KRVbrcc0N6ZVTsj9CLg+SlmJuwgUHfbSguPvuUCYHBBXtSu +UDkiFCbLsjtzdFVHB3mBOagwE0TlBIqulhMlQg+5U8Sb/M3kHN48+qvWBkofZ6aY +MBzdLNvcGJVXZsb/XItW9XcCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNV +HSMEGDAWgBT5YLvU49U09rj1BoAlp3PbRmmonjAdBgNVHQ4EFgQU+WC71OPVNPa4 +9QaAJadz20ZpqJ4wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBW +s47LCp1Jjr+kxJG7ZhcFUZh1++VQLHqe8RT6q9OKPv+RKY9ji9i0qVQBDb6Thi/5 +Sm3HXvVX+cpVHBK+Rw82xd9qt9t1wkclf7nxY/hoLVUE0fKNsKTPvDxeH3jnpaAg +cLAExbf3cqfeIg29MyVGjGSSJuM+LmOW2puMPfgYCdcDzH2GguDKBAdRUNf/ktUM +79qGn5nX67evaOI5JpS6aLe/g9Pqemc9YmeuJeVy6OLk7K4S9ksrPJ/psEDzOFSz +/bdoyNrGj1E8svuR3Bznm53htw1yj+KkxKl4+esUrMZDBcJlOSgYAsOCsp0FvmXt +ll9ldDz7CTUue5wT/RsPXcdtgTpWD8w74a8CLyKsRspGPKAcTNZEtF4uXBVmCeEm +Kf7GUmG6sXP/wwyc5WxqlD8UykAWlYTzWamsX0xhk23RO8yilQwipmdnRC652dKK +QbNmC1r7fSOl8hqw/96bg5Qu0T/fkreRrwU7ZcegbLHNYhLDkBvjJc40vG93drEQ +w/cFGsDWr3RiSBd3kmmQYRzelYB0VI8YHMPzA9C/pEN1hlMYegouCRw2n5H9gooi +S9EOUCXdywMMF8mDAAhONU2Ki+3wApRmLER/y5UnlhetCTCstnEXbosX9hwJ1C07 +mKVx01QT2WDz9UtmT/rx7iASjbSsV7FFY6GsdqnC+w== +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation +# Subject: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation +# Label: "SSL.com EV Root Certification Authority ECC" +# Serial: 3182246526754555285 +# MD5 Fingerprint: 59:53:22:65:83:42:01:54:c0:ce:42:b9:5a:7c:f2:90 +# SHA1 Fingerprint: 4c:dd:51:a3:d1:f5:20:32:14:b0:c6:c5:32:23:03:91:c7:46:42:6d +# SHA256 Fingerprint: 22:a2:c1:f7:bd:ed:70:4c:c1:e7:01:b5:f4:08:c3:10:88:0f:e9:56:b5:de:2a:4a:44:f9:9c:87:3a:25:a7:c8 +-----BEGIN CERTIFICATE----- +MIIClDCCAhqgAwIBAgIILCmcWxbtBZUwCgYIKoZIzj0EAwIwfzELMAkGA1UEBhMC +VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T +U0wgQ29ycG9yYXRpb24xNDAyBgNVBAMMK1NTTC5jb20gRVYgUm9vdCBDZXJ0aWZp +Y2F0aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNTIzWhcNNDEwMjEyMTgx +NTIzWjB/MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv +dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjE0MDIGA1UEAwwrU1NMLmNv +bSBFViBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49 +AgEGBSuBBAAiA2IABKoSR5CYG/vvw0AHgyBO8TCCogbR8pKGYfL2IWjKAMTH6kMA +VIbc/R/fALhBYlzccBYy3h+Z1MzFB8gIH2EWB1E9fVwHU+M1OIzfzZ/ZLg1Kthku +WnBaBu2+8KGwytAJKaNjMGEwHQYDVR0OBBYEFFvKXuXe0oGqzagtZFG22XKbl+ZP +MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX +5k8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2gAMGUCMQCK5kCJN+vp1RPZ +ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg +h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFZTCCA02gAwIBAgIKUlD06gAAAAAAGDANBgkqhkiG9w0BAQ0FADAfMR0wGwYD +VQQDExRZYW5kZXhJbnRlcm5hbFJvb3RDQTAeFw0xODA2MjgxMTE0NTdaFw0zMjA2 +MjgxMTI0NTdaMFsxEjAQBgoJkiaJk/IsZAEZFgJydTEWMBQGCgmSJomT8ixkARkW +BnlhbmRleDESMBAGCgmSJomT8ixkARkWAmxkMRkwFwYDVQQDExBZYW5kZXhJbnRl +cm5hbENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy6Sab1PCbISk +GSAUpr6JJKLXlf4O+cBhjALfQn2QpPL/cDjZ2+MPXuAUgE8KT+/mbAGA2rJID0KY +RjDSkByxnhoX8jwWsmPYXoAmOMPkgKRG9/ZefnMrK4oVhGgLmxnpbEkNbGh88cJ1 +OVzgD5LVHSpDqm7iEuoUPOJCWXQ51+rZ0Lw9zBEU8v3yXXI345iWpLj92pOQDH0G +Tqr7BnQywxcgb5BYdywayacIT7UTJZk7832m5k7Oa3qMIKKXHsx26rNVUVBfpzph +OFvqkLetOKHk7827NDKr3I3OFXzQk4gy6tagv8PZNp+XGOBWfYkbLfI4xbTnjHIW +n5q1gfKPOQIDAQABo4IBZTCCAWEwEAYJKwYBBAGCNxUBBAMCAQIwIwYJKwYBBAGC +NxUCBBYEFNgaef9LcdQKs6qfsfiuWF5p/yqRMB0GA1UdDgQWBBSP3TKDCRNT3ZEa +Zumz1DzFtPJnSDBZBgNVHSAEUjBQME4GBFUdIAAwRjBEBggrBgEFBQcCARY4aHR0 +cDovL2NybHMueWFuZGV4LnJ1L2Nwcy9ZYW5kZXhJbnRlcm5hbENBL3BvbGljaWVz +Lmh0bWwwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8G +A1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUq7nF/6Hv5lMdMzkihNF21DdOLWow +VAYDVR0fBE0wSzBJoEegRYZDaHR0cDovL2NybHMueWFuZGV4LnJ1L1lhbmRleElu +dGVybmFsUm9vdENBL1lhbmRleEludGVybmFsUm9vdENBLmNybDANBgkqhkiG9w0B +AQ0FAAOCAgEAQnOiyykjwtSuCBV6rSiM8Q1rQIcfyqn1JBxSGeBMABc64loWSPaQ +DtYPIW5rwNX7TQ94bjyYgCxhwHqUED/fcBOmXCQ2iBsdy5LOcNEZaC2kBHQuZ7dL +0fSvpE98a41y9yY6CJGFXg8E/4GrQwgQEqT5Qbe9GHPadpRu+ptVvI6uLZG3ks2o +oodjOm5C0SIo1pY4OtPAYE/AzTaYkTFbAqYcPfEfXHEOigBJBeXnQs7cANxX/RaF +PnHEjZbGY57EtBP6p5ckndkfEmqp3PLXbsQteNOVpsUw5eVqEzinSisBmLc28nnr +5QEojRontAaZd7ZzB5zaGkVuE+0laUUWSNBhfGE1R3LrTJEK9L7FEsBBprOxIWww +CvLmAfglouwuNRc2TjRdfnZaEfPLD7NYIF4ahXPAMcfTii23Tlr2uB7LetNykSlX +Z9S5/yf61VFEKnxuipFPNgtKqPcFgFUxlEb+wOeOfYZ7ex8VlpMBWbadj3Go025b +KZUwKwHDQvgJ5pz9g3t+t5Xieu2pwyddWGu+1SItRohRhlyTiep7oW6yTps7Qt0e +8pdLuLG7ZF19h1Pxi+dVbeaeNcsGEAOdRuCk+RTZHNe+J4yC8tNJOepnfYDul6SB +RjFWthiFK45+TZRHAcsG9JuV8JNvgoKaL75v/GUsKaeJ3Cps3rBStfc= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFGTCCAwGgAwIBAgIQJMM7ZIy2SYxCBgK7WcFwnjANBgkqhkiG9w0BAQ0FADAf +MR0wGwYDVQQDExRZYW5kZXhJbnRlcm5hbFJvb3RDQTAeFw0xMzAyMTExMzQxNDNa +Fw0zMzAyMTExMzUxNDJaMB8xHTAbBgNVBAMTFFlhbmRleEludGVybmFsUm9vdENB +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAgb4xoQjBQ7oEFk8EHVGy +1pDEmPWw0Wgw5nX9RM7LL2xQWyUuEq+Lf9Dgh+O725aZ9+SO2oEs47DHHt81/fne +5N6xOftRrCpy8hGtUR/A3bvjnQgjs+zdXvcO9cTuuzzPTFSts/iZATZsAruiepMx +SGj9S1fGwvYws/yiXWNoNBz4Tu1Tlp0g+5fp/ADjnxc6DqNk6w01mJRDbx+6rlBO +aIH2tQmJXDVoFdrhmBK9qOfjxWlIYGy83TnrvdXwi5mKTMtpEREMgyNLX75UjpvO +NkZgBvEXPQq+g91wBGsWIE2sYlguXiBniQgAJOyRuSdTxcJoG8tZkLDPRi5RouWY +gxXr13edn1TRDGco2hkdtSUBlajBMSvAq+H0hkslzWD/R+BXkn9dh0/DFnxVt4XU +5JbFyd/sKV/rF4Vygfw9ssh1ZIWdqkfZ2QXOZ2gH4AEeoN/9vEfUPwqPVzL0XEZK +r4s2WjU9mE5tHrVsQOZ80wnvYHYi2JHbl0hr5ghs4RIyJwx6LEEnj2tzMFec4f7o +dQeSsZpgRJmpvpAfRTxhIRjZBrKxnMytedAkUPguBQwjVCn7+EaKiJfpu42JG8Mm ++/dHi+Q9Tc+0tX5pKOIpQMlMxMHw8MfPmUjC3AAd9lsmCtuybYoeN2IRdbzzchJ8 +l1ZuoI3gH7pcIeElfVSqSBkCAwEAAaNRME8wCwYDVR0PBAQDAgGGMA8GA1UdEwEB +/wQFMAMBAf8wHQYDVR0OBBYEFKu5xf+h7+ZTHTM5IoTRdtQ3Ti1qMBAGCSsGAQQB +gjcVAQQDAgEAMA0GCSqGSIb3DQEBDQUAA4ICAQAVpyJ1qLjqRLC34F1UXkC3vxpO +nV6WgzpzA+DUNog4Y6RhTnh0Bsir+I+FTl0zFCm7JpT/3NP9VjfEitMkHehmHhQK +c7cIBZSF62K477OTvLz+9ku2O/bGTtYv9fAvR4BmzFfyPDoAKOjJSghD1p/7El+1 +eSjvcUBzLnBUtxO/iYXRNo7B3+1qo4F5Hz7rPRLI0UWW/0UAfVCO2fFtyF6C1iEY +/q0Ldbf3YIaMkf2WgGhnX9yH/8OiIij2r0LVNHS811apyycjep8y/NkG4q1Z9jEi +VEX3P6NEL8dWtXQlvlNGMcfDT3lmB+tS32CPEUwce/Ble646rukbERRwFfxXojpf +C6ium+LtJc7qnK6ygnYF4D6mz4H+3WaxJd1S1hGQxOb/3WVw63tZFnN62F6/nc5g +6T44Yb7ND6y3nVcygLpbQsws6HsjX65CoSjrrPn0YhKxNBscF7M7tLTW/5LK9uhk +yjRCkJ0YagpeLxfV1l1ZJZaTPZvY9+ylHnWHhzlq0FzcrooSSsp4i44DB2K7O2ID +87leymZkKUY6PMDa4GkDJx0dG4UXDhRETMf+NkYgtLJ+UIzMNskwVDcxO4kVL+Hi +Pj78bnC5yCw8P5YylR45LdxLzLO68unoXOyFz1etGXzszw8lJI9LNubYxk77mK8H +LpuQKbSbIERsmR+QqQ== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/resources/ydb_sdk_version.txt b/ydb/public/sdk/cpp/src/client/resources/ydb_sdk_version.txt new file mode 100644 index 000000000000..56fea8a08d2f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/resources/ydb_sdk_version.txt @@ -0,0 +1 @@ +3.0.0 \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/result/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/result/CMakeLists.txt new file mode 100644 index 000000000000..9779b1bc4892 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/result/CMakeLists.txt @@ -0,0 +1,17 @@ +_ydb_sdk_add_library(client-ydb_result) + +target_link_libraries(client-ydb_result PUBLIC + yutil + api-protos + client-ydb_types-fatal_error_handlers + client-ydb_value + client-ydb_proto +) + +target_sources(client-ydb_result PRIVATE + proto_accessor.cpp + result.cpp + out.cpp +) + +_ydb_sdk_make_client_component(Result client-ydb_result) diff --git a/ydb/public/sdk/cpp/src/client/result/out.cpp b/ydb/public/sdk/cpp/src/client/result/out.cpp new file mode 100644 index 000000000000..f10426082f75 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/result/out.cpp @@ -0,0 +1,5 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::TColumn, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/result/proto_accessor.cpp b/ydb/public/sdk/cpp/src/client/result/proto_accessor.cpp new file mode 100644 index 000000000000..3e5e35018ef3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/result/proto_accessor.cpp @@ -0,0 +1,11 @@ +#include + +#include + +namespace NYdb::inline V3 { + +const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) { + return resultSet.GetProto(); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/result/result.cpp b/ydb/public/sdk/cpp/src/client/result/result.cpp new file mode 100644 index 000000000000..7770cde0c5a4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/result/result.cpp @@ -0,0 +1,235 @@ +#include + +#include + +#include +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { + +std::string TColumn::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TColumn::Out(IOutputStream& o) const { + o << "{ name: \"" << Name << "\"" + << ", type: " << Type + << " }"; +} + +bool operator==(const TColumn& col1, const TColumn& col2) { + return col1.Name == col2.Name && TypesEqual(col1.Type, col2.Type); +} + +bool operator!=(const TColumn& col1, const TColumn& col2) { + return !(col1 == col2); +} + +class TResultSet::TImpl { +public: + TImpl(const Ydb::ResultSet& proto) + : ProtoResultSet_(proto) + { + Init(); + } + + TImpl(Ydb::ResultSet&& proto) + : ProtoResultSet_(std::move(proto)) + { + Init(); + } + + void Init() { + ColumnsMeta_.reserve(ProtoResultSet_.columns_size()); + for (auto& meta : ProtoResultSet_.columns()) { + ColumnsMeta_.push_back(TColumn(meta.name(), TType(meta.type()))); + } + } + +public: + const Ydb::ResultSet ProtoResultSet_; + std::vector ColumnsMeta_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TResultSet::TResultSet(const Ydb::ResultSet& proto) + : Impl_(new TResultSet::TImpl(proto)) {} + +TResultSet::TResultSet(Ydb::ResultSet&& proto) + : Impl_(new TResultSet::TImpl(std::move(proto))) {} + +size_t TResultSet::ColumnsCount() const { + return Impl_->ColumnsMeta_.size(); +} + +size_t TResultSet::RowsCount() const { + return Impl_->ProtoResultSet_.rows_size(); +} + +bool TResultSet::Truncated() const { + return Impl_->ProtoResultSet_.truncated(); +} + +const std::vector& TResultSet::GetColumnsMeta() const { + return Impl_->ColumnsMeta_; +} + +const Ydb::ResultSet& TResultSet::GetProto() const { + return Impl_->ProtoResultSet_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TResultSetParser::TImpl { +public: + TImpl(const TResultSet& resultSet) + : ResultSet_(resultSet) + { + ColumnParsers.reserve(resultSet.ColumnsCount()); + + auto& columnsMeta = resultSet.GetColumnsMeta(); + for (size_t i = 0; i < columnsMeta.size(); ++i) { + auto& column = columnsMeta[i]; + ColumnIndexMap[column.Name] = i; + ColumnParsers.emplace_back(column.Type); + } + } + + size_t ColumnsCount() const { + return ResultSet_.ColumnsCount(); + } + + size_t RowsCount() const { + return ResultSet_.RowsCount(); + } + + bool TryNextRow() { + if (RowIndex_ == ResultSet_.RowsCount()) { + return false; + } + + auto& row = ResultSet_.GetProto().rows()[RowIndex_]; + + if (static_cast(row.items_size()) != ColumnsCount()) { + FatalError(TStringBuilder() << "Corrupted data: row " << RowIndex_ << " contains " << row.items_size() << " column(s), but metadata contains " << ColumnsCount() << " column(s)"); + } + + for (size_t i = 0; i < ColumnsCount(); ++i) { + ColumnParsers[i].Reset(row.items(i)); + } + + RowIndex_++; + return true; + } + + ssize_t ColumnIndex(const std::string& columnName) { + auto idx = MapFindPtr(ColumnIndexMap, columnName); + return idx ? static_cast(*idx) : -1; + } + + TValueParser& ColumnParser(size_t columnIndex) { + if (columnIndex >= ColumnParsers.size()) { + FatalError(TStringBuilder() << "Column index out of bounds: " << columnIndex); + } + + return ColumnParsers[columnIndex]; + } + + TValueParser& ColumnParser(const std::string& columnName) { + auto idx = MapFindPtr(ColumnIndexMap, columnName); + if (!idx) { + FatalError(TStringBuilder() << "Unknown column: " << columnName); + } + + return ColumnParser(*idx); + } + + TValue GetValue(size_t columnIndex) const { + if (columnIndex >= ColumnParsers.size()) { + FatalError(TStringBuilder() << "Column index out of bounds: " << columnIndex); + } + + if (RowIndex_ == 0) { + FatalError(TStringBuilder() << "Row position is undefined"); + } + + const auto& row = ResultSet_.GetProto().rows()[RowIndex_ - 1]; + const auto& valueType = ResultSet_.GetColumnsMeta()[columnIndex].Type; + + return TValue(valueType, row.items(columnIndex)); + } + + TValue GetValue(const std::string& columnName) const { + auto idx = MapFindPtr(ColumnIndexMap, columnName); + if (!idx) { + FatalError(TStringBuilder() << "Unknown column: " << columnName); + } + + return GetValue(*idx); + } + +private: + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TResultSetParser: " << msg); + } + +private: + TResultSet ResultSet_; + + std::map ColumnIndexMap; + std::vector ColumnParsers; + + size_t RowIndex_ = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TResultSetParser::TResultSetParser(TResultSetParser&&) = default; +TResultSetParser::~TResultSetParser() = default; + +TResultSetParser::TResultSetParser(const TResultSet& resultSet) + : Impl_(new TImpl(resultSet)) {} + +size_t TResultSetParser::ColumnsCount() const { + return Impl_->ColumnsCount(); +} + +size_t TResultSetParser::RowsCount() const { + return Impl_->RowsCount(); +} + +bool TResultSetParser::TryNextRow() { + return Impl_->TryNextRow(); +} + +ssize_t TResultSetParser::ColumnIndex(const std::string& columnName) { + return Impl_->ColumnIndex(columnName); +} + +TValueParser& TResultSetParser::ColumnParser(size_t columnIndex) { + return Impl_->ColumnParser(columnIndex); +} + +TValueParser& TResultSetParser::ColumnParser(const std::string& columnName) { + return Impl_->ColumnParser(columnName); +} + +TValue TResultSetParser::GetValue(size_t columnIndex) const { + return Impl_->GetValue(columnIndex); +} + +TValue TResultSetParser::GetValue(const std::string& columnName) const { + return Impl_->GetValue(columnName); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/result/ya.make b/ydb/public/sdk/cpp/src/client/result/ya.make new file mode 100644 index 000000000000..0cc590d2d906 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/result/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + out.cpp + proto_accessor.cpp + result.cpp +) + +PEERDIR( + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/types/fatal_error_handlers + ydb/public/sdk/cpp/src/client/value + ydb/public/sdk/cpp/src/client/proto +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/scheme/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/scheme/CMakeLists.txt new file mode 100644 index 000000000000..184e96bfdd67 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/scheme/CMakeLists.txt @@ -0,0 +1,22 @@ +_ydb_sdk_add_library(client-ydb_scheme) + +target_link_libraries(client-ydb_scheme PUBLIC + yutil + enum_serialization_runtime + impl-ydb_internal-make_request + client-ydb_common_client-impl + client-ydb_driver +) + +target_sources(client-ydb_scheme PRIVATE + scheme.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_scheme + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/scheme/scheme.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/scheme/scheme.h +) + +_ydb_sdk_make_client_component(Scheme client-ydb_scheme) diff --git a/ydb/public/sdk/cpp/src/client/scheme/out.cpp b/ydb/public/sdk/cpp/src/client/scheme/out.cpp new file mode 100644 index 000000000000..9a9f899b4615 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/scheme/out.cpp @@ -0,0 +1,17 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::NScheme::TVirtualTimestamp, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NScheme::TSchemeEntry, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NScheme::TDescribePathResult, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NScheme::TListDirectoryResult, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp b/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp new file mode 100644 index 000000000000..9b0613c884a0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp @@ -0,0 +1,368 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NScheme { + +using namespace NThreading; +using namespace Ydb::Scheme; + +TPermissions::TPermissions(const ::Ydb::Scheme::Permissions& proto) + : Subject(proto.subject()) + , PermissionNames(proto.permission_names().begin(), proto.permission_names().end()) +{} + +void TPermissions::SerializeTo(::Ydb::Scheme::Permissions& proto) const { + proto.set_subject(TStringType{Subject}); + for (const auto& name : PermissionNames) { + proto.add_permission_names(TStringType{name}); + } +} + +TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) + : PlanStep(planStep) + , TxId(txId) +{} + +TVirtualTimestamp::TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto) + : TVirtualTimestamp(proto.plan_step(), proto.tx_id()) +{} + +std::string TVirtualTimestamp::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TVirtualTimestamp::Out(IOutputStream& out) const { + out << "{ plan_step: " << PlanStep + << ", tx_id: " << TxId + << " }"; +} + +bool TVirtualTimestamp::operator<(const TVirtualTimestamp& rhs) const { + return PlanStep < rhs.PlanStep && TxId < rhs.TxId; +} + +bool TVirtualTimestamp::operator<=(const TVirtualTimestamp& rhs) const { + return PlanStep <= rhs.PlanStep && TxId <= rhs.TxId; +} + +bool TVirtualTimestamp::operator>(const TVirtualTimestamp& rhs) const { + return PlanStep > rhs.PlanStep && TxId > rhs.TxId; +} + +bool TVirtualTimestamp::operator>=(const TVirtualTimestamp& rhs) const { + return PlanStep >= rhs.PlanStep && TxId >= rhs.TxId; +} + +bool TVirtualTimestamp::operator==(const TVirtualTimestamp& rhs) const { + return PlanStep == rhs.PlanStep && TxId == rhs.TxId; +} + +bool TVirtualTimestamp::operator!=(const TVirtualTimestamp& rhs) const { + return !(*this == rhs); +} + +static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) { + switch (entry) { + case ::Ydb::Scheme::Entry::DIRECTORY: + return ESchemeEntryType::Directory; + case ::Ydb::Scheme::Entry::TABLE: + return ESchemeEntryType::Table; + case ::Ydb::Scheme::Entry::COLUMN_TABLE: + return ESchemeEntryType::ColumnTable; + case ::Ydb::Scheme::Entry::PERS_QUEUE_GROUP: + return ESchemeEntryType::PqGroup; + case ::Ydb::Scheme::Entry::DATABASE: + return ESchemeEntryType::SubDomain; + case ::Ydb::Scheme::Entry::RTMR_VOLUME: + return ESchemeEntryType::RtmrVolume; + case ::Ydb::Scheme::Entry::BLOCK_STORE_VOLUME: + return ESchemeEntryType::BlockStoreVolume; + case ::Ydb::Scheme::Entry::COORDINATION_NODE: + return ESchemeEntryType::CoordinationNode; + case ::Ydb::Scheme::Entry::SEQUENCE: + return ESchemeEntryType::Sequence; + case ::Ydb::Scheme::Entry::REPLICATION: + return ESchemeEntryType::Replication; + case ::Ydb::Scheme::Entry::TOPIC: + return ESchemeEntryType::Topic; + case ::Ydb::Scheme::Entry::COLUMN_STORE: + return ESchemeEntryType::ColumnStore; + case ::Ydb::Scheme::Entry::EXTERNAL_TABLE: + return ESchemeEntryType::ExternalTable; + case ::Ydb::Scheme::Entry::EXTERNAL_DATA_SOURCE: + return ESchemeEntryType::ExternalDataSource; + case ::Ydb::Scheme::Entry::VIEW: + return ESchemeEntryType::View; + case ::Ydb::Scheme::Entry::RESOURCE_POOL: + return ESchemeEntryType::ResourcePool; + default: + return ESchemeEntryType::Unknown; + } +} + +TSchemeEntry::TSchemeEntry(const ::Ydb::Scheme::Entry& proto) + : Name(proto.name()) + , Owner(proto.owner()) + , Type(ConvertProtoEntryType(proto.type())) + , SizeBytes(proto.size_bytes()) + , CreatedAt(proto.created_at()) +{ + PermissionToSchemeEntry(proto.effective_permissions(), &EffectivePermissions); + PermissionToSchemeEntry(proto.permissions(), &Permissions); +} + +void TSchemeEntry::Out(IOutputStream& out) const { + out << "{ name: " << Name + << ", owner: " << Owner + << ", type: " << Type + << ", size_bytes: " << SizeBytes + << ", created_at: " << CreatedAt + << " }"; +} + +void TSchemeEntry::SerializeTo(::Ydb::Scheme::ModifyPermissionsRequest& request) const { + request.mutable_actions()->Add()->set_change_owner(TStringType{Owner}); + for (const auto& permission : Permissions) { + permission.SerializeTo(*request.mutable_actions()->Add()->mutable_grant()); + } +} + +TModifyPermissionsSettings::TModifyPermissionsSettings(const ::Ydb::Scheme::ModifyPermissionsRequest& request) { + for (const auto& action : request.actions()) { + switch (action.action_case()) { + case Ydb::Scheme::PermissionsAction::kGrant: + AddGrantPermissions(action.grant()); + break; + case Ydb::Scheme::PermissionsAction::kRevoke: + AddRevokePermissions(action.revoke()); + break; + case Ydb::Scheme::PermissionsAction::kSet: + AddSetPermissions(action.set()); + break; + case Ydb::Scheme::PermissionsAction::kChangeOwner: + AddChangeOwner(action.change_owner()); + break; + case Ydb::Scheme::PermissionsAction::ACTION_NOT_SET: + break; + } + } +} + +class TSchemeClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) {} + + TAsyncStatus MakeDirectory(const std::string& path, const TMakeDirectorySettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + return RunSimple( + std::move(request), + &Ydb::Scheme::V1::SchemeService::Stub::AsyncMakeDirectory, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveDirectory(const std::string& path, const TRemoveDirectorySettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + return RunSimple( + std::move(request), + &Ydb::Scheme::V1::SchemeService::Stub::AsyncRemoveDirectory, + TRpcRequestSettings::Make(settings)); + } + + TAsyncDescribePathResult DescribePath(const std::string& path, const TDescribePathSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + DescribePathResult result; + if (any) { + any->UnpackTo(&result); + } + + promise.SetValue(TDescribePathResult(TStatus(std::move(status)), result.self())); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Scheme::V1::SchemeService::Stub::AsyncDescribePath, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncListDirectoryResult ListDirectory(const std::string& path, const TListDirectorySettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + ListDirectoryResult result; + if (any) { + any->UnpackTo(&result); + } + + std::vector children; + children.reserve(result.children().size()); + for (const auto& child : result.children()) { + children.emplace_back(child); + } + + promise.SetValue(TListDirectoryResult(TStatus(std::move(status)), result.self(), std::move(children))); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Scheme::V1::SchemeService::Stub::AsyncListDirectory, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + void PermissionsToRequest(const TPermissions& permissions, Permissions* to) { + to->set_subject(TStringType{permissions.Subject}); + for (const auto& perm : permissions.PermissionNames) { + to->add_permission_names(TStringType{perm}); + } + } + + TAsyncStatus ModifyPermissions(const std::string& path, const TModifyPermissionsSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + if (settings.ClearAcl_) { + request.set_clear_permissions(true); + } + if (settings.SetInterruptInheritance_) { + request.set_interrupt_inheritance(settings.InterruptInheritanceValue_); + } + + for (const auto& action : settings.Actions_) { + auto protoAction = request.add_actions(); + switch (action.first) { + case EModifyPermissionsAction::Chown: { + protoAction->set_change_owner(TStringType{action.second.Subject}); + } + break; + case EModifyPermissionsAction::Grant: { + PermissionsToRequest(action.second, protoAction->mutable_grant()); + } + break; + case EModifyPermissionsAction::Revoke: { + PermissionsToRequest(action.second, protoAction->mutable_revoke()); + } + break; + case EModifyPermissionsAction::Set: { + PermissionsToRequest(action.second, protoAction->mutable_set()); + } + break; + } + } + + return RunSimple( + std::move(request), + &Ydb::Scheme::V1::SchemeService::Stub::AsyncModifyPermissions, + TRpcRequestSettings::Make(settings)); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TDescribePathResult::TDescribePathResult(TStatus&& status, const TSchemeEntry& entry) + : TStatus(std::move(status)) + , Entry_(entry) +{} + +const TSchemeEntry& TDescribePathResult::GetEntry() const { + CheckStatusOk("TDescribePathResult::GetEntry"); + return Entry_; +} + +void TDescribePathResult::Out(IOutputStream& out) const { + if (IsSuccess()) { + return Entry_.Out(out); + } else { + return TStatus::Out(out); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TListDirectoryResult::TListDirectoryResult(TStatus&& status, const TSchemeEntry& self, std::vector&& children) + : TDescribePathResult(std::move(status), self) + , Children_(std::move(children)) +{} + +const std::vector& TListDirectoryResult::GetChildren() const { + CheckStatusOk("TListDirectoryResult::GetChildren"); + return Children_; +} + +void TListDirectoryResult::Out(IOutputStream& out) const { + if (IsSuccess()) { + out << "{ children [" << JoinSeq(", ", Children_) << "] }"; + } else { + return TStatus::Out(out); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TSchemeClient::TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{} + +TAsyncStatus TSchemeClient::MakeDirectory(const std::string& path, const TMakeDirectorySettings& settings) { + return Impl_->MakeDirectory(path, settings); +} + +TAsyncStatus TSchemeClient::RemoveDirectory(const std::string &path, const TRemoveDirectorySettings& settings) { + return Impl_->RemoveDirectory(path, settings); +} + +TAsyncDescribePathResult TSchemeClient::DescribePath(const std::string& path, const TDescribePathSettings& settings) { + return Impl_->DescribePath(path, settings); +} + +TAsyncListDirectoryResult TSchemeClient::ListDirectory(const std::string& path, + const TListDirectorySettings& settings) +{ + return Impl_->ListDirectory(path, settings); +} + +TAsyncStatus TSchemeClient::ModifyPermissions(const std::string& path, + const TModifyPermissionsSettings& data) +{ + return Impl_->ModifyPermissions(path, data); +} + +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/scheme/ya.make b/ydb/public/sdk/cpp/src/client/scheme/ya.make new file mode 100644 index 000000000000..46a092e9bb7e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/scheme/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + out.cpp + scheme.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/ss_tasks/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/ss_tasks/CMakeLists.txt new file mode 100644 index 000000000000..61092e60f849 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ss_tasks/CMakeLists.txt @@ -0,0 +1,24 @@ +_ydb_sdk_add_library(client-ss_tasks) + +target_link_libraries(client-ss_tasks PUBLIC + yutil + api-grpc + api-protos + client-ydb_common_client-impl + client-ydb_driver + client-ydb_proto + client-ydb_types-operation +) + +target_sources(client-ss_tasks PRIVATE + task.cpp + out.cpp +) + +generate_enum_serilization(client-ss_tasks + ${YDB_SDK_SOURCE_DIR}/src/client/ss_tasks/task.h + INCLUDE_HEADERS + src/client/ss_tasks/task.h +) + +_ydb_sdk_make_client_component(SSTasks client-ss_tasks) diff --git a/ydb/public/sdk/cpp/src/client/ss_tasks/out.cpp b/ydb/public/sdk/cpp/src/client/ss_tasks/out.cpp new file mode 100644 index 000000000000..204c27c5737a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ss_tasks/out.cpp @@ -0,0 +1 @@ +#include "task.h" diff --git a/ydb/public/sdk/cpp/src/client/ss_tasks/task.cpp b/ydb/public/sdk/cpp/src/client/ss_tasks/task.cpp new file mode 100644 index 000000000000..80b3131b6c42 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ss_tasks/task.cpp @@ -0,0 +1,23 @@ +#include "task.h" +#include +#include +#include +#include +#include + +namespace NYdb::inline V3 { +namespace NSchemeShard { + +/// YT +TBackgroundProcessesResponse::TBackgroundProcessesResponse(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + Metadata_.Id = GetProto().DebugString(); +} + +const TBackgroundProcessesResponse::TMetadata& TBackgroundProcessesResponse::Metadata() const { + return Metadata_; +} + +} // namespace NExport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/ss_tasks/task.h b/ydb/public/sdk/cpp/src/client/ss_tasks/task.h new file mode 100644 index 000000000000..5c0c4db1868f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ss_tasks/task.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NSchemeShard { + +class TBackgroundProcessesResponse: public TOperation { +private: + using TBase = TOperation; +public: + struct TMetadata { + std::string Id; + }; + +public: + using TOperation::TOperation; + TBackgroundProcessesResponse(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +} // namespace NExport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/ss_tasks/ya.make b/ydb/public/sdk/cpp/src/client/ss_tasks/ya.make new file mode 100644 index 000000000000..5f7fc1e9c1df --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ss_tasks/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + task.cpp + out.cpp +) + +GENERATE_ENUM_SERIALIZATION(task.h) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/types/operation +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/table/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/table/CMakeLists.txt new file mode 100644 index 000000000000..b7103470cc4b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/CMakeLists.txt @@ -0,0 +1,37 @@ +add_subdirectory(impl) +add_subdirectory(query_stats) + +_ydb_sdk_add_library(client-ydb_table) + +target_link_libraries(client-ydb_table PUBLIC + yutil + enum_serialization_runtime + api-protos + impl-ydb_internal-make_request + impl-ydb_internal-kqp_session_common + impl-ydb_internal-retry + client-ydb_driver + client-ydb_params + client-ydb_proto + client-ydb_result + client-ydb_scheme + client-ydb_table-impl + client-ydb_table-query_stats + client-ydb_types-operation + client-ydb_value + string_utils-misc +) + +target_sources(client-ydb_table PRIVATE + table.cpp + proto_accessor.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_table + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/table/table_enum.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/table/table_enum.h +) + +_ydb_sdk_make_client_component(Table client-ydb_table) diff --git a/ydb/public/sdk/cpp/src/client/table/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/table/impl/CMakeLists.txt new file mode 100644 index 000000000000..5bbeecf52437 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/CMakeLists.txt @@ -0,0 +1,30 @@ +_ydb_sdk_add_library(client-ydb_table-impl) + +target_link_libraries(client-ydb_table-impl + PUBLIC + yutil + threading-future + api-protos + api-grpc + library-operation_id + client-impl-ydb_endpoints + impl-ydb_internal-session_pool + client-ydb_table-query_stats + PRIVATE + OpenSSL::SSL +) + +target_sources(client-ydb_table-impl PRIVATE + client_session.cpp + data_query.cpp + readers.cpp + request_migrator.cpp + table_client.cpp + transaction.cpp +) + +target_compile_options(client-ydb_table-impl PRIVATE + -Wno-deprecated +) + +_ydb_sdk_install_targets(TARGETS client-ydb_table-impl) diff --git a/ydb/public/sdk/cpp/src/client/table/impl/client_session.cpp b/ydb/public/sdk/cpp/src/client/table/impl/client_session.cpp new file mode 100644 index 000000000000..cbab337644cd --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/client_session.cpp @@ -0,0 +1,79 @@ +#include "client_session.h" +#include "data_query.h" + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +TSession::TImpl::TImpl(const std::string& sessionId, const std::string& endpoint, bool useQueryCache, ui32 queryCacheSize, bool isOwnedBySessionPool) + : TKqpSessionCommon(sessionId, endpoint, isOwnedBySessionPool) + , UseQueryCache_(useQueryCache) + , QueryCache_(queryCacheSize) +{} + +void TSession::TImpl::InvalidateQueryInCache(const std::string& key) { + if (!UseQueryCache_) { + return; + } + + std::lock_guard guard(Lock_); + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + QueryCache_.Erase(it); + } +} + +void TSession::TImpl::InvalidateQueryCache() { + if (!UseQueryCache_) { + return; + } + + std::lock_guard guard(Lock_); + QueryCache_.Clear(); +} + +std::optional TSession::TImpl::GetQueryFromCache(const std::string& query, bool allowMigration) { + if (!UseQueryCache_) { + return {}; + } + + auto key = EncodeQuery(query, allowMigration); + + std::lock_guard guard(Lock_); + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + return *it; + } + + return std::nullopt; +} + +void TSession::TImpl::AddQueryToCache(const TDataQuery& query) { + if (!UseQueryCache_) { + return; + } + + const auto& id = query.Impl_->GetId(); + if (id.empty()) { + return; + } + + auto key = query.Impl_->GetTextHash(); + TDataQueryInfo queryInfo(id, query.Impl_->GetParameterTypes()); + + std::lock_guard guard(Lock_); + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + *it = queryInfo; + } else { + QueryCache_.Insert(key, queryInfo); + } +} + +const TLRUCache& TSession::TImpl::GetQueryCacheUnsafe() const { + return QueryCache_; +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/client_session.h b/ydb/public/sdk/cpp/src/client/table/impl/client_session.h new file mode 100644 index 000000000000..bcad0497a0df --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/client_session.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include + +#include + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +//////////////////////////////////////////////////////////////////////////////// + +using TSessionInspectorFn = std::function; + +class TSession::TImpl : public TKqpSessionCommon { + friend class TTableClient; + friend class TSession; + +#ifdef YDB_IMPL_TABLE_CLIENT_SESSION_UT +public: +#endif + TImpl(const std::string& sessionId, const std::string& endpoint, bool useQueryCache, ui32 queryCacheSize, bool isOwnedBySessionPool); +public: + struct TDataQueryInfo { + std::string QueryId; + ::google::protobuf::Map ParameterTypes; + + TDataQueryInfo() {} + + TDataQueryInfo(const std::string& queryId, + const ::google::protobuf::Map& parameterTypes) + : QueryId(queryId) + , ParameterTypes(parameterTypes) {} + }; +public: + ~TImpl() = default; + + void InvalidateQueryInCache(const std::string& key); + void InvalidateQueryCache(); + std::optional GetQueryFromCache(const std::string& query, bool allowMigration); + void AddQueryToCache(const TDataQuery& query); + + const TLRUCache& GetQueryCacheUnsafe() const; + + static TSessionInspectorFn GetSessionInspector( + NThreading::TPromise& promise, + std::shared_ptr client, + const TCreateSessionSettings& settings, + ui32 counter, bool needUpdateActiveSessionCounter); + +private: + bool UseQueryCache_; + TLRUCache QueryCache_; +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/data_query.cpp b/ydb/public/sdk/cpp/src/client/table/impl/data_query.cpp new file mode 100644 index 000000000000..2afe7bd63696 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/data_query.cpp @@ -0,0 +1,61 @@ +#include "data_query.h" + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +std::string GetQueryHash(const std::string& text) { + SHA256_CTX sha; + SHA256_Init(&sha); + SHA256_Update(&sha, text.data(), text.size()); + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, &sha); + return std::string(reinterpret_cast(hash), sizeof(hash)); +} + +std::string EncodeQuery(const std::string& text, bool reversible) { + if (reversible) { + //TODO: may be compress and decompress this query + return text; + } else { + return GetQueryHash(text); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TDataQuery::TImpl::TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration) + : Session_(session) + , Id_(id) + , TextHash_(EncodeQuery(text, allowMigration)) + , Text_(keepText ? text : std::optional()) +{} + +TDataQuery::TImpl::TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration, + const ::google::protobuf::Map& types) + : Session_(session) + , Id_(id) + , ParameterTypes_(types) + , TextHash_(EncodeQuery(text, allowMigration)) + , Text_(keepText ? text : std::optional()) +{} + +const std::string& TDataQuery::TImpl::GetId() const { + return Id_; +} + +const ::google::protobuf::Map& TDataQuery::TImpl::GetParameterTypes() const { + return ParameterTypes_; +} + +const std::string& TDataQuery::TImpl::GetTextHash() const { + return TextHash_; +} + +const std::optional& TDataQuery::TImpl::GetText() const { + return Text_; +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/data_query.h b/ydb/public/sdk/cpp/src/client/table/impl/data_query.h new file mode 100644 index 000000000000..a22b0e8eb84e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/data_query.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +std::string EncodeQuery(const std::string& text, bool reversible); + +//////////////////////////////////////////////////////////////////////////////// + +class TDataQuery::TImpl { + friend class TDataQuery; + +public: + TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration); + + TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration, + const ::google::protobuf::Map& types); + + const std::string& GetId() const; + const ::google::protobuf::Map& GetParameterTypes() const; + const std::string& GetTextHash() const; + const std::optional& GetText() const; + +private: + NTable::TSession Session_; + std::string Id_; + ::google::protobuf::Map ParameterTypes_; + std::string TextHash_; + std::optional Text_; +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/readers.cpp b/ydb/public/sdk/cpp/src/client/table/impl/readers.cpp new file mode 100644 index 000000000000..acb5f6dd58b1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/readers.cpp @@ -0,0 +1,106 @@ +#include "readers.h" + +#include + + +namespace NYdb::inline V3 { +namespace NTable { + +using namespace NThreading; + + +TTablePartIterator::TReaderImpl::TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint) + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) +{} + +TTablePartIterator::TReaderImpl::~TReaderImpl() { + StreamProcessor_->Cancel(); +} + +bool TTablePartIterator::TReaderImpl::IsFinished() { + return Finished_; +} + +TAsyncSimpleStreamPart TTablePartIterator::TReaderImpl::ReadNext(std::shared_ptr self) { + auto promise = NThreading::NewPromise>(); + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + std::optional snapshot; + if (self->Response_.has_snapshot()) { + snapshot.emplace( + self->Response_.snapshot().plan_step(), + self->Response_.snapshot().tx_id()); + } + if (!grpcStatus.Ok()) { + self->Finished_ = true; + promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), + TStatus(TPlainStatus(grpcStatus, self->Endpoint_)), + snapshot}); + } else { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast(self->Response_.status()); + promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), + TStatus(clientStatus, std::move(issues)), + snapshot}); + } + }; + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); +} + + + +TScanQueryPartIterator::TReaderImpl::TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint) + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) +{} + +TScanQueryPartIterator::TReaderImpl::~TReaderImpl() { + StreamProcessor_->Cancel(); +} + +bool TScanQueryPartIterator::TReaderImpl::IsFinished() const { + return Finished_; +} + +TAsyncScanQueryPart TScanQueryPartIterator::TReaderImpl::ReadNext(std::shared_ptr self) { + auto promise = NThreading::NewPromise(); + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + if (!grpcStatus.Ok()) { + self->Finished_ = true; + promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); + } else { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast(self->Response_.status()); + // TODO: Add headers for streaming calls. + TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; + TStatus status{std::move(plainStatus)}; + std::optional queryStats; + std::optional diagnostics; + + if (self->Response_.result().has_query_stats()) { + queryStats = TQueryStats(self->Response_.result().query_stats()); + } + + diagnostics = self->Response_.result().query_full_diagnostics(); + + if (self->Response_.result().has_result_set()) { + promise.SetValue({std::move(status), + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats, diagnostics}); + } else { + promise.SetValue({std::move(status), queryStats, diagnostics}); + } + } + }; + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); +} + +} +} diff --git a/ydb/public/sdk/cpp/src/client/table/impl/readers.h b/ydb/public/sdk/cpp/src/client/table/impl/readers.h new file mode 100644 index 000000000000..fd7738a6094a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/readers.h @@ -0,0 +1,66 @@ +#pragma once + +#include + +#include +#include + +#include + +#include "client_session.h" +#include "data_query.h" +#include "request_migrator.h" + + +namespace NYdb::inline V3 { +namespace NTable { + +using namespace NThreading; + + +class TTablePartIterator::TReaderImpl { +public: + using TSelf = TTablePartIterator::TReaderImpl; + using TResponse = Ydb::Table::ReadTableResponse; + using TStreamProcessorPtr = NYdbGrpc::IStreamRequestReadProcessor::TPtr; + using TReadCallback = NYdbGrpc::IStreamRequestReadProcessor::TReadCallback; + using TGRpcStatus = NYdbGrpc::TGrpcStatus; + using TBatchReadResult = std::pair; + + TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint); + ~TReaderImpl(); + bool IsFinished(); + TAsyncSimpleStreamPart ReadNext(std::shared_ptr self); + +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + std::string Endpoint_; +}; + + +class TScanQueryPartIterator::TReaderImpl { +public: + using TSelf = TScanQueryPartIterator::TReaderImpl; + using TResponse = Ydb::Table::ExecuteScanQueryPartialResponse; + using TStreamProcessorPtr = NYdbGrpc::IStreamRequestReadProcessor::TPtr; + using TReadCallback = NYdbGrpc::IStreamRequestReadProcessor::TReadCallback; + using TGRpcStatus = NYdbGrpc::TGrpcStatus; + using TBatchReadResult = std::pair; + + TReaderImpl(TStreamProcessorPtr streamProcessor, const std::string& endpoint); + ~TReaderImpl(); + bool IsFinished() const; + TAsyncScanQueryPart ReadNext(std::shared_ptr self); + +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + std::string Endpoint_; +}; + + +} +} diff --git a/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.cpp b/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.cpp new file mode 100644 index 000000000000..94d497358172 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.cpp @@ -0,0 +1,87 @@ +#include "request_migrator.h" + +#include +#include +#include + +namespace NYdb::inline V3 { + +namespace NMath { + +TStats CalcCV(const std::vector& in) { + if (in.empty()) + return {0, 0.0}; + + if (in.size() == 1) + return {0, static_cast(in[0])}; + + const size_t sum = std::accumulate(in.begin(), in.end(), 0); + if (!sum) + return {0, 0.0}; + + const float mean = sum / static_cast(in.size()); + + double tmp = 0; + for (float t : in) { + t -= mean; + tmp += t * t; + } + + auto cv = static_cast(llround(100.0f * sqrt(tmp / static_cast(in.size() - 1)) / mean)); + return {cv, mean}; +} + +} // namespace NMath + +namespace NTable { + +void TRequestMigrator::SetHost(ui64 nodeId) { + std::lock_guard lock(Lock_); + CurHost_ = nodeId; +} + +bool TRequestMigrator::IsOurSession(const TKqpSessionCommon* session) const { + if (!CurHost_) + return false; + + if (session->GetEndpointKey().GetNodeId() != CurHost_) + return false; + + return true; +} + +bool TRequestMigrator::Reset() { + if (Lock_.try_lock()) { + if (CurHost_) { + CurHost_ = 0; + Lock_.unlock(); + return true; + } else { + Lock_.unlock(); + return false; + } + } else { + return false; + } +} + +bool TRequestMigrator::DoCheckAndMigrate(const TKqpSessionCommon* session) { + if (session->GetEndpoint().empty()) + return false; + + if (Lock_.try_lock()) { + if (IsOurSession(session)) { + CurHost_ = 0; + Lock_.unlock(); + return true; + } else { + Lock_.unlock(); + return false; + } + } else { + return false; + } +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.h b/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.h new file mode 100644 index 000000000000..e33967d42b6c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/request_migrator.h @@ -0,0 +1,68 @@ +#pragma once + +#include "client_session.h" + +#include + +#include + +#include + +#include +#include + +namespace NYdb::inline V3 { + +namespace NMath { + +struct TStats { + const ui64 Cv; + const float Mean; +}; + +TStats CalcCV(const std::vector&); + +} // namespace NMath + +namespace NTable { + +// TableClientImpl interface for migrator +// Migrator should be able +// - PrepareDataQuery +// - Schedule some cb to be executed +class IMigratorClient { +public: + virtual ~IMigratorClient() = default; + virtual void ScheduleTaskUnsafe(std::function&& fn, TDuration timeout) = 0; +}; + +class TRequestMigrator { +public: + // Set host to migrate session from. + // If the target session set requests will be reprepared on the new session in background. + // Real migration will be performed after DoCheckAndMigrate call + // IMPORTANT: SetHost methods are not reentrant. Moreover new call of this methods allowed only if + // future returned from previous call was set. + // The value of future means number of requests migrated + void SetHost(ui64 nodeId); + + // Checks and perform migration if the session suitable to be removed from host. + // Prepared requests will be migrated if target session was set along with host. + // Returns false if session is not suitable or unable to get lock to start migration + // Returns true if session is suitable in this case Unlink methos on the session is called + // This methos is thread safe. + bool DoCheckAndMigrate(const TKqpSessionCommon* session); + + // Reset migrator to initiall state if migration was not started and returns true + // Returns false if migration was started + bool Reset(); +private: + bool IsOurSession(const TKqpSessionCommon* session) const; + + ui64 CurHost_ = 0; + + mutable std::mutex Lock_; +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/table_client.cpp b/ydb/public/sdk/cpp/src/client/table/impl/table_client.cpp new file mode 100644 index 000000000000..25fdbcf54e9b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/table_client.cpp @@ -0,0 +1,1181 @@ +#include "table_client.h" + +namespace NYdb::inline V3 { + +namespace NTable { + +using namespace NThreading; + +const TKeepAliveSettings TTableClient::TImpl::KeepAliveSettings = TKeepAliveSettings().ClientTimeout(KEEP_ALIVE_CLIENT_TIMEOUT); + + +TDuration GetMinTimeToTouch(const TSessionPoolSettings& settings) { + return Min(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); +} + +TDuration GetMaxTimeToTouch(const TSessionPoolSettings& settings) { + return Max(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); +} + +TTableClient::TImpl::TImpl(std::shared_ptr&& connections, const TClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + , Settings_(settings) + , SessionPool_(Settings_.SessionPoolSettings_.MaxActiveSessions_) +{ + if (!DbDriverState_->StatCollector.IsCollecting()) { + return; + } + + SetStatCollector(DbDriverState_->StatCollector.GetClientStatCollector()); + SessionPool_.SetStatCollector(DbDriverState_->StatCollector.GetSessionPoolStatCollector()); +} + +TTableClient::TImpl::~TImpl() { + if (Connections_->GetDrainOnDtors()) { + Drain().Wait(); + } +} + +bool TTableClient::TImpl::LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag) { + return DbDriverState_->EndpointPool.LinkObjToEndpoint(endpoint, obj, tag); +} + +void TTableClient::TImpl::InitStopper() { + std::weak_ptr weak = shared_from_this(); + auto cb = [weak]() mutable { + auto strong = weak.lock(); + if (!strong) { + auto promise = NThreading::NewPromise(); + promise.SetException("no more client"); + return promise.GetFuture(); + } + return strong->Drain(); + }; + + DbDriverState_->AddCb(std::move(cb), TDbDriverState::ENotifyType::STOP); +} + +NThreading::TFuture TTableClient::TImpl::Drain() { + std::vector> sessions; + // No realocations under lock + sessions.reserve(Settings_.SessionPoolSettings_.MaxActiveSessions_); + auto drainer = [&sessions](std::unique_ptr&& impl) mutable { + sessions.push_back(std::move(impl)); + return true; + }; + SessionPool_.Drain(drainer, true); + std::vector closeResults; + for (auto& s : sessions) { + if (!s->GetId().empty()) { + closeResults.push_back(CloseInternal(s.get())); + } + } + sessions.clear(); + return NThreading::WaitExceptionOrAll(closeResults); +} + +NThreading::TFuture TTableClient::TImpl::Stop() { + return Drain(); +} + +void TTableClient::TImpl::ScheduleTaskUnsafe(std::function&& fn, TDuration timeout) { + Connections_->ScheduleOneTimeTask(std::move(fn), timeout); +} + +void TTableClient::TImpl::StartPeriodicSessionPoolTask() { + // Session pool guarantees than client is alive during call callbacks + auto deletePredicate = [this](TKqpSessionCommon* s, size_t sessionsCount) { + + const auto& sessionPoolSettings = Settings_.SessionPoolSettings_; + const auto spentTime = s->GetTimeToTouchFast() - s->GetTimeInPastFast(); + + if (spentTime >= sessionPoolSettings.CloseIdleThreshold_) { + if (sessionsCount > sessionPoolSettings.MinPoolSize_) { + return true; + } + } + + return false; + }; + + auto keepAliveCmd = [this](TKqpSessionCommon* s) { + auto strongClient = shared_from_this(); + TSession session( + strongClient, + std::shared_ptr( + static_cast(s), + TSession::TImpl::GetSmartDeleter(strongClient) + )); + + Y_ABORT_UNLESS(!session.GetId().empty()); + + const auto sessionPoolSettings = session.Client_->Settings_.SessionPoolSettings_; + const auto spentTime = session.SessionImpl_->GetTimeToTouchFast() - session.SessionImpl_->GetTimeInPastFast(); + + const auto maxTimeToTouch = GetMaxTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); + const auto minTimeToTouch = GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); + + auto calcTimeToNextTouch = [maxTimeToTouch, minTimeToTouch] (const TDuration spent) { + auto timeToNextTouch = minTimeToTouch; + if (maxTimeToTouch > spent) { + auto t = maxTimeToTouch - spent; + timeToNextTouch = Min(t, minTimeToTouch); + } + return timeToNextTouch; + }; + + if (spentTime >= sessionPoolSettings.KeepAliveIdleThreshold_) { + + // Handle of session status will be done inside InjectSessionStatusInterception routine. + // We just need to reschedule time to next call because InjectSessionStatusInterception doesn't + // update timeInPast for calls from internal keep alive routine + session.KeepAlive(KeepAliveSettings) + .Subscribe([spentTime, session, maxTimeToTouch, calcTimeToNextTouch](TAsyncKeepAliveResult asyncResult) { + if (!asyncResult.GetValue().IsSuccess()) + return; + + if (spentTime >= maxTimeToTouch) { + auto timeToNextTouch = calcTimeToNextTouch(spentTime); + session.SessionImpl_->ScheduleTimeToTouchFast(timeToNextTouch, true); + } + }); + return; + } + + auto timeToNextTouch = calcTimeToNextTouch(spentTime); + session.SessionImpl_->ScheduleTimeToTouchFast( + NSessionPool::RandomizeThreshold(timeToNextTouch), + spentTime >= maxTimeToTouch + ); + }; + + std::weak_ptr weak = weak_from_this(); + Connections_->AddPeriodicTask( + SessionPool_.CreatePeriodicTask( + weak, + std::move(keepAliveCmd), + std::move(deletePredicate) + ), NSessionPool::PERIODIC_ACTION_INTERVAL); +} + +ui64 TTableClient::TImpl::ScanForeignLocations(std::shared_ptr client) { + size_t max = 0; + ui64 result = 0; + + auto cb = [&result, &max](ui64 nodeId, const IObjRegistryHandle& handle) { + const auto sz = handle.Size(); + if (sz > max) { + result = nodeId; + max = sz; + } + }; + + client->DbDriverState_->ForEachForeignEndpoint(cb, client.get()); + + return result; +} + +std::pair TTableClient::TImpl::ScanLocation(std::shared_ptr client, + std::unordered_map& sessions, bool allNodes) +{ + std::pair result = {0, 0}; + + auto cb = [&result, &sessions](ui64 nodeId, const IObjRegistryHandle& handle) { + const auto sz = handle.Size(); + sessions.insert({nodeId, sz}); + if (sz > result.second) { + result.first = nodeId; + result.second = sz; + } + }; + + if (allNodes) { + client->DbDriverState_->ForEachEndpoint(cb, client.get()); + } else { + client->DbDriverState_->ForEachLocalEndpoint(cb, client.get()); + } + + return result; +} + +NMath::TStats TTableClient::TImpl::CalcCV(const std::unordered_map& in) { + std::vector t; + t.reserve(in.size()); + std::transform(in.begin(), in.end(), std::back_inserter(t), [](const std::pair& pair) { + return pair.second; + }); + return NMath::CalcCV(t); +} + +void TTableClient::TImpl::StartPeriodicHostScanTask() { + std::weak_ptr weak = shared_from_this(); + + // The future in completed when we have finished current migrate task + // and ready to accept new one + std::pair winner = {0, 0}; + + auto periodicCb = [weak, winner](NYdb::NIssue::TIssues&&, EStatus status) mutable -> bool { + + if (status != EStatus::SUCCESS) { + return false; + } + + auto strongClient = weak.lock(); + if (!strongClient) { + return false; + } else { + TRequestMigrator& migrator = strongClient->RequestMigrator_; + + const auto balancingPolicy = strongClient->DbDriverState_->GetBalancingPolicy(); + + // Try to find any host at foreign locations if prefer local dc + const ui64 foreignHost = (balancingPolicy == EBalancingPolicy::UsePreferableLocation) ? + ScanForeignLocations(strongClient) : 0; + + std::unordered_map hostMap; + + winner = ScanLocation(strongClient, hostMap, + balancingPolicy == EBalancingPolicy::UseAllNodes); + + bool forceMigrate = false; + + // There is host in foreign locations + if (foreignHost) { + // But no hosts at local + if (hostMap.empty()) { + Y_ABORT_UNLESS(!winner.second); + // Scan whole cluster - we have no local dc + winner = ScanLocation(strongClient, hostMap, true); + } else { + // We have local and foreign hosts, so force migration to local one + forceMigrate = true; + // Just replace source + winner.first = foreignHost; + winner.second++; + } + } + + const auto minCv = strongClient->Settings_.MinSessionCV_; + + const auto stats = CalcCV(hostMap); + + strongClient->DbDriverState_->StatCollector.SetSessionCV(stats.Cv); + + // Just scan to update monitoring counter ^^ + // Balancing feature is disabled. + if (!minCv) + return true; + + if (hostMap.size() < 2) + return true; + + // Migrate last session only if move from foreign to local + if (!forceMigrate && winner.second < 2) + return true; + + if (stats.Cv > minCv || forceMigrate) { + migrator.SetHost(winner.first); + } else { + migrator.SetHost(0); + } + return true; + } + }; + + Connections_->AddPeriodicTask(std::move(periodicCb), HOSTSCAN_PERIODIC_ACTION_INTERVAL); +} + +TAsyncCreateSessionResult TTableClient::TImpl::GetSession(const TCreateSessionSettings& settings) { + using namespace NSessionPool; + + class TTableClientGetSessionCtx : public IGetSessionCtx { + public: + TTableClientGetSessionCtx(std::shared_ptr client, TDuration clientTimeout) + : Promise(NewPromise()) + , Client(client) + , ClientTimeout(clientTimeout) + {} + + TAsyncCreateSessionResult GetFuture() { + return Promise.GetFuture(); + } + + void ReplyError(TStatus status) override { + TSession session(Client, "", "", false); + ScheduleReply(TCreateSessionResult(std::move(status), std::move(session))); + } + + void ReplySessionToUser(TKqpSessionCommon* session) override { + TCreateSessionResult val( + TStatus(TPlainStatus()), + TSession( + Client, + std::shared_ptr( + static_cast(session), + TSession::TImpl::GetSmartDeleter(Client) + ) + ) + ); + + ScheduleReply(std::move(val)); + } + + void ReplyNewSession() override { + TCreateSessionSettings settings; + settings.ClientTimeout(ClientTimeout); + const auto& sessionResult = Client->CreateSession(settings, false); + sessionResult.Subscribe(TSession::TImpl::GetSessionInspector(Promise, Client, settings, 0, true)); + } + + private: + void ScheduleReply(TCreateSessionResult val) { + //TODO: Do we realy need it? + Client->ScheduleTaskUnsafe([promise{std::move(Promise)}, val{std::move(val)}]() mutable { + promise.SetValue(std::move(val)); + }, TDuration()); + } + NThreading::TPromise Promise; + std::shared_ptr Client; + const TDuration ClientTimeout; + }; + + auto ctx = std::make_unique(shared_from_this(), settings.ClientTimeout_); + auto future = ctx->GetFuture(); + SessionPool_.GetSession(std::move(ctx)); + return future; +} + +i64 TTableClient::TImpl::GetActiveSessionCount() const { + return SessionPool_.GetActiveSessions(); +} + +i64 TTableClient::TImpl::GetActiveSessionsLimit() const { + return SessionPool_.GetActiveSessionsLimit(); +} + +i64 TTableClient::TImpl::GetCurrentPoolSize() const { + return SessionPool_.GetCurrentPoolSize(); +} + +TAsyncCreateSessionResult TTableClient::TImpl::CreateSession(const TCreateSessionSettings& settings, bool standalone, + std::string preferredLocation) +{ + auto request = MakeOperationRequest(settings); + + auto createSessionPromise = NewPromise(); + auto self = shared_from_this(); + auto rpcSettings = TRpcRequestSettings::Make(settings, TEndpointKey(preferredLocation, 0)); + rpcSettings.Header.push_back({NYdb::YDB_CLIENT_CAPABILITIES, NYdb::YDB_CLIENT_CAPABILITY_SESSION_BALANCER}); + + auto createSessionExtractor = [createSessionPromise, self, standalone] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::CreateSessionResult result; + if (any) { + any->UnpackTo(&result); + } + auto session = TSession(self, result.session_id(), status.Endpoint, !standalone); + if (status.Ok()) { + if (!standalone) { + session.SessionImpl_->MarkActive(); + } + self->DbDriverState_->StatCollector.IncSessionsOnHost(status.Endpoint); + } else { + // We do not use SessionStatusInterception for CreateSession request + session.SessionImpl_->MarkBroken(); + } + TCreateSessionResult val(TStatus(std::move(status)), std::move(session)); + createSessionPromise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + createSessionExtractor, + &Ydb::Table::V1::TableService::Stub::AsyncCreateSession, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + rpcSettings); + + std::weak_ptr state = DbDriverState_; + + return createSessionPromise.GetFuture(); +} + +TAsyncKeepAliveResult TTableClient::TImpl::KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session->GetId()}); + + auto keepAliveResultPromise = NewPromise(); + auto self = shared_from_this(); + + auto keepAliveExtractor = [keepAliveResultPromise, self] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::KeepAliveResult result; + ESessionStatus sessionStatus = ESessionStatus::Unspecified; + if (any) { + any->UnpackTo(&result); + + switch (result.session_status()) { + case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_READY: + sessionStatus = ESessionStatus::Ready; + break; + case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_BUSY: + sessionStatus = ESessionStatus::Busy; + break; + default: + sessionStatus = ESessionStatus::Unspecified; + } + } + TKeepAliveResult val(TStatus(std::move(status)), sessionStatus); + keepAliveResultPromise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + keepAliveExtractor, + &Ydb::Table::V1::TableService::Stub::AsyncKeepAlive, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session->GetEndpointKey()) + ); + + return keepAliveResultPromise.GetFuture(); +} + +TFuture TTableClient::TImpl::CreateTable(Ydb::Table::CreateTableRequest&& request, const TCreateTableSettings& settings) +{ + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncCreateTable, + TRpcRequestSettings::Make(settings)); +} + +TFuture TTableClient::TImpl::AlterTable(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings) +{ + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncAlterTable, + TRpcRequestSettings::Make(settings)); +} + +TAsyncOperation TTableClient::TImpl::AlterTableLong(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings) +{ + using Ydb::Table::V1::TableService; + using Ydb::Table::AlterTableRequest; + using Ydb::Table::AlterTableResponse; + return RunOperation( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncAlterTable, + TRpcRequestSettings::Make(settings)); +} + +TFuture TTableClient::TImpl::CopyTable(const std::string& sessionId, const std::string& src, const std::string& dst, + const TCopyTableSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionId}); + request.set_source_path(TStringType{src}); + request.set_destination_path(TStringType{dst}); + + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncCopyTable, + TRpcRequestSettings::Make(settings)); +} + +TFuture TTableClient::TImpl::CopyTables(Ydb::Table::CopyTablesRequest&& request, const TCopyTablesSettings& settings) +{ + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncCopyTables, + TRpcRequestSettings::Make(settings)); +} + +TFuture TTableClient::TImpl::RenameTables(Ydb::Table::RenameTablesRequest&& request, const TRenameTablesSettings& settings) +{ + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncRenameTables, + TRpcRequestSettings::Make(settings)); +} + +TFuture TTableClient::TImpl::DropTable(const std::string& sessionId, const std::string& path, const TDropTableSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); + + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncDropTable, + TRpcRequestSettings::Make(settings)); +} + +TAsyncDescribeTableResult TTableClient::TImpl::DescribeTable(const std::string& sessionId, const std::string& path, const TDescribeTableSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); + if (settings.WithKeyShardBoundary_) { + request.set_include_shard_key_bounds(true); + } + + if (settings.WithTableStatistics_) { + request.set_include_table_stats(true); + } + + if (settings.WithPartitionStatistics_) { + request.set_include_partition_stats(true); + } + + if (settings.WithSetVal_) { + request.set_include_set_val(true); + } + + if (settings.WithShardNodesInfo_) { + request.set_include_shard_nodes_info(true); + } + + auto promise = NewPromise(); + + auto extractor = [promise, settings] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::DescribeTableResult result; + if (any) { + any->UnpackTo(&result); + } + TDescribeTableResult describeTableResult(TStatus(std::move(status)), + std::move(result), settings); + promise.SetValue(std::move(describeTableResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncDescribeTable, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); +} + +TAsyncPrepareQueryResult TTableClient::TImpl::PrepareDataQuery(const TSession& session, const std::string& query, + const TPrepareDataQuerySettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + request.set_yql_text(TStringType{query}); + + auto promise = NewPromise(); + + //See ExecuteDataQueryInternal for explanation + auto sessionPtr = new TSession(session); + auto extractor = [promise, sessionPtr, query] + (google::protobuf::Any* any, TPlainStatus status) mutable { + TDataQuery dataQuery(*sessionPtr, query, ""); + + if (any) { + Ydb::Table::PrepareQueryResult result; + any->UnpackTo(&result); + + if (status.Ok()) { + dataQuery = TDataQuery(*sessionPtr, query, std::string{result.query_id()}, result.parameters_types()); + sessionPtr->SessionImpl_->AddQueryToCache(dataQuery); + } + } + + TPrepareQueryResult prepareQueryResult(TStatus(std::move(status)), + dataQuery, false); + delete sessionPtr; + promise.SetValue(std::move(prepareQueryResult)); + }; + + CollectQuerySize(query, QuerySizeHistogram); + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncPrepareDataQuery, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); + + return promise.GetFuture(); +} + +TAsyncStatus TTableClient::TImpl::ExecuteSchemeQuery(const std::string& sessionId, const std::string& query, + const TExecSchemeQuerySettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionId}); + request.set_yql_text(TStringType{query}); + + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncExecuteSchemeQuery, + TRpcRequestSettings::Make(settings)); +} + +TAsyncBeginTransactionResult TTableClient::TImpl::BeginTransaction(const TSession& session, const TTxSettings& txSettings, + const TBeginTxSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + SetTxSettings(txSettings, request.mutable_tx_settings()); + + auto promise = NewPromise(); + + auto extractor = [promise, session] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string txId; + if (any) { + Ydb::Table::BeginTransactionResult result; + any->UnpackTo(&result); + txId = result.tx_meta().id(); + } + + TBeginTransactionResult beginTxResult(TStatus(std::move(status)), + TTransaction(session, txId)); + promise.SetValue(std::move(beginTxResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncBeginTransaction, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); + + return promise.GetFuture(); +} + +TAsyncCommitTransactionResult TTableClient::TImpl::CommitTransaction(const TSession& session, const std::string& txId, + const TCommitTxSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::optional queryStats; + if (any) { + Ydb::Table::CommitTransactionResult result; + any->UnpackTo(&result); + + if (result.has_query_stats()) { + queryStats = TQueryStats(result.query_stats()); + } + } + + TCommitTransactionResult commitTxResult(TStatus(std::move(status)), queryStats); + promise.SetValue(std::move(commitTxResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncCommitTransaction, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); + + return promise.GetFuture(); +} + +TAsyncStatus TTableClient::TImpl::RollbackTransaction(const TSession& session, const std::string& txId, + const TRollbackTxSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); + + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncRollbackTransaction, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); +} + +TAsyncExplainDataQueryResult TTableClient::TImpl::ExplainDataQuery(const TSession& session, const std::string& query, + const TExplainDataQuerySettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + request.set_yql_text(TStringType{query}); + request.set_collect_full_diagnostics(settings.WithCollectFullDiagnostics_); + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::string ast; + std::string plan; + std::string diagnostics; + if (any) { + Ydb::Table::ExplainQueryResult result; + any->UnpackTo(&result); + ast = result.query_ast(); + plan = result.query_plan(); + diagnostics = result.query_full_diagnostics(); + } + TExplainQueryResult val(TStatus(std::move(status)), + std::move(plan), std::move(ast), std::move(diagnostics)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncExplainDataQuery, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); + + return promise.GetFuture(); +} + +void TTableClient::TImpl::SetTypedValue(Ydb::TypedValue* protoValue, const TValue& value) { + protoValue->mutable_type()->CopyFrom(TProtoAccessor::GetProto(value.GetType())); + protoValue->mutable_value()->CopyFrom(TProtoAccessor::GetProto(value)); +} + +NThreading::TFuture> TTableClient::TImpl::ReadTable( + const std::string& sessionId, + const std::string& path, + const TReadTableSettings& settings) +{ + auto request = MakeRequest(); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); + request.set_ordered(settings.Ordered_); + if (settings.RowLimit_) { + request.set_row_limit(settings.RowLimit_.value()); + } + for (const auto& col : settings.Columns_) { + request.add_columns(TStringType{col}); + } + if (settings.UseSnapshot_) { + request.set_use_snapshot( + settings.UseSnapshot_.value() + ? Ydb::FeatureFlag::ENABLED + : Ydb::FeatureFlag::DISABLED); + } + + if (settings.From_) { + const auto& from = settings.From_.value(); + if (from.IsInclusive()) { + SetTypedValue(request.mutable_key_range()->mutable_greater_or_equal(), from.GetValue()); + } else { + SetTypedValue(request.mutable_key_range()->mutable_greater(), from.GetValue()); + } + } + + if (settings.To_) { + const auto& to = settings.To_.value(); + if (to.IsInclusive()) { + SetTypedValue(request.mutable_key_range()->mutable_less_or_equal(), to.GetValue()); + } else { + SetTypedValue(request.mutable_key_range()->mutable_less(), to.GetValue()); + } + } + + if (settings.BatchLimitBytes_) { + request.set_batch_limit_bytes(*settings.BatchLimitBytes_); + } + + if (settings.BatchLimitRows_) { + request.set_batch_limit_rows(*settings.BatchLimitRows_); + } + + if (settings.ReturnNotNullAsOptional_) { + request.set_return_not_null_data_as_optional( + settings.ReturnNotNullAsOptional_.value() + ? Ydb::FeatureFlag::ENABLED + : Ydb::FeatureFlag::DISABLED); + } + + auto promise = NewPromise>(); + + Connections_->StartReadStream( + std::move(request), + [promise] (TPlainStatus status, TReadTableStreamProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Table::V1::TableService::Stub::AsyncStreamReadTable, + DbDriverState_, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + +} + +TAsyncReadRowsResult TTableClient::TImpl::ReadRows(const std::string& path, TValue&& keys, const std::vector& columns, const TReadRowsSettings& settings) { + auto request = MakeRequest(); + request.set_path(TStringType{path}); + auto* protoKeys = request.mutable_keys(); + *protoKeys->mutable_type() = TProtoAccessor::GetProto(keys.GetType()); + *protoKeys->mutable_value() = TProtoAccessor::GetProto(keys); + auto* protoColumns = request.mutable_columns(); + for (const auto& column: columns) + { + *protoColumns->Add() = column; + } + + auto promise = NewPromise(); + + auto responseCb = [promise] (Ydb::Table::ReadRowsResponse* response, TPlainStatus status) mutable { + Ydb::ResultSet resultSet; + // if there is no response status contains transport errors + if (response) { + Ydb::StatusIds::StatusCode msgStatus = response->status(); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(response->issues(), issues); + status = TPlainStatus(static_cast(msgStatus), std::move(issues)); + resultSet = std::move(*response->mutable_result_set()); + } + TReadRowsResult val(TStatus(std::move(status)), std::move(resultSet)); + promise.SetValue(std::move(val)); + }; + + Connections_->Run( + std::move(request), + responseCb, + &Ydb::Table::V1::TableService::Stub::AsyncReadRows, + DbDriverState_, + TRpcRequestSettings::Make(settings) + ); + + return promise.GetFuture(); +} + +TAsyncStatus TTableClient::TImpl::Close(const TKqpSessionCommon* sessionImpl, const TCloseSessionSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionImpl->GetId()}); + return RunSimple( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncDeleteSession, + TRpcRequestSettings::Make(settings, sessionImpl->GetEndpointKey()) + ); +} + +TAsyncStatus TTableClient::TImpl::CloseInternal(const TKqpSessionCommon* sessionImpl) { + static const auto internalCloseSessionSettings = TCloseSessionSettings() + .ClientTimeout(TDuration::Seconds(2)); + + auto driver = Connections_; + return Close(sessionImpl, internalCloseSessionSettings) + .Apply([driver{std::move(driver)}](TAsyncStatus status) mutable + { + driver.reset(); + return status; + }); +} + +bool TTableClient::TImpl::ReturnSession(TKqpSessionCommon* sessionImpl) { + Y_ABORT_UNLESS(sessionImpl->GetState() == TSession::TImpl::S_ACTIVE || + sessionImpl->GetState() == TSession::TImpl::S_IDLE); + + if (RequestMigrator_.DoCheckAndMigrate(sessionImpl)) { + SessionRemovedDueBalancing.Inc(); + return false; + } + + bool needUpdateCounter = sessionImpl->NeedUpdateActiveCounter(); + // Also removes NeedUpdateActiveCounter flag + sessionImpl->MarkIdle(); + sessionImpl->SetTimeInterval(TDuration::Zero()); + if (!SessionPool_.ReturnSession(sessionImpl, needUpdateCounter)) { + sessionImpl->SetNeedUpdateActiveCounter(needUpdateCounter); + return false; + } + return true; +} + +void TTableClient::TImpl::DeleteSession(TKqpSessionCommon* sessionImpl) { + // Closing not owned by session pool session should not fire getting new session + if (sessionImpl->IsOwnedBySessionPool()) { + if (SessionPool_.CheckAndFeedWaiterNewSession(sessionImpl->NeedUpdateActiveCounter())) { + // We requested new session for waiter which already incremented + // active session counter and old session will be deleted + // - skip update active counter in this case + sessionImpl->SetNeedUpdateActiveCounter(false); + } + } + + if (sessionImpl->NeedUpdateActiveCounter()) { + SessionPool_.DecrementActiveCounter(); + } + + if (!sessionImpl->GetId().empty()) { + CloseInternal(sessionImpl); + DbDriverState_->StatCollector.DecSessionsOnHost(sessionImpl->GetEndpoint()); + } + + delete sessionImpl; +} + +ui32 TTableClient::TImpl::GetSessionRetryLimit() const { + return Settings_.SessionPoolSettings_.RetryLimit_; +} + +void TTableClient::TImpl::SetStatCollector(const NSdkStats::TStatCollector::TClientStatCollector& collector) { + CacheMissCounter.Set(collector.CacheMiss); + QuerySizeHistogram.Set(collector.QuerySize); + ParamsSizeHistogram.Set(collector.ParamsSize); + RetryOperationStatCollector = collector.RetryOperationStatCollector; + SessionRemovedDueBalancing.Set(collector.SessionRemovedDueBalancing); + RequestMigrated.Set(collector.RequestMigrated); +} + +TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_table(TStringType{table}); + *request.mutable_rows()->mutable_type() = TProtoAccessor::GetProto(rows.GetType()); + *request.mutable_rows()->mutable_value() = rows.GetProto(); + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Y_UNUSED(any); + TBulkUpsertResult val(TStatus(std::move(status))); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncBulkUpsert, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); +} + +TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, EDataFormat format, + const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_table(TStringType{table}); + if (format == EDataFormat::ApacheArrow) { + request.mutable_arrow_batch_settings()->set_schema(TStringType{schema}); + } else if (format == EDataFormat::CSV) { + auto* csv_settings = request.mutable_csv_settings(); + const auto& format_settings = settings.FormatSettings_; + if (!format_settings.empty()) { + bool ok = csv_settings->ParseFromString(TStringType{format_settings}); + if (!ok) { + return {}; + } + } + } + request.set_data(TStringType{data}); + + auto promise = NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Y_UNUSED(any); + TBulkUpsertResult val(TStatus(std::move(status))); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncBulkUpsert, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); +} + +TFuture> TTableClient::TImpl::StreamExecuteScanQueryInternal(const std::string& query, + const ::google::protobuf::Map* params, + const TStreamExecScanQuerySettings& settings) +{ + auto request = MakeRequest(); + request.mutable_query()->set_yql_text(TStringType{query}); + + if (params) { + *request.mutable_parameters() = *params; + } + + if (settings.Explain_) { + request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN); + } else { + request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC); + } + + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + request.set_collect_full_diagnostics(settings.CollectFullDiagnostics_); + + auto promise = NewPromise>(); + + Connections_->StartReadStream< + Ydb::Table::V1::TableService, + Ydb::Table::ExecuteScanQueryRequest, + Ydb::Table::ExecuteScanQueryPartialResponse> + ( + std::move(request), + [promise] (TPlainStatus status, TScanQueryProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Table::V1::TableService::Stub::AsyncStreamExecuteScanQuery, + DbDriverState_, + TRpcRequestSettings::Make(settings) + ); + + return promise.GetFuture(); +} + +TAsyncScanQueryPartIterator TTableClient::TImpl::StreamExecuteScanQuery(const std::string& query, + const ::google::protobuf::Map* params, + const TStreamExecScanQuerySettings& settings) +{ + auto promise = NewPromise(); + + auto iteratorCallback = [promise](TFuture> future) mutable + { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TScanQueryPartIterator( + pair.second + ? std::make_shared(pair.second, pair.first.Endpoint) + : nullptr, + std::move(pair.first)) + ); + }; + + StreamExecuteScanQueryInternal(query, params, settings).Subscribe(iteratorCallback); + return promise.GetFuture(); +} + +// void TTableClient::TImpl::CloseAndDeleteSession( +// std::unique_ptr&& impl, +// std::shared_ptr client); + + +void TTableClient::TImpl::SetParams( + ::google::protobuf::Map* params, + Ydb::Table::ExecuteDataQueryRequest* request) +{ + if (params) { + request->mutable_parameters()->swap(*params); + } +} + +void TTableClient::TImpl::SetParams( + const ::google::protobuf::Map& params, + Ydb::Table::ExecuteDataQueryRequest* request) +{ + *request->mutable_parameters() = params; +} + +void TTableClient::TImpl::CollectParams( + ::google::protobuf::Map* params, + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram) +{ + + if (params && histgoram.IsCollecting()) { + size_t size = 0; + for (auto& keyvalue: *params) { + size += keyvalue.second.ByteSizeLong(); + } + histgoram.Record(size); + } +} + +void TTableClient::TImpl::CollectParams( + const ::google::protobuf::Map& params, + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram) +{ + + if (histgoram.IsCollecting()) { + size_t size = 0; + for (auto& keyvalue: params) { + size += keyvalue.second.ByteSizeLong(); + } + histgoram.Record(size); + } +} + +void TTableClient::TImpl::CollectQuerySize(const std::string& query, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram>& querySizeHistogram) { + if (querySizeHistogram.IsCollecting()) { + querySizeHistogram.Record(query.size()); + } +} + +void TTableClient::TImpl::CollectQuerySize(const TDataQuery&, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram>&) {} + +void TTableClient::TImpl::SetTxSettings(const TTxSettings& txSettings, Ydb::Table::TransactionSettings* proto) +{ + switch (txSettings.Mode_) { + case TTxSettings::TS_SERIALIZABLE_RW: + proto->mutable_serializable_read_write(); + break; + case TTxSettings::TS_ONLINE_RO: + proto->mutable_online_read_only()->set_allow_inconsistent_reads( + txSettings.OnlineSettings_.AllowInconsistentReads_); + break; + case TTxSettings::TS_STALE_RO: + proto->mutable_stale_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RO: + proto->mutable_snapshot_read_only(); + break; + case TTxSettings::TS_SNAPSHOT_RW: + proto->mutable_snapshot_read_write(); + break; + default: + throw TContractViolation("Unexpected transaction mode."); + } +} + +void TTableClient::TImpl::SetQuery(const std::string& queryText, Ydb::Table::Query* query) { + query->set_yql_text(TStringType{queryText}); +} + +void TTableClient::TImpl::SetQuery(const TDataQuery& queryData, Ydb::Table::Query* query) { + query->set_id(TStringType{queryData.GetId()}); +} + +void TTableClient::TImpl::SetQueryCachePolicy(const std::string&, const TExecDataQuerySettings& settings, + Ydb::Table::QueryCachePolicy* queryCachePolicy) +{ + queryCachePolicy->set_keep_in_cache(settings.KeepInQueryCache_.value_or(false)); +} + +void TTableClient::TImpl::SetQueryCachePolicy(const TDataQuery&, const TExecDataQuerySettings& settings, + Ydb::Table::QueryCachePolicy* queryCachePolicy) { + queryCachePolicy->set_keep_in_cache(settings.KeepInQueryCache_.value_or(true)); +} + +std::optional TTableClient::TImpl::GetQueryText(const std::string& queryText) { + return queryText; +} + +std::optional TTableClient::TImpl::GetQueryText(const TDataQuery& queryData) { + return queryData.GetText(); +} + +void TTableClient::TImpl::CollectRetryStatAsync(EStatus status) { + RetryOperationStatCollector.IncAsyncRetryOperation(status); +} + +void TTableClient::TImpl::CollectRetryStatSync(EStatus status) { + RetryOperationStatCollector.IncSyncRetryOperation(status); +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/impl/table_client.h b/ydb/public/sdk/cpp/src/client/table/impl/table_client.h new file mode 100644 index 000000000000..002214d6da55 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/table_client.h @@ -0,0 +1,325 @@ +#pragma once + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#include +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include + +#include "client_session.h" +#include "data_query.h" +#include "request_migrator.h" +#include "readers.h" + + +namespace NYdb::inline V3 { +namespace NTable { + +//How ofter run host scan to perform session balancing +constexpr TDuration HOSTSCAN_PERIODIC_ACTION_INTERVAL = TDuration::Seconds(2); +constexpr TDuration KEEP_ALIVE_CLIENT_TIMEOUT = TDuration::Seconds(5); + +TDuration GetMinTimeToTouch(const TSessionPoolSettings& settings); +TDuration GetMaxTimeToTouch(const TSessionPoolSettings& settings); + +class TTableClient::TImpl: public TClientImplCommon, public ISessionClient { +public: + using TReadTableStreamProcessorPtr = TTablePartIterator::TReaderImpl::TStreamProcessorPtr; + using TScanQueryProcessorPtr = TScanQueryPartIterator::TReaderImpl::TStreamProcessorPtr; + + TImpl(std::shared_ptr&& connections, const TClientSettings& settings); + ~TImpl(); + + bool LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag); + void InitStopper(); + NThreading::TFuture Drain(); + NThreading::TFuture Stop(); + void ScheduleTaskUnsafe(std::function&& fn, TDuration timeout); + void StartPeriodicSessionPoolTask(); + static ui64 ScanForeignLocations(std::shared_ptr client); + static std::pair ScanLocation(std::shared_ptr client, + std::unordered_map& sessions, bool allNodes); + static NMath::TStats CalcCV(const std::unordered_map& in); + void StartPeriodicHostScanTask(); + + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings); + i64 GetActiveSessionCount() const; + i64 GetActiveSessionsLimit() const; + i64 GetCurrentPoolSize() const; + TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings, bool standalone, + std::string preferredLocation = std::string()); + TAsyncKeepAliveResult KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings); + + TFuture CreateTable(Ydb::Table::CreateTableRequest&& request, const TCreateTableSettings& settings); + TFuture AlterTable(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings); + TAsyncOperation AlterTableLong(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings); + TFuture CopyTable(const std::string& sessionId, const std::string& src, const std::string& dst, + const TCopyTableSettings& settings); + TFuture CopyTables(Ydb::Table::CopyTablesRequest&& request, const TCopyTablesSettings& settings); + TFuture RenameTables(Ydb::Table::RenameTablesRequest&& request, const TRenameTablesSettings& settings); + TFuture DropTable(const std::string& sessionId, const std::string& path, const TDropTableSettings& settings); + TAsyncDescribeTableResult DescribeTable(const std::string& sessionId, const std::string& path, const TDescribeTableSettings& settings); + + template + TAsyncDataQueryResult ExecuteDataQuery(TSession& session, const std::string& query, const TTxControl& txControl, + TParamsType params, const TExecDataQuerySettings& settings) { + auto maybeQuery = session.SessionImpl_->GetQueryFromCache(query, Settings_.AllowRequestMigration_); + if (maybeQuery) { + TDataQuery dataQuery(session, query, maybeQuery->QueryId, maybeQuery->ParameterTypes); + return ExecuteDataQuery(session, dataQuery, txControl, params, settings, true); + } + + CacheMissCounter.Inc(); + + return ::NYdb::NSessionPool::InjectSessionStatusInterception(session.SessionImpl_, + ExecuteDataQueryImpl(session, query, txControl, params, settings, false), + true, GetMinTimeToTouch(Settings_.SessionPoolSettings_)); + } + + template + TAsyncDataQueryResult ExecuteDataQuery(TSession& session, const TDataQuery& dataQuery, const TTxControl& txControl, + TParamsType params, const TExecDataQuerySettings& settings, + bool fromCache) { + std::string queryKey = dataQuery.Impl_->GetTextHash(); + auto cb = [queryKey](const TDataQueryResult& result, TKqpSessionCommon& session) { + if (result.GetStatus() == EStatus::NOT_FOUND) { + static_cast(session).InvalidateQueryInCache(queryKey); + } + }; + + return ::NYdb::NSessionPool::InjectSessionStatusInterception( + session.SessionImpl_, + session.Client_->ExecuteDataQueryImpl(session, dataQuery, txControl, params, settings, fromCache), + true, + GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_), + cb); + } + + TAsyncPrepareQueryResult PrepareDataQuery(const TSession& session, const std::string& query, + const TPrepareDataQuerySettings& settings); + TAsyncStatus ExecuteSchemeQuery(const std::string& sessionId, const std::string& query, + const TExecSchemeQuerySettings& settings); + + TAsyncBeginTransactionResult BeginTransaction(const TSession& session, const TTxSettings& txSettings, + const TBeginTxSettings& settings); + TAsyncCommitTransactionResult CommitTransaction(const TSession& session, const std::string& txId, + const TCommitTxSettings& settings); + TAsyncStatus RollbackTransaction(const TSession& session, const std::string& txId, + const TRollbackTxSettings& settings); + + TAsyncExplainDataQueryResult ExplainDataQuery(const TSession& session, const std::string& query, + const TExplainDataQuerySettings& settings); + + static void SetTypedValue(Ydb::TypedValue* protoValue, const TValue& value); + + NThreading::TFuture> ReadTable( + const std::string& sessionId, + const std::string& path, + const TReadTableSettings& settings); + TAsyncReadRowsResult ReadRows(const std::string& path, TValue&& keys, const std::vector& columns, const TReadRowsSettings& settings); + + TAsyncStatus Close(const TKqpSessionCommon* sessionImpl, const TCloseSessionSettings& settings); + TAsyncStatus CloseInternal(const TKqpSessionCommon* sessionImpl); + + bool ReturnSession(TKqpSessionCommon* sessionImpl) override; + void DeleteSession(TKqpSessionCommon* sessionImpl) override; + ui32 GetSessionRetryLimit() const; + + void SetStatCollector(const NSdkStats::TStatCollector::TClientStatCollector& collector); + + TAsyncBulkUpsertResult BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings); + TAsyncBulkUpsertResult BulkUpsert(const std::string& table, EDataFormat format, + const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings); + + TFuture> StreamExecuteScanQueryInternal(const std::string& query, + const ::google::protobuf::Map* params, + const TStreamExecScanQuerySettings& settings); + TAsyncScanQueryPartIterator StreamExecuteScanQuery(const std::string& query, + const ::google::protobuf::Map* params, + const TStreamExecScanQuerySettings& settings); + void CollectRetryStatAsync(EStatus status); + void CollectRetryStatSync(EStatus status); + +public: + TClientSettings Settings_; + +private: + static void SetParams( + ::google::protobuf::Map* params, + Ydb::Table::ExecuteDataQueryRequest* request); + + static void SetParams( + const ::google::protobuf::Map& params, + Ydb::Table::ExecuteDataQueryRequest* request); + + static void CollectParams( + ::google::protobuf::Map* params, + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram); + + static void CollectParams( + const ::google::protobuf::Map& params, + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram); + + static void CollectQuerySize(const std::string& query, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram>& querySizeHistogram); + + static void CollectQuerySize(const TDataQuery&, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram>&); + + template + TAsyncDataQueryResult ExecuteDataQueryImpl(const TSession& session, const TQueryType& query, + const TTxControl& txControl, TParamsType params, + const TExecDataQuerySettings& settings, bool fromCache + ) { + if (!txControl.Tx_.has_value() || !txControl.CommitTx_) { + return ExecuteDataQueryInternal(session, query, txControl, params, settings, fromCache); + } + + auto onPrecommitCompleted = [this, session, query, txControl, params, settings, fromCache](const NThreading::TFuture& f) { + TStatus status = f.GetValueSync(); + if (!status.IsSuccess()) { + return NThreading::MakeFuture(TDataQueryResult(std::move(status), + {}, + txControl.Tx_, + std::nullopt, + false, + std::nullopt)); + } + + return ExecuteDataQueryInternal(session, query, txControl, params, settings, fromCache); + }; + + return txControl.Tx_->Precommit().Apply(onPrecommitCompleted); + } + + template + TAsyncDataQueryResult ExecuteDataQueryInternal(const TSession& session, const TQueryType& query, + const TTxControl& txControl, TParamsType params, + const TExecDataQuerySettings& settings, bool fromCache + ) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{session.GetId()}); + auto txControlProto = request.mutable_tx_control(); + txControlProto->set_commit_tx(txControl.CommitTx_); + if (txControl.Tx_.has_value()) { + txControlProto->set_tx_id(TStringType{txControl.Tx_->GetId()}); + } else { + SetTxSettings(txControl.BeginTx_, txControlProto->mutable_begin_tx()); + } + + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + + SetQuery(query, request.mutable_query()); + CollectQuerySize(query, QuerySizeHistogram); + + SetParams(params, &request); + CollectParams(params, ParamsSizeHistogram); + + SetQueryCachePolicy(query, settings, request.mutable_query_cache_policy()); + + auto promise = NewPromise(); + bool keepInCache = settings.KeepInQueryCache_ && settings.KeepInQueryCache_.value(); + + // We don't want to delay call of TSession dtor, so we can't capture it by copy + // otherwise we break session pool and other clients logic. + // Same problem with TDataQuery and TTransaction + // + // The fast solution is: + // - create copy of TSession out of lambda + // - capture pointer + // - call free just before SetValue call + auto sessionPtr = new TSession(session); + auto extractor = [promise, sessionPtr, query, fromCache, keepInCache] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector res; + std::optional tx; + std::optional dataQuery; + std::optional queryStats; + + auto queryText = GetQueryText(query); + if (any) { + Ydb::Table::ExecuteQueryResult result; + any->UnpackTo(&result); + + for (size_t i = 0; i < static_cast(result.result_sets_size()); i++) { + res.push_back(TResultSet(*result.mutable_result_sets(i))); + } + + if (result.has_tx_meta()) { + tx = TTransaction(*sessionPtr, result.tx_meta().id()); + } + + if (result.has_query_meta()) { + if (queryText) { + auto& query_meta = result.query_meta(); + dataQuery = TDataQuery(*sessionPtr, *queryText, query_meta.id(), query_meta.parameters_types()); + } + } + + if (result.has_query_stats()) { + queryStats = TQueryStats(result.query_stats()); + } + } + + if (keepInCache && dataQuery && queryText) { + sessionPtr->SessionImpl_->AddQueryToCache(*dataQuery); + } + + TDataQueryResult dataQueryResult(TStatus(std::move(status)), + std::move(res), tx, dataQuery, fromCache, queryStats); + + delete sessionPtr; + tx.reset(); + dataQuery.reset(); + promise.SetValue(std::move(dataQueryResult)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncExecuteDataQuery, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()) + ); + + return promise.GetFuture(); + } + + static void SetTxSettings(const TTxSettings& txSettings, Ydb::Table::TransactionSettings* proto); + + static void SetQuery(const std::string& queryText, Ydb::Table::Query* query); + + static void SetQuery(const TDataQuery& queryData, Ydb::Table::Query* query); + + static void SetQueryCachePolicy(const std::string&, const TExecDataQuerySettings& settings, + Ydb::Table::QueryCachePolicy* queryCachePolicy); + + static void SetQueryCachePolicy(const TDataQuery&, const TExecDataQuerySettings& settings, + Ydb::Table::QueryCachePolicy* queryCachePolicy); + + static std::optional GetQueryText(const std::string& queryText); + + static std::optional GetQueryText(const TDataQuery& queryData); + +public: + NSdkStats::TAtomicCounter<::NMonitoring::TRate> CacheMissCounter; + NSdkStats::TStatCollector::TClientRetryOperationStatCollector RetryOperationStatCollector; + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> QuerySizeHistogram; + NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> ParamsSizeHistogram; + NSdkStats::TAtomicCounter<::NMonitoring::TRate> SessionRemovedDueBalancing; + NSdkStats::TAtomicCounter<::NMonitoring::TRate> RequestMigrated; + +private: + NSessionPool::TSessionPool SessionPool_; + TRequestMigrator RequestMigrator_; + static const TKeepAliveSettings KeepAliveSettings; +}; + +} +} diff --git a/ydb/public/sdk/cpp/src/client/table/impl/transaction.cpp b/ydb/public/sdk/cpp/src/client/table/impl/transaction.cpp new file mode 100644 index 000000000000..6627804a1863 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/transaction.cpp @@ -0,0 +1,74 @@ +#include "transaction.h" +#include "table_client.h" + +namespace NYdb::inline V3::NTable { + +TTransaction::TImpl::TImpl(const TSession& session, const std::string& txId) + : Session_(session) + , TxId_(txId) +{ +} + +TAsyncStatus TTransaction::TImpl::Precommit() const +{ + auto result = NThreading::MakeFuture(TStatus(EStatus::SUCCESS, {})); + + for (auto& callback : PrecommitCallbacks) { + if (!callback) { + continue; + } + + // If you send multiple requests in parallel, the `KQP` service can respond with `SESSION_BUSY`. + // Therefore, precommit operations are performed sequentially. Here we capture the closure to + // trigger it later. + auto action = [callback = std::move(callback)](const TAsyncStatus& prev) { + if (const TStatus& status = prev.GetValue(); !status.IsSuccess()) { + return prev; + } + + return callback(); + }; + + result = result.Apply(action); + } + + return result; +} + +TAsyncCommitTransactionResult TTransaction::TImpl::Commit(const TCommitTxSettings& settings) +{ + ChangesAreAccepted = false; + + auto result = Precommit(); + + auto precommitsCompleted = [this, settings](const TAsyncStatus& result) mutable { + if (const TStatus& status = result.GetValue(); !status.IsSuccess()) { + return NThreading::MakeFuture(TCommitTransactionResult(TStatus(status), std::nullopt)); + } + + PrecommitCallbacks.clear(); + + return Session_.Client_->CommitTransaction(Session_, + TxId_, + settings); + }; + + return result.Apply(precommitsCompleted); +} + +TAsyncStatus TTransaction::TImpl::Rollback(const TRollbackTxSettings& settings) +{ + ChangesAreAccepted = false; + return Session_.Client_->RollbackTransaction(Session_, TxId_, settings); +} + +void TTransaction::TImpl::AddPrecommitCallback(TPrecommitTransactionCallback cb) +{ + if (!ChangesAreAccepted) { + ythrow TContractViolation("Changes are no longer accepted"); + } + + PrecommitCallbacks.push_back(std::move(cb)); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/table/impl/transaction.h b/ydb/public/sdk/cpp/src/client/table/impl/transaction.h new file mode 100644 index 000000000000..c226d8ca3501 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/transaction.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NTable { + +class TTransaction::TImpl { +public: + TImpl(const TSession& session, const std::string& txId); + + const std::string& GetId() const { + return TxId_; + } + + bool IsActive() const { + return !TxId_.empty(); + } + + TAsyncStatus Precommit() const; + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); + TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); + + TSession GetSession() const { + return Session_; + } + + void AddPrecommitCallback(TPrecommitTransactionCallback cb); + +private: + TSession Session_; + std::string TxId_; + + bool ChangesAreAccepted = true; // haven't called Commit or Rollback yet + mutable std::vector PrecommitCallbacks; +}; + +} diff --git a/ydb/public/sdk/cpp/src/client/table/impl/ya.make b/ydb/public/sdk/cpp/src/client/table/impl/ya.make new file mode 100644 index 000000000000..158409f4bbe0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/impl/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + client_session.cpp + data_query.cpp + readers.cpp + request_migrator.cpp + table_client.cpp + transaction.cpp +) + +PEERDIR( + library/cpp/threading/future + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/impl/ydb_endpoints + ydb/public/sdk/cpp/src/client/impl/ydb_internal/session_pool + ydb/public/sdk/cpp/src/client/table/query_stats + ydb/public/sdk/cpp/src/library/string_utils/helpers +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/table/out.cpp b/ydb/public/sdk/cpp/src/client/table/out.cpp new file mode 100644 index 000000000000..9d7a957219ee --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/out.cpp @@ -0,0 +1,85 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TCopyItem, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TIndexDescription, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TChangefeedDescription, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TValueSinceUnixEpochModeSettings::EUnit, o, x) { + return NYdb::NTable::TValueSinceUnixEpochModeSettings::Out(o, x); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TTxSettings, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TCreateSessionResult, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TDescribeTableResult, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EMetric, stream, value) { + auto convertDistance = [&] { + switch (value) { + case NYdb::NTable::TVectorIndexSettings::EMetric::InnerProduct: + return "similarity: inner_product"; + case NYdb::NTable::TVectorIndexSettings::EMetric::CosineSimilarity: + return "similarity: cosine"; + case NYdb::NTable::TVectorIndexSettings::EMetric::CosineDistance: + return "distance: cosine"; + case NYdb::NTable::TVectorIndexSettings::EMetric::Manhattan: + return "distance: manhattan"; + case NYdb::NTable::TVectorIndexSettings::EMetric::Euclidean: + return "distance: euclidean"; + case NYdb::NTable::TVectorIndexSettings::EMetric::Unspecified: + return "metric: unspecified"; + } + }; + + stream << convertDistance(); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EVectorType, stream, value) { + auto convertVectorType = [&] { + switch (value) { + case NYdb::NTable::TVectorIndexSettings::EVectorType::Float: + return "float"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8: + return "uint8"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Int8: + return "int8"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Bit: + return "bit"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Unspecified: + return "unspecified"; + } + }; + + stream << convertVectorType(); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings, stream, value) { + stream << + "{ " << value.Metric << + ", vector_type: " << value.VectorType << + ", vector_dimension: " << value.VectorDimension << + " }"; +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TKMeansTreeSettings, stream, value) { + stream << + "{ settings: " << value.Settings << + ", clusters: " << value.Clusters << + ", levels: " << value.Levels << + " }"; +} diff --git a/ydb/public/sdk/cpp/src/client/table/proto_accessor.cpp b/ydb/public/sdk/cpp/src/client/table/proto_accessor.cpp new file mode 100644 index 000000000000..98224b2578d7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/proto_accessor.cpp @@ -0,0 +1,69 @@ +#include + +#include + +namespace NYdb::inline V3 { + +const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) { + return queryStats.GetProto(); +} + +const Ydb::Table::DescribeTableResult& TProtoAccessor::GetProto(const NTable::TTableDescription& tableDescription) { + return tableDescription.GetProto(); +} + +NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) { + return NTable::TQueryStats(queryStats); +} + +NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTableRequest& request) { + return NTable::TTableDescription(request); +} + +NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex& tableIndex) { + return NTable::TIndexDescription(tableIndex); +} + +NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc) { + return NTable::TIndexDescription(tableIndexDesc); +} + +NTable::TChangefeedDescription TProtoAccessor::FromProto(const Ydb::Table::Changefeed& changefeed) { + return NTable::TChangefeedDescription(changefeed); +} + +NTable::TChangefeedDescription TProtoAccessor::FromProto(const Ydb::Table::ChangefeedDescription& changefeed) { + return NTable::TChangefeedDescription(changefeed); +} + +Ydb::Table::ValueSinceUnixEpochModeSettings::Unit TProtoAccessor::GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value) { + switch (value) { + case NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_UNSPECIFIED; + } +} + +NTable::TValueSinceUnixEpochModeSettings::EUnit TProtoAccessor::FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value) { + switch (value) { + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds; + default: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown; + } +} + +} diff --git a/ydb/public/sdk/cpp/src/client/table/query_stats/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/table/query_stats/CMakeLists.txt new file mode 100644 index 000000000000..4240c6dcc21c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/query_stats/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(client-ydb_table-query_stats) + +target_link_libraries(client-ydb_table-query_stats PUBLIC + yutil + protobuf::libprotobuf + api-protos + client-ydb_query +) + +target_sources(client-ydb_table-query_stats PRIVATE + stats.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_table-query_stats) diff --git a/ydb/public/sdk/cpp/src/client/table/query_stats/stats.cpp b/ydb/public/sdk/cpp/src/client/table/query_stats/stats.cpp new file mode 100644 index 000000000000..cf683df98249 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/query_stats/stats.cpp @@ -0,0 +1,36 @@ +#include + +namespace NYdb::inline V3 { +namespace NTable { + +std::optional ParseQueryStatsMode(std::string_view statsMode) +{ + if (statsMode == "none") { + return ECollectQueryStatsMode::None; + } else if (statsMode == "basic") { + return ECollectQueryStatsMode::Basic; + } else if (statsMode == "full") { + return ECollectQueryStatsMode::Full; + } else if (statsMode == "profile") { + return ECollectQueryStatsMode::Profile; + } + + return {}; +} + +std::string_view QueryStatsModeToString(ECollectQueryStatsMode statsMode) +{ + switch (statsMode) { + case ECollectQueryStatsMode::None: + return "none"; + case ECollectQueryStatsMode::Basic: + return "basic"; + case ECollectQueryStatsMode::Full: + return "full"; + case ECollectQueryStatsMode::Profile: + return "profile"; + } +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/query_stats/ya.make b/ydb/public/sdk/cpp/src/client/table/query_stats/ya.make new file mode 100644 index 000000000000..59ed87e31f6f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/query_stats/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + stats.cpp +) + +PEERDIR( + contrib/libs/protobuf + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/query +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/table/table.cpp b/ydb/public/sdk/cpp/src/client/table/table.cpp new file mode 100644 index 000000000000..b9983c84a49c --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/table.cpp @@ -0,0 +1,3310 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#include +#include +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NTable { + +using namespace NThreading; +using namespace NSessionPool; + +using TRetryContextAsync = NRetry::Async::TRetryContext; + +//////////////////////////////////////////////////////////////////////////////// + +class TStorageSettings::TImpl { +public: + TImpl() { } + + explicit TImpl(const Ydb::Table::StorageSettings& proto) + : Proto_(proto) + { } + +public: + const Ydb::Table::StorageSettings Proto_; +}; + +TStorageSettings::TStorageSettings() + : Impl_(std::make_shared()) +{ } + +TStorageSettings::TStorageSettings(const Ydb::Table::StorageSettings& proto) + : Impl_(std::make_shared(proto)) +{ } + +const Ydb::Table::StorageSettings& TStorageSettings::GetProto() const { + return Impl_->Proto_; +} + +std::optional TStorageSettings::GetTabletCommitLog0() const { + if (GetProto().has_tablet_commit_log0()) { + return GetProto().tablet_commit_log0().media(); + } else { + return { }; + } +} + +std::optional TStorageSettings::GetTabletCommitLog1() const { + if (GetProto().has_tablet_commit_log1()) { + return GetProto().tablet_commit_log1().media(); + } else { + return { }; + } +} + +std::optional TStorageSettings::GetExternal() const { + if (GetProto().has_external()) { + return GetProto().external().media(); + } else { + return { }; + } +} + +std::optional TStorageSettings::GetStoreExternalBlobs() const { + switch (GetProto().store_external_blobs()) { + case Ydb::FeatureFlag::ENABLED: + return true; + case Ydb::FeatureFlag::DISABLED: + return false; + default: + return { }; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TColumnFamilyDescription::TImpl { +public: + explicit TImpl(const Ydb::Table::ColumnFamily& desc) + : Proto_(desc) + { } + +public: + const Ydb::Table::ColumnFamily Proto_; +}; + +TColumnFamilyDescription::TColumnFamilyDescription(const Ydb::Table::ColumnFamily& desc) + : Impl_(std::make_shared(desc)) +{ } + +const Ydb::Table::ColumnFamily& TColumnFamilyDescription::GetProto() const { + return Impl_->Proto_; +} + +const std::string& TColumnFamilyDescription::GetName() const { + return GetProto().name(); +} + +std::optional TColumnFamilyDescription::GetData() const { + if (GetProto().has_data()) { + return GetProto().data().media(); + } else { + return { }; + } +} + +std::optional TColumnFamilyDescription::GetCompression() const { + switch (GetProto().compression()) { + case Ydb::Table::ColumnFamily::COMPRESSION_NONE: + return EColumnFamilyCompression::None; + case Ydb::Table::ColumnFamily::COMPRESSION_LZ4: + return EColumnFamilyCompression::LZ4; + default: + return { }; + } +} + +std::optional TColumnFamilyDescription::GetKeepInMemory() const { + switch (GetProto().keep_in_memory()) { + case Ydb::FeatureFlag::ENABLED: + return true; + case Ydb::FeatureFlag::DISABLED: + return false; + default: + return { }; + } +} + +TBuildIndexOperation::TBuildIndexOperation(TStatus &&status, Ydb::Operations::Operation &&operation) + : TOperation(std::move(status), std::move(operation)) +{ + Ydb::Table::IndexBuildMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + Metadata_.State = static_cast(metadata.state()); + Metadata_.Progress = metadata.progress(); + const auto& desc = metadata.description(); + Metadata_.Path = desc.path(); + Metadata_.Desctiption = TProtoAccessor::FromProto(desc.index()); +} + +const TBuildIndexOperation::TMetadata& TBuildIndexOperation::Metadata() const { + return Metadata_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPartitioningSettings::TImpl { +public: + TImpl() { } + + explicit TImpl(const Ydb::Table::PartitioningSettings& proto) + : Proto_(proto) + { } + +public: + const Ydb::Table::PartitioningSettings Proto_; +}; + +TPartitioningSettings::TPartitioningSettings() + : Impl_(std::make_shared()) +{ } + +TPartitioningSettings::TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto) + : Impl_(std::make_shared(proto)) +{ } + +const Ydb::Table::PartitioningSettings& TPartitioningSettings::GetProto() const { + return Impl_->Proto_; +} + +std::optional TPartitioningSettings::GetPartitioningBySize() const { + switch (GetProto().partitioning_by_size()) { + case Ydb::FeatureFlag::ENABLED: + return true; + case Ydb::FeatureFlag::DISABLED: + return false; + default: + return { }; + } +} + +std::optional TPartitioningSettings::GetPartitioningByLoad() const { + switch (GetProto().partitioning_by_load()) { + case Ydb::FeatureFlag::ENABLED: + return true; + case Ydb::FeatureFlag::DISABLED: + return false; + default: + return { }; + } +} + +uint64_t TPartitioningSettings::GetPartitionSizeMb() const { + return GetProto().partition_size_mb(); +} + +uint64_t TPartitioningSettings::GetMinPartitionsCount() const { + return GetProto().min_partitions_count(); +} + +uint64_t TPartitioningSettings::GetMaxPartitionsCount() const { + return GetProto().max_partitions_count(); +} + +//////////////////////////////////////////////////////////////////////////////// + +struct TTableStats { + uint64_t Rows = 0; + uint64_t Size = 0; + uint64_t Partitions = 0; + TInstant ModificationTime; + TInstant CreationTime; +}; + +static TInstant ProtobufTimestampToTInstant(const google::protobuf::Timestamp& timestamp) { + uint64_t lastModificationUs = timestamp.seconds() * 1000000; + lastModificationUs += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(lastModificationUs); +} + +static void SerializeTo(const TRenameIndex& rename, Ydb::Table::RenameIndexItem& proto) { + proto.set_source_name(TStringType{rename.SourceName_}); + proto.set_destination_name(TStringType{rename.DestinationName_}); + proto.set_replace_destination(rename.ReplaceDestination_); +} + +TExplicitPartitions TExplicitPartitions::FromProto(const Ydb::Table::ExplicitPartitions& proto) { + TExplicitPartitions out; + for (const auto& splitPoint : proto.split_points()) { + TValue value(TType(splitPoint.type()), splitPoint.value()); + out.AppendSplitPoints(value); + } + return out; +} + +void TExplicitPartitions::SerializeTo(Ydb::Table::ExplicitPartitions& proto) const { + for (const auto& splitPoint : SplitPoints_) { + auto* boundary = proto.add_split_points(); + boundary->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); + boundary->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); + } +} + +class TTableDescription::TImpl { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + template + TImpl(const TProto& proto) + : StorageSettings_(proto.storage_settings()) + , PartitioningSettings_(proto.partitioning_settings()) + , HasStorageSettings_(proto.has_storage_settings()) + , HasPartitioningSettings_(proto.has_partitioning_settings()) + { + // primary key + for (const auto& pk : proto.primary_key()) { + PrimaryKey_.push_back(pk); + } + + // columns + for (const auto& col : proto.columns()) { + std::optional not_null; + if (col.has_not_null()) { + not_null = col.not_null(); + } + std::optional sequenceDescription; + switch (col.default_value_case()) { + case Ydb::Table::ColumnMeta::kFromSequence: { + if (col.from_sequence().name() == "_serial_column_" + col.name()) { + TSequenceDescription currentSequenceDescription; + if (col.from_sequence().has_set_val()) { + TSequenceDescription::TSetVal setVal; + setVal.NextUsed = col.from_sequence().set_val().next_used(); + setVal.NextValue = col.from_sequence().set_val().next_value(); + currentSequenceDescription.SetVal = std::move(setVal); + } + sequenceDescription = std::move(currentSequenceDescription); + } + break; + } + default: break; + } + Columns_.emplace_back(col.name(), col.type(), col.family(), not_null, std::move(sequenceDescription)); + } + + // indexes + Indexes_.reserve(proto.indexes_size()); + for (const auto& index : proto.indexes()) { + Indexes_.emplace_back(TProtoAccessor::FromProto(index)); + } + + if constexpr (std::is_same_v) { + // changefeeds + Changefeeds_.reserve(proto.changefeeds_size()); + for (const auto& changefeed : proto.changefeeds()) { + Changefeeds_.emplace_back(TProtoAccessor::FromProto(changefeed)); + } + } + + // ttl settings + if (auto ttlSettings = TTtlSettings::FromProto(proto.ttl_settings())) { + TtlSettings_ = std::move(*ttlSettings); + } + + if (proto.store_type()) { + StoreType_ = (proto.store_type() == Ydb::Table::STORE_TYPE_COLUMN) ? EStoreType::Column : EStoreType::Row; + } + + // column families + ColumnFamilies_.reserve(proto.column_families_size()); + for (const auto& family : proto.column_families()) { + ColumnFamilies_.emplace_back(family); + } + + // attributes + for (auto [key, value] : proto.attributes()) { + Attributes_[key] = value; + } + + // key bloom filter + switch (proto.key_bloom_filter()) { + case Ydb::FeatureFlag::ENABLED: + KeyBloomFilter_ = true; + break; + case Ydb::FeatureFlag::DISABLED: + KeyBloomFilter_ = false; + break; + default: + break; + } + + // read replicas settings + if (proto.has_read_replicas_settings()) { + const auto settings = proto.read_replicas_settings(); + switch (settings.settings_case()) { + case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: + ReadReplicasSettings_ = TReadReplicasSettings( + TReadReplicasSettings::EMode::PerAz, + settings.per_az_read_replicas_count()); + break; + case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: + ReadReplicasSettings_ = TReadReplicasSettings( + TReadReplicasSettings::EMode::AnyAz, + settings.any_az_read_replicas_count()); + break; + default: + break; + } + } + } + +public: + TImpl(Ydb::Table::DescribeTableResult&& desc, const TDescribeTableSettings& describeSettings) + : TImpl(desc) + { + Proto_ = std::move(desc); + + Owner_ = Proto_.self().owner(); + PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); + PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); + + std::optional leftValue; + for (const auto& bound : Proto_.shard_key_bounds()) { + std::optional fromBound = leftValue + ? TKeyBound::Inclusive(*leftValue) + : std::optional(); + + TValue value(TType(bound.type()), bound.value()); + const TKeyBound& toBound = TKeyBound::Exclusive(value); + + Ranges_.emplace_back(TKeyRange(fromBound, toBound)); + leftValue = value; + } + + for (const auto& shardStats : Proto_.table_stats().partition_stats()) { + PartitionStats_.emplace_back(TPartitionStats{ shardStats.rows_estimate(), shardStats.store_size(), shardStats.leader_node_id() }); + } + + TableStats.Rows = Proto_.table_stats().rows_estimate(); + TableStats.Size = Proto_.table_stats().store_size(); + TableStats.Partitions = Proto_.table_stats().partitions(); + + TableStats.ModificationTime = ProtobufTimestampToTInstant(Proto_.table_stats().modification_time()); + TableStats.CreationTime = ProtobufTimestampToTInstant(Proto_.table_stats().creation_time()); + + if (describeSettings.WithKeyShardBoundary_) { + Ranges_.emplace_back(TKeyRange( + leftValue ? TKeyBound::Inclusive(*leftValue) : std::optional(), + std::optional())); + } + } + + struct TCreateTableRequestTag {}; // to avoid delegation cycle + + TImpl(const Ydb::Table::CreateTableRequest& request, TCreateTableRequestTag) + : TImpl(request) + { + if (!request.compaction_policy().empty()) { + SetCompactionPolicy(request.compaction_policy()); + } + + switch (request.partitions_case()) { + case Ydb::Table::CreateTableRequest::kUniformPartitions: + SetUniformPartitions(request.uniform_partitions()); + break; + + case Ydb::Table::CreateTableRequest::kPartitionAtKeys: { + SetPartitionAtKeys(TExplicitPartitions::FromProto(request.partition_at_keys())); + break; + } + + default: + break; + } + } + + TImpl() = default; + + const Ydb::Table::DescribeTableResult& GetProto() const { + return Proto_; + } + + void AddColumn(const std::string& name, const Ydb::Type& type, const std::string& family, std::optional notNull, std::optional sequenceDescription) { + Columns_.emplace_back(name, type, family, notNull, std::move(sequenceDescription)); + } + + void SetPrimaryKeyColumns(const std::vector& primaryKeyColumns) { + PrimaryKey_ = primaryKeyColumns; + } + + void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns)); + } + + void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); + } + + void AddSecondaryIndex(const TIndexDescription& indexDescription) { + Indexes_.emplace_back(indexDescription); + } + + void AddVectorKMeansTreeIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, indexSettings)); + } + + void AddVectorKMeansTreeIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns, {}, indexSettings)); + } + + void AddChangefeed(const std::string& name, EChangefeedMode mode, EChangefeedFormat format) { + Changefeeds_.emplace_back(name, mode, format); + } + + void SetTtlSettings(TTtlSettings&& settings) { + TtlSettings_ = std::move(settings); + } + + void SetTtlSettings(const TTtlSettings& settings) { + TtlSettings_ = settings; + } + + void SetStorageSettings(const TStorageSettings& settings) { + StorageSettings_ = settings; + HasStorageSettings_ = true; + } + + void AddColumnFamily(const TColumnFamilyDescription& desc) { + ColumnFamilies_.emplace_back(desc); + } + + void AddAttribute(const std::string& key, const std::string& value) { + Attributes_[key] = value; + } + + void SetAttributes(const std::unordered_map& attrs) { + Attributes_ = attrs; + } + + void SetAttributes(std::unordered_map&& attrs) { + Attributes_ = std::move(attrs); + } + + void SetCompactionPolicy(const std::string& name) { + CompactionPolicy_ = name; + } + + void SetUniformPartitions(uint64_t partitionsCount) { + UniformPartitions_ = partitionsCount; + } + + void SetPartitionAtKeys(const TExplicitPartitions& keys) { + PartitionAtKeys_ = keys; + } + + void SetPartitioningSettings(const TPartitioningSettings& settings) { + PartitioningSettings_ = settings; + HasPartitioningSettings_ = true; + } + + void SetKeyBloomFilter(bool enabled) { + KeyBloomFilter_ = enabled; + } + + void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { + ReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); + } + + void SetStoreType(EStoreType type) { + StoreType_ = type; + } + + const std::vector& GetPrimaryKeyColumns() const { + return PrimaryKey_; + } + + const std::vector& GetColumns() const { + return Columns_; + } + + const std::vector& GetIndexDescriptions() const { + return Indexes_; + } + + const std::vector& GetChangefeedDescriptions() const { + return Changefeeds_; + } + + const std::optional& GetTtlSettings() const { + return TtlSettings_; + } + + EStoreType GetStoreType() const { + return StoreType_; + } + + const std::string& GetOwner() const { + return Owner_; + } + + const std::vector& GetPermissions() const { + return Permissions_; + } + + const std::vector& GetEffectivePermissions() const { + return EffectivePermissions_; + } + + const std::vector& GetKeyRanges() const { + return Ranges_; + } + + const std::vector& GetPartitionStats() const { + return PartitionStats_; + } + + const TTableStats& GetTableStats() const { + return TableStats; + } + + bool HasStorageSettings() const { + return HasStorageSettings_; + } + + const TStorageSettings& GetStorageSettings() const { + return StorageSettings_; + } + + const std::vector& GetColumnFamilies() const { + return ColumnFamilies_; + } + + const std::unordered_map& GetAttributes() const { + return Attributes_; + } + + const std::string& GetCompactionPolicy() const { + return CompactionPolicy_; + } + + const std::optional& GetUniformPartitions() const { + return UniformPartitions_; + } + + const std::optional& GetPartitionAtKeys() const { + return PartitionAtKeys_; + } + + bool HasPartitioningSettings() const { + return HasPartitioningSettings_; + } + + const TPartitioningSettings& GetPartitioningSettings() const { + return PartitioningSettings_; + } + + std::optional GetKeyBloomFilter() const { + return KeyBloomFilter_; + } + + const std::optional& GetReadReplicasSettings() const { + return ReadReplicasSettings_; + } + +private: + Ydb::Table::DescribeTableResult Proto_; + TStorageSettings StorageSettings_; + std::vector PrimaryKey_; + std::vector Columns_; + std::vector Indexes_; + std::vector Changefeeds_; + std::optional TtlSettings_; + std::string Owner_; + std::vector Permissions_; + std::vector EffectivePermissions_; + std::vector Ranges_; + std::vector PartitionStats_; + TTableStats TableStats; + std::vector ColumnFamilies_; + std::unordered_map Attributes_; + std::string CompactionPolicy_; + std::optional UniformPartitions_; + std::optional PartitionAtKeys_; + TPartitioningSettings PartitioningSettings_; + std::optional KeyBloomFilter_; + std::optional ReadReplicasSettings_; + bool HasStorageSettings_ = false; + bool HasPartitioningSettings_ = false; + EStoreType StoreType_ = EStoreType::Row; +}; + +TTableDescription::TTableDescription() + : Impl_(new TImpl) +{ +} + +TTableDescription::TTableDescription(Ydb::Table::DescribeTableResult&& desc, + const TDescribeTableSettings& describeSettings) + : Impl_(new TImpl(std::move(desc), describeSettings)) +{ +} + +TTableDescription::TTableDescription(const Ydb::Table::CreateTableRequest& request) + : Impl_(new TImpl(request, TImpl::TCreateTableRequestTag())) +{ +} + +const std::vector& TTableDescription::GetPrimaryKeyColumns() const { + return Impl_->GetPrimaryKeyColumns(); +} + +std::vector TTableDescription::GetColumns() const { + // Conversion to TColumn for API compatibility + const auto& columns = Impl_->GetColumns(); + std::vector legacy; + legacy.reserve(columns.size()); + for (const auto& column : columns) { + legacy.emplace_back(column.Name, column.Type); + } + return legacy; +} + +std::vector TTableDescription::GetTableColumns() const { + return Impl_->GetColumns(); +} + +std::vector TTableDescription::GetIndexDescriptions() const { + return Impl_->GetIndexDescriptions(); +} + +std::vector TTableDescription::GetChangefeedDescriptions() const { + return Impl_->GetChangefeedDescriptions(); +} + +std::optional TTableDescription::GetTtlSettings() const { + return Impl_->GetTtlSettings(); +} + +std::optional TTableDescription::GetTiering() const { + return std::nullopt; +} + +EStoreType TTableDescription::GetStoreType() const { + return Impl_->GetStoreType(); +} + +const std::string& TTableDescription::GetOwner() const { + return Impl_->GetOwner(); +} + +const std::vector& TTableDescription::GetPermissions() const { + return Impl_->GetPermissions(); +} + +const std::vector& TTableDescription::GetEffectivePermissions() const { + return Impl_->GetEffectivePermissions(); +} + +const std::vector& TTableDescription::GetKeyRanges() const { + return Impl_->GetKeyRanges(); +} + +void TTableDescription::AddColumn(const std::string& name, const Ydb::Type& type, const std::string& family, std::optional notNull, std::optional sequenceDescription) { + Impl_->AddColumn(name, type, family, notNull, std::move(sequenceDescription)); +} + +void TTableDescription::SetPrimaryKeyColumns(const std::vector& primaryKeyColumns) { + Impl_->SetPrimaryKeyColumns(primaryKeyColumns); +} + +void TTableDescription::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns) { + Impl_->AddSecondaryIndex(indexName, type, indexColumns); +} + +void TTableDescription::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns) { + Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); +} + +void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) { + Impl_->AddSecondaryIndex(indexDescription); +} + +void TTableDescription::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); +} + +void TTableDescription::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); +} + +void TTableDescription::AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); +} + +void TTableDescription::AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); +} + +void TTableDescription::AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns); +} + +void TTableDescription::AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns, dataColumns); +} + +void TTableDescription::AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { + Impl_->AddVectorKMeansTreeIndex(indexName, EIndexType::GlobalVectorKMeansTree, indexColumns, indexSettings); +} + +void TTableDescription::AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings) { + Impl_->AddVectorKMeansTreeIndex(indexName, EIndexType::GlobalVectorKMeansTree, indexColumns, dataColumns, indexSettings); +} + +void TTableDescription::AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + AddSyncSecondaryIndex(indexName, indexColumns); +} + +void TTableDescription::AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); +} + +void TTableDescription::SetTtlSettings(TTtlSettings&& settings) { + Impl_->SetTtlSettings(std::move(settings)); +} + +void TTableDescription::SetTtlSettings(const TTtlSettings& settings) { + Impl_->SetTtlSettings(settings); +} + +void TTableDescription::SetStorageSettings(const TStorageSettings& settings) { + Impl_->SetStorageSettings(settings); +} + +void TTableDescription::AddColumnFamily(const TColumnFamilyDescription& desc) { + Impl_->AddColumnFamily(desc); +} + +void TTableDescription::AddAttribute(const std::string& key, const std::string& value) { + Impl_->AddAttribute(key, value); +} + +void TTableDescription::SetAttributes(const std::unordered_map& attrs) { + Impl_->SetAttributes(attrs); +} + +void TTableDescription::SetAttributes(std::unordered_map&& attrs) { + Impl_->SetAttributes(std::move(attrs)); +} + +void TTableDescription::SetCompactionPolicy(const std::string& name) { + Impl_->SetCompactionPolicy(name); +} + +void TTableDescription::SetUniformPartitions(uint64_t partitionsCount) { + Impl_->SetUniformPartitions(partitionsCount); +} + +void TTableDescription::SetPartitionAtKeys(const TExplicitPartitions& keys) { + Impl_->SetPartitionAtKeys(keys); +} + +void TTableDescription::SetPartitioningSettings(const TPartitioningSettings& settings) { + Impl_->SetPartitioningSettings(settings); +} + +void TTableDescription::SetKeyBloomFilter(bool enabled) { + Impl_->SetKeyBloomFilter(enabled); +} + +void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { + Impl_->SetReadReplicasSettings(mode, readReplicasCount); +} + +void TTableDescription::SetStoreType(EStoreType type) { + Impl_->SetStoreType(type); +} + +const std::vector& TTableDescription::GetPartitionStats() const { + return Impl_->GetPartitionStats(); +} + +TInstant TTableDescription::GetModificationTime() const { + return Impl_->GetTableStats().ModificationTime; +} + +TInstant TTableDescription::GetCreationTime() const { + return Impl_->GetTableStats().CreationTime; +} + +uint64_t TTableDescription::GetTableSize() const { + return Impl_->GetTableStats().Size; +} + +uint64_t TTableDescription::GetTableRows() const { + return Impl_->GetTableStats().Rows; +} + +uint64_t TTableDescription::GetPartitionsCount() const { + return Impl_->GetTableStats().Partitions; +} + +const TStorageSettings& TTableDescription::GetStorageSettings() const { + return Impl_->GetStorageSettings(); +} + +const std::vector& TTableDescription::GetColumnFamilies() const { + return Impl_->GetColumnFamilies(); +} + +const std::unordered_map& TTableDescription::GetAttributes() const { + return Impl_->GetAttributes(); +} + +const TPartitioningSettings& TTableDescription::GetPartitioningSettings() const { + return Impl_->GetPartitioningSettings(); +} + +std::optional TTableDescription::GetKeyBloomFilter() const { + return Impl_->GetKeyBloomFilter(); +} + +std::optional TTableDescription::GetReadReplicasSettings() const { + return Impl_->GetReadReplicasSettings(); +} + +const Ydb::Table::DescribeTableResult& TTableDescription::GetProto() const { + return Impl_->GetProto(); +} + +void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) const { + for (const auto& column : Impl_->GetColumns()) { + auto& protoColumn = *request.add_columns(); + protoColumn.set_name(TStringType{column.Name}); + protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); + protoColumn.set_family(TStringType{column.Family}); + if (column.NotNull.has_value()) { + protoColumn.set_not_null(column.NotNull.value()); + } + if (column.SequenceDescription.has_value()) { + auto* fromSequence = protoColumn.mutable_from_sequence(); + if (column.SequenceDescription->SetVal.has_value()) { + auto* setVal = fromSequence->mutable_set_val(); + setVal->set_next_value(column.SequenceDescription->SetVal->NextValue); + setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed); + } + fromSequence->set_name("_serial_column_" + column.Name); + } + } + + for (const auto& pk : Impl_->GetPrimaryKeyColumns()) { + request.add_primary_key(TStringType{pk}); + } + + for (const auto& index : Impl_->GetIndexDescriptions()) { + index.SerializeTo(*request.add_indexes()); + } + + if (const auto& ttl = Impl_->GetTtlSettings()) { + ttl->SerializeTo(*request.mutable_ttl_settings()); + } + + if (Impl_->GetStoreType() == EStoreType::Column) { + request.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN); + } + + if (Impl_->HasStorageSettings()) { + request.mutable_storage_settings()->CopyFrom(Impl_->GetStorageSettings().GetProto()); + } + + for (const auto& family : Impl_->GetColumnFamilies()) { + auto* f = request.add_column_families(); + f->CopyFrom(family.GetProto()); + } + + for (const auto& [key, value] : Impl_->GetAttributes()) { + (*request.mutable_attributes())[key] = value; + } + + if (!Impl_->GetCompactionPolicy().empty()) { + request.set_compaction_policy(TStringType{Impl_->GetCompactionPolicy()}); + } + + if (const auto& uniformPartitions = Impl_->GetUniformPartitions()) { + request.set_uniform_partitions(uniformPartitions.value()); + } + + if (const auto& partitionAtKeys = Impl_->GetPartitionAtKeys()) { + partitionAtKeys->SerializeTo(*request.mutable_partition_at_keys()); + } else if (Impl_->GetProto().shard_key_bounds_size()) { + request.mutable_partition_at_keys()->mutable_split_points()->CopyFrom(Impl_->GetProto().shard_key_bounds()); + } + + if (Impl_->HasPartitioningSettings()) { + request.mutable_partitioning_settings()->CopyFrom(Impl_->GetPartitioningSettings().GetProto()); + } + + if (auto keyBloomFilter = Impl_->GetKeyBloomFilter()) { + if (keyBloomFilter.value()) { + request.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); + } else { + request.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); + } + } + + if (const auto& settings = Impl_->GetReadReplicasSettings()) { + switch (settings->GetMode()) { + case TReadReplicasSettings::EMode::PerAz: + request.mutable_read_replicas_settings()->set_per_az_read_replicas_count(settings->GetReadReplicasCount()); + break; + case TReadReplicasSettings::EMode::AnyAz: + request.mutable_read_replicas_settings()->set_any_az_read_replicas_count(settings->GetReadReplicasCount()); + break; + default: + break; + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TStorageSettingsBuilder::TImpl { +public: + Ydb::Table::StorageSettings Proto; +}; + +TStorageSettingsBuilder::TStorageSettingsBuilder() + : Impl_(new TImpl) +{ } + +TStorageSettingsBuilder::~TStorageSettingsBuilder() { } + +TStorageSettingsBuilder& TStorageSettingsBuilder::SetTabletCommitLog0(const std::string& media) { + Impl_->Proto.mutable_tablet_commit_log0()->set_media(TStringType{media}); + return *this; +} + +TStorageSettingsBuilder& TStorageSettingsBuilder::SetTabletCommitLog1(const std::string& media) { + Impl_->Proto.mutable_tablet_commit_log1()->set_media(TStringType{media}); + return *this; +} + +TStorageSettingsBuilder& TStorageSettingsBuilder::SetExternal(const std::string& media) { + Impl_->Proto.mutable_external()->set_media(TStringType{media}); + return *this; +} + +TStorageSettingsBuilder& TStorageSettingsBuilder::SetStoreExternalBlobs(bool enabled) { + Impl_->Proto.set_store_external_blobs( + enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + return *this; +} + +TStorageSettings TStorageSettingsBuilder::Build() const { + return TStorageSettings(Impl_->Proto); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPartitioningSettingsBuilder::TImpl { +public: + Ydb::Table::PartitioningSettings Proto; +}; + +TPartitioningSettingsBuilder::TPartitioningSettingsBuilder() + : Impl_(new TImpl) +{ } + +TPartitioningSettingsBuilder::~TPartitioningSettingsBuilder() { } + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningBySize(bool enabled) { + Impl_->Proto.set_partitioning_by_size( + enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningByLoad(bool enabled) { + Impl_->Proto.set_partitioning_by_load( + enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitionSizeMb(uint64_t sizeMb) { + Impl_->Proto.set_partition_size_mb(sizeMb); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMinPartitionsCount(uint64_t count) { + Impl_->Proto.set_min_partitions_count(count); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMaxPartitionsCount(uint64_t count) { + Impl_->Proto.set_max_partitions_count(count); + return *this; +} + +TPartitioningSettings TPartitioningSettingsBuilder::Build() const { + return TPartitioningSettings(Impl_->Proto); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TColumnFamilyBuilder::TImpl { +public: + Ydb::Table::ColumnFamily Proto; +}; + +TColumnFamilyBuilder::TColumnFamilyBuilder(const std::string& name) + : Impl_(new TImpl) +{ + Impl_->Proto.set_name(TStringType{name}); +} + +TColumnFamilyBuilder::~TColumnFamilyBuilder() { } + +TColumnFamilyBuilder& TColumnFamilyBuilder::SetData(const std::string& media) { + Impl_->Proto.mutable_data()->set_media(TStringType{media}); + return *this; +} + +TColumnFamilyBuilder& TColumnFamilyBuilder::SetCompression(EColumnFamilyCompression compression) { + switch (compression) { + case EColumnFamilyCompression::None: + Impl_->Proto.set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); + break; + case EColumnFamilyCompression::LZ4: + Impl_->Proto.set_compression(Ydb::Table::ColumnFamily::COMPRESSION_LZ4); + break; + } + return *this; +} + +TColumnFamilyBuilder& TColumnFamilyBuilder::SetKeepInMemory(bool enabled) { + Impl_->Proto.set_keep_in_memory(enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + return *this; +} + +TColumnFamilyDescription TColumnFamilyBuilder::Build() const { + return TColumnFamilyDescription(Impl_->Proto); +} + +//////////////////////////////////////////////////////////////////////////////// + +TTableBuilder& TTableBuilder::SetStoreType(EStoreType type) { + TableDescription_.SetStoreType(type); + return *this; +} + +TTableBuilder& TTableBuilder::AddNullableColumn(const std::string& name, const EPrimitiveType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .BeginOptional() + .Primitive(type) + .EndOptional() + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddNullableColumn(const std::string& name, const TDecimalType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .BeginOptional() + .Decimal(type) + .EndOptional() + .Build(); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddNullableColumn(const std::string& name, const TPgType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .Pg(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddNonNullableColumn(const std::string& name, const EPrimitiveType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .Primitive(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddNonNullableColumn(const std::string& name, const TDecimalType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .Decimal(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddNonNullableColumn(const std::string& name, const TPgType& type, const std::string& family) { + auto columnType = TTypeBuilder() + .Pg(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddSerialColumn(const std::string& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const std::string& family) { + auto columnType = TTypeBuilder() + .Primitive(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::move(sequenceDescription)); + return *this; +} + +TTableBuilder& TTableBuilder::SetPrimaryKeyColumns(const std::vector& primaryKeyColumns) { + TableDescription_.SetPrimaryKeyColumns(primaryKeyColumns); + return *this; +} + +TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const std::string& primaryKeyColumn) { + TableDescription_.SetPrimaryKeyColumns(std::vector{primaryKeyColumn}); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) { + TableDescription_.AddSecondaryIndex(indexDescription); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns) { + TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns) { + TableDescription_.AddSecondaryIndex(indexName, type, indexColumns); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::string& indexColumn) { + TableDescription_.AddSecondaryIndex(indexName, type, std::vector{indexColumn}); + return *this; +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const std::string& indexName, const std::string& indexColumn) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumn); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const std::string& indexName, const std::string& indexColumn) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumn); +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + return AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + return AddSyncSecondaryIndex(indexName, indexColumns); +} + +TTableBuilder& TTableBuilder::AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns); +} + +TTableBuilder& TTableBuilder::AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings) { + TableDescription_.AddVectorKMeansTreeIndex(indexName, indexColumns, dataColumns, indexSettings); + return *this; +} + +TTableBuilder& TTableBuilder::AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { + TableDescription_.AddVectorKMeansTreeIndex(indexName, indexColumns, indexSettings); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, const std::string& indexColumn) { + return AddSyncSecondaryIndex(indexName, indexColumn); +} + +TTableBuilder& TTableBuilder::SetTtlSettings(TTtlSettings&& settings) { + TableDescription_.SetTtlSettings(std::move(settings)); + return *this; +} + +TTableBuilder& TTableBuilder::SetTtlSettings(const TTtlSettings& settings) { + TableDescription_.SetTtlSettings(settings); + return *this; +} + +TTableBuilder& TTableBuilder::SetTtlSettings(const std::string& columnName, const TDuration& expireAfter) { + return SetTtlSettings(TTtlSettings(columnName, expireAfter)); +} + +TTableBuilder& TTableBuilder::SetTtlSettings(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter) { + return SetTtlSettings(TTtlSettings(columnName, columnUnit, expireAfter)); +} + +TTableBuilder& TTableBuilder::SetStorageSettings(const TStorageSettings& settings) { + TableDescription_.SetStorageSettings(settings); + return *this; +} + +TTableBuilder& TTableBuilder::AddColumnFamily(const TColumnFamilyDescription& desc) { + TableDescription_.AddColumnFamily(std::move(desc)); + return *this; +} + +TTableBuilder& TTableBuilder::AddAttribute(const std::string& key, const std::string& value) { + TableDescription_.AddAttribute(key, value); + return *this; +} + +TTableBuilder& TTableBuilder::SetAttributes(const std::unordered_map& attrs) { + TableDescription_.SetAttributes(attrs); + return *this; +} + +TTableBuilder& TTableBuilder::SetAttributes(std::unordered_map&& attrs) { + TableDescription_.SetAttributes(std::move(attrs)); + return *this; +} + +TTableBuilder& TTableBuilder::SetCompactionPolicy(const std::string& name) { + TableDescription_.SetCompactionPolicy(name); + return *this; +} + +TTableBuilder& TTableBuilder::SetUniformPartitions(uint64_t partitionsCount) { + TableDescription_.SetUniformPartitions(partitionsCount); + return *this; +} + +TTableBuilder& TTableBuilder::SetPartitionAtKeys(const TExplicitPartitions& keys) { + TableDescription_.SetPartitionAtKeys(keys); + return *this; +} + +TTableBuilder& TTableBuilder::SetPartitioningSettings(const TPartitioningSettings& settings) { + TableDescription_.SetPartitioningSettings(settings); + return *this; +} + +TTableBuilder& TTableBuilder::SetKeyBloomFilter(bool enabled) { + TableDescription_.SetKeyBloomFilter(enabled); + return *this; +} + +TTableBuilder& TTableBuilder::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { + TableDescription_.SetReadReplicasSettings(mode, readReplicasCount); + return *this; +} + +TTableDescription TTableBuilder::Build() { + return TableDescription_; +} + + +TTablePartIterator::TTablePartIterator( + std::shared_ptr impl, + TPlainStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) +{} + +TAsyncSimpleStreamPart TTablePartIterator::ReadNext() { + if (ReaderImpl_->IsFinished()) + RaiseError("Attempt to perform read on invalid or finished stream"); + return ReaderImpl_->ReadNext(ReaderImpl_); +} + +TScanQueryPartIterator::TScanQueryPartIterator( + std::shared_ptr impl, + TPlainStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) +{} + +TAsyncScanQueryPart TScanQueryPartIterator::ReadNext() { + if (ReaderImpl_->IsFinished()) + RaiseError("Attempt to perform read on invalid or finished stream"); + return ReaderImpl_->ReadNext(ReaderImpl_); +} + + + +static bool IsSessionStatusRetriable(const TCreateSessionResult& res) { + switch (res.GetStatus()) { + case EStatus::OVERLOADED: + // For CreateSession request we can retry some of transport errors + // - endpoind will be pessimized and session will be created on the + // another endpoint + case EStatus::CLIENT_DEADLINE_EXCEEDED: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::TRANSPORT_UNAVAILABLE: + return true; + default: + return false; + } +} + +TSessionInspectorFn TSession::TImpl::GetSessionInspector( + NThreading::TPromise& promise, + std::shared_ptr client, + const TCreateSessionSettings& settings, + ui32 counter, bool needUpdateActiveSessionCounter) +{ + return [promise, client, settings, counter, needUpdateActiveSessionCounter](TAsyncCreateSessionResult future) mutable { + Y_ASSERT(future.HasValue()); + auto session = future.ExtractValue(); + if (IsSessionStatusRetriable(session) && counter < client->GetSessionRetryLimit()) { + counter++; + client->CreateSession(settings, false) + .Subscribe(GetSessionInspector( + promise, + client, + settings, + counter, + needUpdateActiveSessionCounter) + ); + } else { + session.Session_.SessionImpl_->SetNeedUpdateActiveCounter(needUpdateActiveSessionCounter); + promise.SetValue(std::move(session)); + } + }; +} + +TTableClient::TTableClient(const TDriver& driver, const TClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) { + Impl_->StartPeriodicSessionPoolTask(); + Impl_->StartPeriodicHostScanTask(); + Impl_->InitStopper(); +} + +TAsyncCreateSessionResult TTableClient::CreateSession(const TCreateSessionSettings& settings) { + // Returns standalone session + return Impl_->CreateSession(settings, true); +} + +TAsyncCreateSessionResult TTableClient::GetSession(const TCreateSessionSettings& settings) { + // Returns session from session pool + return Impl_->GetSession(settings); +} + +int64_t TTableClient::GetActiveSessionCount() const { + return Impl_->GetActiveSessionCount(); +} + +int64_t TTableClient::GetActiveSessionsLimit() const { + return Impl_->GetActiveSessionsLimit(); +} + +int64_t TTableClient::GetCurrentPoolSize() const { + return Impl_->GetCurrentPoolSize(); +} + +TTableBuilder TTableClient::GetTableBuilder() { + return TTableBuilder(); +} + +TParamsBuilder TTableClient::GetParamsBuilder() const { + return TParamsBuilder(); +} + +TTypeBuilder TTableClient::GetTypeBuilder() { + return TTypeBuilder(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TAsyncStatus TTableClient::RetryOperation(TOperationFunc&& operation, const TRetryOperationSettings& settings) { + TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(operation), settings)); + return ctx->Execute(); +} + +TAsyncStatus TTableClient::RetryOperation(TOperationWithoutSessionFunc&& operation, const TRetryOperationSettings& settings) { + TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithoutSession(*this, std::move(operation), settings)); + return ctx->Execute(); +} + +TStatus TTableClient::RetryOperationSync(const TOperationWithoutSessionSyncFunc& operation, const TRetryOperationSettings& settings) { + NRetry::Sync::TRetryWithoutSession ctx(*this, operation, settings); + return ctx.Execute(); +} + +TStatus TTableClient::RetryOperationSync(const TOperationSyncFunc& operation, const TRetryOperationSettings& settings) { + NRetry::Sync::TRetryWithSession ctx(*this, operation, settings); + return ctx.Execute(); +} + +NThreading::TFuture TTableClient::Stop() { + return Impl_->Stop(); +} + +TAsyncBulkUpsertResult TTableClient::BulkUpsert(const std::string& table, TValue&& rows, + const TBulkUpsertSettings& settings) +{ + return Impl_->BulkUpsert(table, std::move(rows), settings); +} + +TAsyncBulkUpsertResult TTableClient::BulkUpsert(const std::string& table, EDataFormat format, + const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings) +{ + return Impl_->BulkUpsert(table, format, data, schema, settings); +} + +TAsyncReadRowsResult TTableClient::ReadRows(const std::string& table, TValue&& rows, const std::vector& columns, + const TReadRowsSettings& settings) +{ + return Impl_->ReadRows(table, std::move(rows), columns, settings); +} + +TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const std::string& query, const TParams& params, + const TStreamExecScanQuerySettings& settings) +{ + return Impl_->StreamExecuteScanQuery(query, ¶ms.GetProtoMap(), settings); +} + +TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const std::string& query, + const TStreamExecScanQuerySettings& settings) +{ + return Impl_->StreamExecuteScanQuery(query, nullptr, settings); +} + +//////////////////////////////////////////////////////////////////////////////// + +static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settings, Ydb::Table::TableProfile* proto) { + if (settings.PresetName_) { + proto->set_preset_name(TStringType{settings.PresetName_.value()}); + } + if (settings.ExecutionPolicy_) { + proto->mutable_execution_policy()->set_preset_name(TStringType{settings.ExecutionPolicy_.value()}); + } + if (settings.CompactionPolicy_) { + proto->mutable_compaction_policy()->set_preset_name(TStringType{settings.CompactionPolicy_.value()}); + } + if (settings.PartitioningPolicy_) { + const auto& policy = settings.PartitioningPolicy_.value(); + if (policy.PresetName_) { + proto->mutable_partitioning_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); + } + if (policy.AutoPartitioning_) { + proto->mutable_partitioning_policy()->set_auto_partitioning(static_cast(policy.AutoPartitioning_.value())); + } + if (policy.UniformPartitions_) { + proto->mutable_partitioning_policy()->set_uniform_partitions(policy.UniformPartitions_.value()); + } + if (policy.ExplicitPartitions_) { + auto* borders = proto->mutable_partitioning_policy()->mutable_explicit_partitions(); + for (const auto& splitPoint : policy.ExplicitPartitions_->SplitPoints_) { + auto* border = borders->add_split_points(); + border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); + border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); + } + } + } + if (settings.StoragePolicy_) { + const auto& policy = settings.StoragePolicy_.value(); + if (policy.PresetName_) { + proto->mutable_storage_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); + } + if (policy.SysLog_) { + proto->mutable_storage_policy()->mutable_syslog()->set_media(TStringType{policy.SysLog_.value()}); + } + if (policy.Log_) { + proto->mutable_storage_policy()->mutable_log()->set_media(TStringType{policy.Log_.value()}); + } + if (policy.Data_) { + proto->mutable_storage_policy()->mutable_data()->set_media(TStringType{policy.Data_.value()}); + } + if (policy.External_) { + proto->mutable_storage_policy()->mutable_external()->set_media(TStringType{policy.External_.value()}); + } + for (const auto& familyPolicy : policy.ColumnFamilies_) { + auto* familyProto = proto->mutable_storage_policy()->add_column_families(); + if (familyPolicy.Name_) { + familyProto->set_name(TStringType{familyPolicy.Name_.value()}); + } + if (familyPolicy.Data_) { + familyProto->mutable_data()->set_media(TStringType{familyPolicy.Data_.value()}); + } + if (familyPolicy.External_) { + familyProto->mutable_external()->set_media(TStringType{familyPolicy.External_.value()}); + } + if (familyPolicy.KeepInMemory_) { + familyProto->set_keep_in_memory( + familyPolicy.KeepInMemory_.value() + ? Ydb::FeatureFlag_Status::FeatureFlag_Status_ENABLED + : Ydb::FeatureFlag_Status::FeatureFlag_Status_DISABLED + ); + } + if (familyPolicy.Compressed_) { + familyProto->set_compression(familyPolicy.Compressed_.value() + ? Ydb::Table::ColumnFamilyPolicy::COMPRESSED + : Ydb::Table::ColumnFamilyPolicy::UNCOMPRESSED); + } + } + } + if (settings.ReplicationPolicy_) { + const auto& policy = settings.ReplicationPolicy_.value(); + if (policy.PresetName_) { + proto->mutable_replication_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); + } + if (policy.ReplicasCount_) { + proto->mutable_replication_policy()->set_replicas_count(policy.ReplicasCount_.value()); + } + if (policy.CreatePerAvailabilityZone_) { + proto->mutable_replication_policy()->set_create_per_availability_zone( + policy.CreatePerAvailabilityZone_.value() + ? Ydb::FeatureFlag_Status::FeatureFlag_Status_ENABLED + : Ydb::FeatureFlag_Status::FeatureFlag_Status_DISABLED + ); + } + if (policy.AllowPromotion_) { + proto->mutable_replication_policy()->set_allow_promotion( + policy.AllowPromotion_.value() + ? Ydb::FeatureFlag_Status::FeatureFlag_Status_ENABLED + : Ydb::FeatureFlag_Status::FeatureFlag_Status_DISABLED + ); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TSession::TSession(std::shared_ptr client, const std::string& sessionId, const std::string& endpointId, bool isOwnedBySessionPool) + : Client_(client) + , SessionImpl_(new TSession::TImpl( + sessionId, + endpointId, + client->Settings_.UseQueryCache_, + client->Settings_.QueryCacheSize_, + isOwnedBySessionPool), + TSession::TImpl::GetSmartDeleter(client)) +{ + if (!endpointId.empty()) { + Client_->LinkObjToEndpoint(SessionImpl_->GetEndpointKey(), SessionImpl_.get(), Client_.get()); + } +} + +TSession::TSession(std::shared_ptr client, std::shared_ptr sessionid) + : Client_(client) + , SessionImpl_(sessionid) +{} + +TFuture TSession::CreateTable(const std::string& path, TTableDescription&& tableDesc, + const TCreateTableSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{SessionImpl_->GetId()}); + request.set_path(TStringType{path}); + + tableDesc.SerializeTo(request); + + ConvertCreateTableSettingsToProto(settings, request.mutable_profile()); + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->CreateTable(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TFuture TSession::DropTable(const std::string& path, const TDropTableSettings& settings) { + return InjectSessionStatusInterception( + SessionImpl_, + Client_->DropTable(SessionImpl_->GetId(), path, settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( + const std::string& path, const TAlterTableSettings& settings, const std::string& sessionId) +{ + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); + + for (const auto& column : settings.AddColumns_) { + auto& protoColumn = *request.add_add_columns(); + protoColumn.set_name(TStringType{column.Name}); + protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); + protoColumn.set_family(TStringType{column.Family}); + } + + for (const auto& columnName : settings.DropColumns_) { + request.add_drop_columns(TStringType{columnName}); + } + + for (const auto& alter : settings.AlterColumns_) { + auto& protoAlter = *request.add_alter_columns(); + protoAlter.set_name(TStringType{alter.Name}); + protoAlter.set_family(TStringType{alter.Family}); + } + + for (const auto& addIndex : settings.AddIndexes_) { + addIndex.SerializeTo(*request.add_add_indexes()); + } + + for (const auto& name : settings.DropIndexes_) { + request.add_drop_indexes(TStringType{name}); + } + + for (const auto& rename : settings.RenameIndexes_) { + SerializeTo(rename, *request.add_rename_indexes()); + } + + for (const auto& addChangefeed : settings.AddChangefeeds_) { + addChangefeed.SerializeTo(*request.add_add_changefeeds()); + } + + for (const auto& name : settings.DropChangefeeds_) { + request.add_drop_changefeeds(TStringType{name}); + } + + if (settings.AlterStorageSettings_) { + request.mutable_alter_storage_settings()->CopyFrom(settings.AlterStorageSettings_->GetProto()); + } + + for (const auto& family : settings.AddColumnFamilies_) { + request.add_add_column_families()->CopyFrom(family.GetProto()); + } + + for (const auto& family : settings.AlterColumnFamilies_) { + request.add_alter_column_families()->CopyFrom(family.GetProto()); + } + + if (const auto& ttl = settings.GetAlterTtlSettings()) { + switch (ttl->GetAction()) { + case TAlterTtlSettings::EAction::Set: + ttl->GetTtlSettings().SerializeTo(*request.mutable_set_ttl_settings()); + break; + case TAlterTtlSettings::EAction::Drop: + request.mutable_drop_ttl_settings(); + break; + } + } + + for (const auto& [key, value] : settings.AlterAttributes_) { + (*request.mutable_alter_attributes())[key] = value; + } + + if (!settings.SetCompactionPolicy_.empty()) { + request.set_set_compaction_policy(TStringType{settings.SetCompactionPolicy_}); + } + + if (settings.AlterPartitioningSettings_) { + request.mutable_alter_partitioning_settings()->CopyFrom(settings.AlterPartitioningSettings_->GetProto()); + } + + if (settings.SetKeyBloomFilter_.has_value()) { + request.set_set_key_bloom_filter( + settings.SetKeyBloomFilter_.value() ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + } + + if (settings.SetReadReplicasSettings_.has_value()) { + const auto& replSettings = settings.SetReadReplicasSettings_.value(); + switch (replSettings.GetMode()) { + case TReadReplicasSettings::EMode::PerAz: + request.mutable_set_read_replicas_settings()->set_per_az_read_replicas_count( + replSettings.GetReadReplicasCount()); + break; + case TReadReplicasSettings::EMode::AnyAz: + request.mutable_set_read_replicas_settings()->set_any_az_read_replicas_count( + replSettings.GetReadReplicasCount()); + break; + default: + break; + } + } + + return request; +} + +TAsyncStatus TSession::AlterTable(const std::string& path, const TAlterTableSettings& settings) { + auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->AlterTable(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncOperation TSession::AlterTableLong(const std::string& path, const TAlterTableSettings& settings) { + auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->AlterTableLong(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncStatus TSession::RenameTables(const std::vector& renameItems, const TRenameTablesSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{SessionImpl_->GetId()}); + + for (const auto& item: renameItems) { + auto add = request.add_tables(); + add->set_source_path(TStringType{item.SourcePath()}); + add->set_destination_path(TStringType{item.DestinationPath()}); + add->set_replace_destination(item.ReplaceDestination()); + } + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->RenameTables(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncStatus TSession::CopyTables(const std::vector& copyItems, const TCopyTablesSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_session_id(TStringType{SessionImpl_->GetId()}); + + for (const auto& item: copyItems) { + auto add = request.add_tables(); + add->set_source_path(TStringType{item.SourcePath()}); + add->set_destination_path(TStringType{item.DestinationPath()}); + add->set_omit_indexes(item.OmitIndexes()); + } + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->CopyTables(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TFuture TSession::CopyTable(const std::string& src, const std::string& dst, const TCopyTableSettings& settings) { + return InjectSessionStatusInterception( + SessionImpl_, + Client_->CopyTable(SessionImpl_->GetId(), src, dst, settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncDescribeTableResult TSession::DescribeTable(const std::string& path, const TDescribeTableSettings& settings) { + return Client_->DescribeTable(SessionImpl_->GetId(), path, settings); +} + +TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + const TExecDataQuerySettings& settings) +{ + return Client_->ExecuteDataQuery(*this, query, txControl, nullptr, settings); +} + +TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + TParams&& params, const TExecDataQuerySettings& settings) +{ + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + return Client_->ExecuteDataQuery(*this, query, txControl, paramsPtr, settings); +} + +TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const TTxControl& txControl, + const TParams& params, const TExecDataQuerySettings& settings) +{ + if (params.Empty()) { + return Client_->ExecuteDataQuery( + *this, + query, + txControl, + nullptr, + settings); + } else { + using TProtoParamsType = const ::google::protobuf::Map; + return Client_->ExecuteDataQuery( + *this, + query, + txControl, + params.GetProtoMap(), + settings); + } +} + +TAsyncPrepareQueryResult TSession::PrepareDataQuery(const std::string& query, const TPrepareDataQuerySettings& settings) { + auto maybeQuery = SessionImpl_->GetQueryFromCache(query, Client_->Settings_.AllowRequestMigration_); + if (maybeQuery) { + TStatus status(EStatus::SUCCESS, NYdb::NIssue::TIssues()); + TDataQuery dataQuery(*this, query, maybeQuery->QueryId, maybeQuery->ParameterTypes); + TPrepareQueryResult result(std::move(status), dataQuery, true); + return MakeFuture(result); + } + + Client_->CacheMissCounter.Inc(); + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->PrepareDataQuery(*this, query, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncStatus TSession::ExecuteSchemeQuery(const std::string& query, const TExecSchemeQuerySettings& settings) { + return InjectSessionStatusInterception( + SessionImpl_, + Client_->ExecuteSchemeQuery(SessionImpl_->GetId(), query, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncBeginTransactionResult TSession::BeginTransaction(const TTxSettings& txSettings, + const TBeginTxSettings& settings) +{ + return InjectSessionStatusInterception( + SessionImpl_, + Client_->BeginTransaction(*this, txSettings, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncExplainDataQueryResult TSession::ExplainDataQuery(const std::string& query, + const TExplainDataQuerySettings& settings) +{ + return InjectSessionStatusInterception( + SessionImpl_, + Client_->ExplainDataQuery(*this, query, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncTablePartIterator TSession::ReadTable(const std::string& path, + const TReadTableSettings& settings) +{ + auto promise = NThreading::NewPromise(); + auto readTableIteratorBuilder = [promise](NThreading::TFuture> future) mutable { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TTablePartIterator( + pair.second ? std::make_shared( + pair.second, pair.first.Endpoint) : nullptr, std::move(pair.first)) + ); + }; + Client_->ReadTable(SessionImpl_->GetId(), path, settings).Subscribe(readTableIteratorBuilder); + return InjectSessionStatusInterception( + SessionImpl_, + promise.GetFuture(), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +void TSession::InvalidateQueryCache() { + SessionImpl_->InvalidateQueryCache(); +} + +TAsyncStatus TSession::Close(const TCloseSessionSettings& settings) { + return Client_->Close(SessionImpl_.get(), settings); +} + +TAsyncKeepAliveResult TSession::KeepAlive(const TKeepAliveSettings &settings) { + return InjectSessionStatusInterception( + SessionImpl_, + Client_->KeepAlive(SessionImpl_.get(), settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TTableBuilder TSession::GetTableBuilder() { + return TTableBuilder(); +} + +TParamsBuilder TSession::GetParamsBuilder() { + return TParamsBuilder(); +} + +TTypeBuilder TSession::GetTypeBuilder() { + return TTypeBuilder(); +} + +const std::string& TSession::GetId() const { + return SessionImpl_->GetId(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TTxControl::TTxControl(const TTransaction& tx) + : Tx_(tx) +{} + +TTxControl::TTxControl(const TTxSettings& begin) + : BeginTx_(begin) +{} + +//////////////////////////////////////////////////////////////////////////////// + +TTransaction::TTransaction(const TSession& session, const std::string& txId) + : TransactionImpl_(new TTransaction::TImpl(session, txId)) +{} + +const std::string& TTransaction::GetId() const +{ + return TransactionImpl_->GetId(); +} + +bool TTransaction::IsActive() const +{ + return TransactionImpl_->IsActive(); +} + +TAsyncStatus TTransaction::Precommit() const +{ + return TransactionImpl_->Precommit(); +} + +TAsyncCommitTransactionResult TTransaction::Commit(const TCommitTxSettings& settings) { + return TransactionImpl_->Commit(settings); +} + +TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) { + return TransactionImpl_->Rollback(settings); +} + +TSession TTransaction::GetSession() const +{ + return TransactionImpl_->GetSession(); +} + +void TTransaction::AddPrecommitCallback(TPrecommitTransactionCallback cb) +{ + TransactionImpl_->AddPrecommitCallback(std::move(cb)); +} + +//////////////////////////////////////////////////////////////////////////////// + +TDataQuery::TDataQuery(const TSession& session, const std::string& text, const std::string& id) + : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, + session.Client_->Settings_.AllowRequestMigration_)) +{} + +TDataQuery::TDataQuery(const TSession& session, const std::string& text, const std::string& id, + const ::google::protobuf::Map& types) + : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, + session.Client_->Settings_.AllowRequestMigration_, types)) +{} + +const std::string& TDataQuery::GetId() const { + return Impl_->GetId(); +} + +const std::optional& TDataQuery::GetText() const { + return Impl_->GetText(); +} + +TParamsBuilder TDataQuery::GetParamsBuilder() const { + return TParamsBuilder(Impl_->ParameterTypes_); +} + +TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, + const TExecDataQuerySettings& settings) +{ + return Impl_->Session_.Client_->ExecuteDataQuery(Impl_->Session_, *this, txControl, nullptr, settings, false); +} + +TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, TParams&& params, + const TExecDataQuerySettings& settings) +{ + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + return Impl_->Session_.Client_->ExecuteDataQuery( + Impl_->Session_, + *this, + txControl, + paramsPtr, + settings, + false); +} + +TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TParams& params, + const TExecDataQuerySettings& settings) +{ + if (params.Empty()) { + return Impl_->Session_.Client_->ExecuteDataQuery( + Impl_->Session_, + *this, + txControl, + nullptr, + settings, + false); + } else { + using TProtoParamsType = const ::google::protobuf::Map; + return Impl_->Session_.Client_->ExecuteDataQuery( + Impl_->Session_, + *this, + txControl, + params.GetProtoMap(), + settings, + false); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TCreateSessionResult::TCreateSessionResult(TStatus&& status, TSession&& session) + : TStatus(std::move(status)) + , Session_(std::move(session)) +{} + +TSession TCreateSessionResult::GetSession() const { + CheckStatusOk("TCreateSessionResult::GetSession"); + return Session_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TKeepAliveResult::TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus) + : TStatus(std::move(status)) + , SessionStatus(sessionStatus) +{} + +ESessionStatus TKeepAliveResult::GetSessionStatus() const { + return SessionStatus; +} + +//////////////////////////////////////////////////////////////////////////////// + +TPrepareQueryResult::TPrepareQueryResult(TStatus&& status, const TDataQuery& query, bool fromCache) + : TStatus(std::move(status)) + , PreparedQuery_(query) + , FromCache_(fromCache) +{} + +TDataQuery TPrepareQueryResult::GetQuery() const { + CheckStatusOk("TPrepareQueryResult"); + return PreparedQuery_; +} + +bool TPrepareQueryResult::IsQueryFromCache() const { + CheckStatusOk("TPrepareQueryResult"); + return FromCache_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TExplainQueryResult::TExplainQueryResult(TStatus&& status, std::string&& plan, std::string&& ast, std::string&& diagnostics) + : TStatus(std::move(status)) + , Plan_(std::move(plan)) + , Ast_(std::move(ast)) + , Diagnostics_(std::move(diagnostics)) +{} + +const std::string& TExplainQueryResult::GetPlan() const { + CheckStatusOk("TExplainQueryResult::GetPlan"); + return Plan_; +} + +const std::string& TExplainQueryResult::GetAst() const { + CheckStatusOk("TExplainQueryResult::GetAst"); + return Ast_; +} + +const std::string& TExplainQueryResult::GetDiagnostics() const { + CheckStatusOk("TExplainQueryResult::GetDiagnostics"); + return Diagnostics_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TDescribeTableResult::TDescribeTableResult(TStatus&& status, Ydb::Table::DescribeTableResult&& desc, + const TDescribeTableSettings& describeSettings) + : NScheme::TDescribePathResult(std::move(status), desc.self()) + , TableDescription_(std::move(desc), describeSettings) +{} + +TTableDescription TDescribeTableResult::GetTableDescription() const { + CheckStatusOk("TDescribeTableResult::GetTableDescription"); + return TableDescription_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TDataQueryResult::TDataQueryResult(TStatus&& status, std::vector&& resultSets, + const std::optional& transaction, const std::optional& dataQuery, bool fromCache, const std::optional &queryStats) + : TStatus(std::move(status)) + , Transaction_(transaction) + , ResultSets_(std::move(resultSets)) + , DataQuery_(dataQuery) + , FromCache_(fromCache) + , QueryStats_(queryStats) +{} + +const std::vector& TDataQueryResult::GetResultSets() const { + return ResultSets_; +} + +std::vector TDataQueryResult::ExtractResultSets() && { + return std::move(ResultSets_); +} + +TResultSet TDataQueryResult::GetResultSet(size_t resultIndex) const { + if (resultIndex >= ResultSets_.size()) { + RaiseError(std::string("Requested index out of range\n")); + } + + return ResultSets_[resultIndex]; +} + +TResultSetParser TDataQueryResult::GetResultSetParser(size_t resultIndex) const { + return TResultSetParser(GetResultSet(resultIndex)); +} + +std::optional TDataQueryResult::GetTransaction() const { + return Transaction_; +} + +std::optional TDataQueryResult::GetQuery() const { + return DataQuery_; +} + +bool TDataQueryResult::IsQueryFromCache() const { + return FromCache_; +} + +const std::optional& TDataQueryResult::GetStats() const { + return QueryStats_; +} + +const std::string TDataQueryResult::GetQueryPlan() const { + if (QueryStats_.has_value()) { + return NYdb::TProtoAccessor::GetProto(*QueryStats_).query_plan(); + } else { + return ""; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TBeginTransactionResult::TBeginTransactionResult(TStatus&& status, TTransaction transaction) + : TStatus(std::move(status)) + , Transaction_(transaction) +{} + +const TTransaction& TBeginTransactionResult::GetTransaction() const { + CheckStatusOk("TDataQueryResult::GetTransaction"); + return Transaction_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TCommitTransactionResult::TCommitTransactionResult(TStatus&& status, const std::optional& queryStats) + : TStatus(std::move(status)) + , QueryStats_(queryStats) +{} + +const std::optional& TCommitTransactionResult::GetStats() const { + return QueryStats_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TCopyItem::TCopyItem(const std::string& source, const std::string& destination) + : Source_(source) + , Destination_(destination) + , OmitIndexes_(false) { +} + +const std::string& TCopyItem::SourcePath() const { + return Source_; +} + +const std::string& TCopyItem::DestinationPath() const { + return Destination_; +} + +TCopyItem& TCopyItem::SetOmitIndexes() { + OmitIndexes_ = true; + return *this; +} + +bool TCopyItem::OmitIndexes() const { + return OmitIndexes_; +} + +void TCopyItem::Out(IOutputStream& o) const { + o << "{ src: \"" << Source_ << "\"" + << ", dst: \"" << Destination_ << "\"" + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////// + +TRenameItem::TRenameItem(const std::string& source, const std::string& destination) + : Source_(source) + , Destination_(destination) + , ReplaceDestination_(false) { +} + +const std::string& TRenameItem::SourcePath() const { + return Source_; +} + +const std::string& TRenameItem::DestinationPath() const { + return Destination_; +} + +TRenameItem& TRenameItem::SetReplaceDestination() { + ReplaceDestination_ = true; + return *this; +} + +bool TRenameItem::ReplaceDestination() const { + return ReplaceDestination_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TIndexDescription::TIndexDescription( + const std::string& name, + EIndexType type, + const std::vector& indexColumns, + const std::vector& dataColumns, + const std::vector& globalIndexSettings, + const std::variant& specializedIndexSettings +) : IndexName_(name) + , IndexType_(type) + , IndexColumns_(indexColumns) + , DataColumns_(dataColumns) + , GlobalIndexSettings_(globalIndexSettings) + , SpecializedIndexSettings_(specializedIndexSettings) +{} + +TIndexDescription::TIndexDescription( + const std::string& name, + const std::vector& indexColumns, + const std::vector& dataColumns, + const std::vector& globalIndexSettings +) : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns, globalIndexSettings) +{} + +TIndexDescription::TIndexDescription(const Ydb::Table::TableIndex& tableIndex) + : TIndexDescription(FromProto(tableIndex)) +{} + +TIndexDescription::TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc) + : TIndexDescription(FromProto(tableIndexDesc)) +{} + +const std::string& TIndexDescription::GetIndexName() const { + return IndexName_; +} + +EIndexType TIndexDescription::GetIndexType() const { + return IndexType_; +} + +const std::vector& TIndexDescription::GetIndexColumns() const { + return IndexColumns_; +} + +const std::vector& TIndexDescription::GetDataColumns() const { + return DataColumns_; +} + +const std::variant& TIndexDescription::GetVectorIndexSettings() const { + return SpecializedIndexSettings_; +} + +uint64_t TIndexDescription::GetSizeBytes() const { + return SizeBytes_; +} + +TGlobalIndexSettings TGlobalIndexSettings::FromProto(const Ydb::Table::GlobalIndexSettings& proto) { + auto partitionsFromProto = [](const Ydb::Table::GlobalIndexSettings& proto) -> TUniformOrExplicitPartitions { + switch (proto.partitions_case()) { + case Ydb::Table::GlobalIndexSettings::kUniformPartitions: + return proto.uniform_partitions(); + case Ydb::Table::GlobalIndexSettings::kPartitionAtKeys: + return TExplicitPartitions::FromProto(proto.partition_at_keys()); + default: + return {}; + } + }; + + return { + .PartitioningSettings = TPartitioningSettings(proto.partitioning_settings()), + .Partitions = partitionsFromProto(proto) + }; +} + +void TGlobalIndexSettings::SerializeTo(Ydb::Table::GlobalIndexSettings& settings) const { + *settings.mutable_partitioning_settings() = PartitioningSettings.GetProto(); + + auto variantVisitor = [&settings](auto&& partitions) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + settings.set_uniform_partitions(partitions); + } else if constexpr (std::is_same_v) { + partitions.SerializeTo(*settings.mutable_partition_at_keys()); + } + }; + std::visit(std::move(variantVisitor), Partitions); +} + +TVectorIndexSettings TVectorIndexSettings::FromProto(const Ydb::Table::VectorIndexSettings& proto) { + auto covertMetric = [&] { + switch (proto.metric()) { + case Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT: + return EMetric::InnerProduct; + case Ydb::Table::VectorIndexSettings::SIMILARITY_COSINE: + return EMetric::CosineSimilarity; + case Ydb::Table::VectorIndexSettings::DISTANCE_COSINE: + return EMetric::CosineDistance; + case Ydb::Table::VectorIndexSettings::DISTANCE_MANHATTAN: + return EMetric::Manhattan; + case Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN: + return EMetric::Euclidean; + default: + return EMetric::Unspecified; + } + }; + + auto convertVectorType = [&] { + switch (proto.vector_type()) { + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT: + return EVectorType::Float; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8: + return EVectorType::Uint8; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8: + return EVectorType::Int8; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BIT: + return EVectorType::Bit; + default: + return EVectorType::Unspecified; + } + }; + + return { + .Metric = covertMetric(), + .VectorType = convertVectorType(), + .VectorDimension = proto.vector_dimension(), + }; +} + +void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings) const { + auto convertMetric = [&] { + switch (Metric) { + case EMetric::InnerProduct: + return Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT; + case EMetric::CosineSimilarity: + return Ydb::Table::VectorIndexSettings::SIMILARITY_COSINE; + case EMetric::CosineDistance: + return Ydb::Table::VectorIndexSettings::DISTANCE_COSINE; + case EMetric::Manhattan: + return Ydb::Table::VectorIndexSettings::DISTANCE_MANHATTAN; + case EMetric::Euclidean: + return Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN; + case EMetric::Unspecified: + return Ydb::Table::VectorIndexSettings::METRIC_UNSPECIFIED; + } + }; + + auto convertVectorType = [&] { + switch (VectorType) { + case EVectorType::Float: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT; + case EVectorType::Uint8: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8; + case EVectorType::Int8: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8; + case EVectorType::Bit: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BIT; + case EVectorType::Unspecified: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED; + } + }; + + settings.set_metric(convertMetric()); + settings.set_vector_type(convertVectorType()); + settings.set_vector_dimension(VectorDimension); +} + +void TVectorIndexSettings::Out(IOutputStream& o) const { + o << *this; +} + +TKMeansTreeSettings TKMeansTreeSettings::FromProto(const Ydb::Table::KMeansTreeSettings& proto) { + return { + .Settings = TVectorIndexSettings::FromProto(proto.settings()), + .Clusters = proto.clusters(), + .Levels = proto.levels(), + }; +} + +void TKMeansTreeSettings::SerializeTo(Ydb::Table::KMeansTreeSettings& settings) const { + Settings.SerializeTo(*settings.mutable_settings()); + settings.set_clusters(Clusters); + settings.set_levels(Levels); +} + +void TKMeansTreeSettings::Out(IOutputStream& o) const { + o << *this; +} + +template +TIndexDescription TIndexDescription::FromProto(const TProto& proto) { + EIndexType type; + std::vector indexColumns; + std::vector dataColumns; + std::vector globalIndexSettings; + std::variant specializedIndexSettings; + + indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end()); + dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end()); + + switch (proto.type_case()) { + case TProto::kGlobalIndex: + type = EIndexType::GlobalSync; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_index().settings())); + break; + case TProto::kGlobalAsyncIndex: + type = EIndexType::GlobalAsync; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_async_index().settings())); + break; + case TProto::kGlobalUniqueIndex: + type = EIndexType::GlobalUnique; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_unique_index().settings())); + break; + case TProto::kGlobalVectorKmeansTreeIndex: { + type = EIndexType::GlobalVectorKMeansTree; + const auto &vectorProto = proto.global_vector_kmeans_tree_index(); + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.level_table_settings())); + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.posting_table_settings())); + specializedIndexSettings = TKMeansTreeSettings::FromProto(vectorProto.vector_settings()); + break; + } + default: // fallback to global sync + type = EIndexType::GlobalSync; + globalIndexSettings.resize(1); + break; + } + + auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns, globalIndexSettings, specializedIndexSettings); + if constexpr (std::is_same_v) { + result.SizeBytes_ = proto.size_bytes(); + } + + return result; +} + +void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { + proto.set_name(TStringType{IndexName_}); + for (const auto& indexCol : IndexColumns_) { + proto.add_index_columns(TStringType{indexCol}); + } + + *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; + + switch (IndexType_) { + case EIndexType::GlobalSync: { + auto& settings = *proto.mutable_global_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); + break; + } + case EIndexType::GlobalAsync: { + auto& settings = *proto.mutable_global_async_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); + break; + } + case EIndexType::GlobalUnique: { + auto& settings = *proto.mutable_global_unique_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); + break; + } + case EIndexType::GlobalVectorKMeansTree: { + auto* global_vector_kmeans_tree_index = proto.mutable_global_vector_kmeans_tree_index(); + auto& level_settings = *global_vector_kmeans_tree_index->mutable_level_table_settings(); + auto& posting_settings = *global_vector_kmeans_tree_index->mutable_posting_table_settings(); + auto& vector_settings = *global_vector_kmeans_tree_index->mutable_vector_settings(); + if (GlobalIndexSettings_.size() == 2) { + GlobalIndexSettings_[0].SerializeTo(level_settings); + GlobalIndexSettings_[1].SerializeTo(posting_settings); + } + if (const auto* settings = std::get_if(&SpecializedIndexSettings_)) { + settings->SerializeTo(vector_settings); + } + break; + } + case EIndexType::Unknown: + break; + } +} + +std::string TIndexDescription::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TIndexDescription::Out(IOutputStream& o) const { + o << "{ name: \"" << IndexName_ << "\""; + o << ", type: " << IndexType_ << ""; + o << ", index_columns: [" << JoinSeq(", ", IndexColumns_) << "]"; + + if (!DataColumns_.empty()) { + o << ", data_columns: [" << JoinSeq(", ", DataColumns_) << "]"; + } + + std::visit([&](const T& settings) { + if constexpr (!std::is_same_v) { + o << ", vector_settings: " << settings; + } + }, SpecializedIndexSettings_); + + o << " }"; +} + +bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs) { + return lhs.GetIndexName() == rhs.GetIndexName() + && lhs.GetIndexType() == rhs.GetIndexType() + && lhs.GetIndexColumns() == rhs.GetIndexColumns() + && lhs.GetDataColumns() == rhs.GetDataColumns(); +} + +bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs) { + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////////////// + +TChangefeedDescription::TChangefeedDescription(const std::string& name, EChangefeedMode mode, EChangefeedFormat format) + : Name_(name) + , Mode_(mode) + , Format_(format) +{} + +TChangefeedDescription::TChangefeedDescription(const Ydb::Table::Changefeed& proto) + : TChangefeedDescription(FromProto(proto)) +{} + +TChangefeedDescription::TChangefeedDescription(const Ydb::Table::ChangefeedDescription& proto) + : TChangefeedDescription(FromProto(proto)) +{} + +TChangefeedDescription::TInitialScanProgress::TInitialScanProgress() + : PartsTotal(0) + , PartsCompleted(0) +{} + +TChangefeedDescription::TInitialScanProgress::TInitialScanProgress(uint32_t total, uint32_t completed) + : PartsTotal(total) + , PartsCompleted(completed) +{} + +TChangefeedDescription::TInitialScanProgress& TChangefeedDescription::TInitialScanProgress::operator+=(const TInitialScanProgress& other) { + PartsTotal += other.PartsTotal; + PartsCompleted += other.PartsCompleted; + return *this; +} + +uint32_t TChangefeedDescription::TInitialScanProgress::GetPartsTotal() const { + return PartsTotal; +} + +uint32_t TChangefeedDescription::TInitialScanProgress::GetPartsCompleted() const { + return PartsCompleted; +} + +float TChangefeedDescription::TInitialScanProgress::GetProgress() const { + if (PartsTotal == 0) { + return 0; + } + + return 100 * float(PartsCompleted) / float(PartsTotal); +} + +TChangefeedDescription& TChangefeedDescription::WithVirtualTimestamps() { + VirtualTimestamps_ = true; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::WithResolvedTimestamps(const TDuration& value) { + ResolvedTimestamps_ = value; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::WithRetentionPeriod(const TDuration& value) { + RetentionPeriod_ = value; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::WithInitialScan() { + InitialScan_ = true; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::AddAttribute(const std::string& key, const std::string& value) { + Attributes_[key] = value; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::SetAttributes(const std::unordered_map& attrs) { + Attributes_ = attrs; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::SetAttributes(std::unordered_map&& attrs) { + Attributes_ = std::move(attrs); + return *this; +} + +TChangefeedDescription& TChangefeedDescription::WithAwsRegion(const std::string& value) { + AwsRegion_ = value; + return *this; +} + +const std::string& TChangefeedDescription::GetName() const { + return Name_; +} + +EChangefeedMode TChangefeedDescription::GetMode() const { + return Mode_; +} + +EChangefeedFormat TChangefeedDescription::GetFormat() const { + return Format_; +} + +EChangefeedState TChangefeedDescription::GetState() const { + return State_; +} + +bool TChangefeedDescription::GetVirtualTimestamps() const { + return VirtualTimestamps_; +} + +const std::optional& TChangefeedDescription::GetResolvedTimestamps() const { + return ResolvedTimestamps_; +} + +bool TChangefeedDescription::GetInitialScan() const { + return InitialScan_; +} + +const std::unordered_map& TChangefeedDescription::GetAttributes() const { + return Attributes_; +} + +const std::string& TChangefeedDescription::GetAwsRegion() const { + return AwsRegion_; +} + +const std::optional& TChangefeedDescription::GetInitialScanProgress() const { + return InitialScanProgress_; +} + +template +TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { + EChangefeedMode mode; + switch (proto.mode()) { + case Ydb::Table::ChangefeedMode::MODE_KEYS_ONLY: + mode = EChangefeedMode::KeysOnly; + break; + case Ydb::Table::ChangefeedMode::MODE_UPDATES: + mode = EChangefeedMode::Updates; + break; + case Ydb::Table::ChangefeedMode::MODE_NEW_IMAGE: + mode = EChangefeedMode::NewImage; + break; + case Ydb::Table::ChangefeedMode::MODE_OLD_IMAGE: + mode = EChangefeedMode::OldImage; + break; + case Ydb::Table::ChangefeedMode::MODE_NEW_AND_OLD_IMAGES: + mode = EChangefeedMode::NewAndOldImages; + break; + default: + mode = EChangefeedMode::Unknown; + break; + } + + EChangefeedFormat format; + switch (proto.format()) { + case Ydb::Table::ChangefeedFormat::FORMAT_JSON: + format = EChangefeedFormat::Json; + break; + case Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON: + format = EChangefeedFormat::DynamoDBStreamsJson; + break; + case Ydb::Table::ChangefeedFormat::FORMAT_DEBEZIUM_JSON: + format = EChangefeedFormat::DebeziumJson; + break; + default: + format = EChangefeedFormat::Unknown; + break; + } + + auto ret = TChangefeedDescription(proto.name(), mode, format); + if (proto.virtual_timestamps()) { + ret.WithVirtualTimestamps(); + } + if (proto.has_resolved_timestamps_interval()) { + ret.WithResolvedTimestamps(TDuration::MilliSeconds( + ::google::protobuf::util::TimeUtil::DurationToMilliseconds(proto.resolved_timestamps_interval()))); + } + if (!proto.aws_region().empty()) { + ret.WithAwsRegion(proto.aws_region()); + } + + if constexpr (std::is_same_v) { + switch (proto.state()) { + case Ydb::Table::ChangefeedDescription::STATE_ENABLED: + ret.State_= EChangefeedState::Enabled; + break; + case Ydb::Table::ChangefeedDescription::STATE_DISABLED: + ret.State_ = EChangefeedState::Disabled; + break; + case Ydb::Table::ChangefeedDescription::STATE_INITIAL_SCAN: + ret.State_ = EChangefeedState::InitialScan; + break; + default: + ret.State_ = EChangefeedState::Unknown; + break; + } + + if (proto.has_initial_scan_progress()) { + ret.InitialScanProgress_ = std::make_optional( + proto.initial_scan_progress().parts_total(), + proto.initial_scan_progress().parts_completed() + ); + } + } + + for (const auto& [key, value] : proto.attributes()) { + ret.Attributes_[key] = value; + } + + return ret; +} + +template +void TChangefeedDescription::SerializeCommonFields(TProto& proto) const { + proto.set_name(TStringType{Name_}); + proto.set_virtual_timestamps(VirtualTimestamps_); + proto.set_aws_region(TStringType{AwsRegion_}); + + switch (Mode_) { + case EChangefeedMode::KeysOnly: + proto.set_mode(Ydb::Table::ChangefeedMode::MODE_KEYS_ONLY); + break; + case EChangefeedMode::Updates: + proto.set_mode(Ydb::Table::ChangefeedMode::MODE_UPDATES); + break; + case EChangefeedMode::NewImage: + proto.set_mode(Ydb::Table::ChangefeedMode::MODE_NEW_IMAGE); + break; + case EChangefeedMode::OldImage: + proto.set_mode(Ydb::Table::ChangefeedMode::MODE_OLD_IMAGE); + break; + case EChangefeedMode::NewAndOldImages: + proto.set_mode(Ydb::Table::ChangefeedMode::MODE_NEW_AND_OLD_IMAGES); + break; + case EChangefeedMode::Unknown: + break; + } + + switch (Format_) { + case EChangefeedFormat::Json: + proto.set_format(Ydb::Table::ChangefeedFormat::FORMAT_JSON); + break; + case EChangefeedFormat::DynamoDBStreamsJson: + proto.set_format(Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON); + break; + case EChangefeedFormat::DebeziumJson: + proto.set_format(Ydb::Table::ChangefeedFormat::FORMAT_DEBEZIUM_JSON); + break; + case EChangefeedFormat::Unknown: + break; + } + + if (ResolvedTimestamps_) { + SetDuration(*ResolvedTimestamps_, *proto.mutable_resolved_timestamps_interval()); + } + + for (const auto& [key, value] : Attributes_) { + (*proto.mutable_attributes())[key] = value; + } +} + +void TChangefeedDescription::SerializeTo(Ydb::Table::Changefeed& proto) const { + SerializeCommonFields(proto); + proto.set_initial_scan(InitialScan_); + + if (RetentionPeriod_) { + SetDuration(*RetentionPeriod_, *proto.mutable_retention_period()); + } +} + +void TChangefeedDescription::SerializeTo(Ydb::Table::ChangefeedDescription& proto) const { + SerializeCommonFields(proto); + + switch (State_) { + case EChangefeedState::Enabled: + proto.set_state(Ydb::Table::ChangefeedDescription_State::ChangefeedDescription_State_STATE_ENABLED); + break; + case EChangefeedState::Disabled: + proto.set_state(Ydb::Table::ChangefeedDescription_State::ChangefeedDescription_State_STATE_DISABLED); + break; + case EChangefeedState::InitialScan: + proto.set_state(Ydb::Table::ChangefeedDescription_State::ChangefeedDescription_State_STATE_INITIAL_SCAN); + break; + case EChangefeedState::Unknown: + break; + } +} + +std::string TChangefeedDescription::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TChangefeedDescription::Out(IOutputStream& o) const { + o << "{ name: \"" << Name_ << "\"" + << ", mode: " << Mode_ << "" + << ", format: " << Format_ << "" + << ", virtual_timestamps: " << (VirtualTimestamps_ ? "on": "off") << ""; + + if (ResolvedTimestamps_) { + o << ", resolved_timestamps: " << *ResolvedTimestamps_; + } + + if (RetentionPeriod_) { + o << ", retention_period: " << *RetentionPeriod_; + } + + if (!AwsRegion_.empty()) { + o << ", aws_region: " << AwsRegion_; + } + + if (InitialScanProgress_) { + o << ", initial_scan_progress: " << InitialScanProgress_->GetProgress() << "%"; + } + + o << " }"; +} + +bool operator==(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs) { + return lhs.GetName() == rhs.GetName() + && lhs.GetMode() == rhs.GetMode() + && lhs.GetFormat() == rhs.GetFormat() + && lhs.GetVirtualTimestamps() == rhs.GetVirtualTimestamps() + && lhs.GetResolvedTimestamps() == rhs.GetResolvedTimestamps() + && lhs.GetAwsRegion() == rhs.GetAwsRegion(); +} + +bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs) { + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////////////// + +TTtlTierSettings::TTtlTierSettings(const TExpression& expression, const TAction& action) + : Expression_(expression) + , Action_(action) +{ } + +std::optional TTtlTierSettings::FromProto(const Ydb::Table::TtlTier& tier) { + std::optional expression; + switch (tier.expression_case()) { + case Ydb::Table::TtlTier::kDateTypeColumn: + expression = TDateTypeColumnModeSettings( + tier.date_type_column().column_name(), TDuration::Seconds(tier.date_type_column().expire_after_seconds())); + break; + case Ydb::Table::TtlTier::kValueSinceUnixEpoch: + expression = TValueSinceUnixEpochModeSettings(tier.value_since_unix_epoch().column_name(), + TProtoAccessor::FromProto(tier.value_since_unix_epoch().column_unit()), + TDuration::Seconds(tier.value_since_unix_epoch().expire_after_seconds())); + break; + case Ydb::Table::TtlTier::EXPRESSION_NOT_SET: + return std::nullopt; + } + + TAction action; + + switch (tier.action_case()) { + case Ydb::Table::TtlTier::kDelete: + action = TTtlDeleteAction(); + break; + case Ydb::Table::TtlTier::kEvictToExternalStorage: + action = TTtlEvictToExternalStorageAction(tier.evict_to_external_storage().storage()); + break; + case Ydb::Table::TtlTier::ACTION_NOT_SET: + return std::nullopt; + } + + return TTtlTierSettings(std::move(*expression), std::move(action)); +} + +void TTtlTierSettings::SerializeTo(Ydb::Table::TtlTier& proto) const { + std::visit(TOverloaded{ + [&proto](const TDateTypeColumnModeSettings& expr) { expr.SerializeTo(*proto.mutable_date_type_column()); }, + [&proto](const TValueSinceUnixEpochModeSettings& expr) { expr.SerializeTo(*proto.mutable_value_since_unix_epoch()); }, + }, + Expression_); + + std::visit(TOverloaded{ + [&proto](const TTtlDeleteAction&) { proto.mutable_delete_(); }, + [&proto](const TTtlEvictToExternalStorageAction& action) { action.SerializeTo(*proto.mutable_evict_to_external_storage()); }, + }, + Action_); +} + +const TTtlTierSettings::TExpression& TTtlTierSettings::GetExpression() const { + return Expression_; +} + +const TTtlTierSettings::TAction& TTtlTierSettings::GetAction() const { + return Action_; +} + +TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const std::string& columnName, const TDuration& applyAfter) + : ColumnName_(columnName) + , ApplyAfter_(applyAfter) +{} + +void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const { + proto.set_column_name(TStringType{ColumnName_}); + proto.set_expire_after_seconds(ApplyAfter_.Seconds()); +} + +const std::string& TDateTypeColumnModeSettings::GetColumnName() const { + return ColumnName_; +} + +const TDuration& TDateTypeColumnModeSettings::GetExpireAfter() const { + return ApplyAfter_; +} + +TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const std::string& columnName, EUnit columnUnit, const TDuration& applyAfter) + : ColumnName_(columnName) + , ColumnUnit_(columnUnit) + , ApplyAfter_(applyAfter) +{} + +void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const { + proto.set_column_name(TStringType{ColumnName_}); + proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); + proto.set_expire_after_seconds(ApplyAfter_.Seconds()); +} + +const std::string& TValueSinceUnixEpochModeSettings::GetColumnName() const { + return ColumnName_; +} + +TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::GetColumnUnit() const { + return ColumnUnit_; +} + +const TDuration& TValueSinceUnixEpochModeSettings::GetExpireAfter() const { + return ApplyAfter_; +} + +void TValueSinceUnixEpochModeSettings::Out(IOutputStream& out, EUnit unit) { +#define PRINT_UNIT(x) \ + case EUnit::x: \ + out << #x; \ + break + + switch (unit) { + PRINT_UNIT(Seconds); + PRINT_UNIT(MilliSeconds); + PRINT_UNIT(MicroSeconds); + PRINT_UNIT(NanoSeconds); + PRINT_UNIT(Unknown); + } + +#undef PRINT_UNIT +} + +std::string TValueSinceUnixEpochModeSettings::ToString(EUnit unit) { + TString result; + TStringOutput out(result); + Out(out, unit); + return result; +} + +TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::UnitFromString(const std::string& value) { + const auto norm = NUtils::ToLower(value); + + if (norm == "s" || norm == "sec" || norm == "seconds") { + return EUnit::Seconds; + } else if (norm == "ms" || norm == "msec" || norm == "milliseconds") { + return EUnit::MilliSeconds; + } else if (norm == "us" || norm == "usec" || norm == "microseconds") { + return EUnit::MicroSeconds; + } else if (norm == "ns" || norm == "nsec" || norm == "nanoseconds") { + return EUnit::NanoSeconds; + } + + return EUnit::Unknown; +} + +TTtlEvictToExternalStorageAction::TTtlEvictToExternalStorageAction(const std::string& storageName) + : Storage_(storageName) +{} + +void TTtlEvictToExternalStorageAction::SerializeTo(Ydb::Table::EvictionToExternalStorageSettings& proto) const { + proto.set_storage(Storage_); +} + +std::string TTtlEvictToExternalStorageAction::GetStorage() const { + return Storage_; +} + +TTtlSettings::TTtlSettings(const std::vector& tiers) + : Tiers_(tiers) +{} + +TTtlSettings::TTtlSettings(const std::string& columnName, const TDuration& expireAfter) + : TTtlSettings({TTtlTierSettings(TDateTypeColumnModeSettings(columnName, expireAfter), TTtlDeleteAction())}) +{} + +TTtlSettings::TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds) + : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds())) +{ + RunInterval_ = TDuration::Seconds(runIntervalSeconds); +} + +const TDateTypeColumnModeSettings& TTtlSettings::GetDateTypeColumn() const { + return std::get(Tiers_.front().GetExpression()); +} + +TTtlSettings::TTtlSettings(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter) + : TTtlSettings({TTtlTierSettings(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter), TTtlDeleteAction())}) +{} + +TTtlSettings::TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds) + : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds())) +{ + RunInterval_ = TDuration::Seconds(runIntervalSeconds); +} + +const TValueSinceUnixEpochModeSettings& TTtlSettings::GetValueSinceUnixEpoch() const { + return std::get(Tiers_.front().GetExpression()); +} + +std::optional TTtlSettings::FromProto(const Ydb::Table::TtlSettings& proto) { + switch(proto.mode_case()) { + case Ydb::Table::TtlSettings::kDateTypeColumn: + return TTtlSettings(proto.date_type_column(), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: + return TTtlSettings(proto.value_since_unix_epoch(), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::kTieredTtl: { + std::vector tiers; + for (const auto& tier : proto.tiered_ttl().tiers()) { + if (auto deserialized = TTtlTierSettings::FromProto(tier)) { + tiers.emplace_back(std::move(*deserialized)); + } else { + return std::nullopt; + } + } + auto settings = TTtlSettings(std::move(tiers)); + settings.SetRunInterval(TDuration::Seconds(proto.run_interval_seconds())); + return settings; + } + case Ydb::Table::TtlSettings::MODE_NOT_SET: + return std::nullopt; + } +} + +void TTtlSettings::SerializeTo(Ydb::Table::TtlSettings& proto) const { + if (Tiers_.size() == 1 && std::holds_alternative(Tiers_.back().GetAction())) { + // serialize DELETE-only TTL to legacy format for backwards-compatibility + std::visit(TOverloaded{ + [&proto](const TDateTypeColumnModeSettings& expr) { expr.SerializeTo(*proto.mutable_date_type_column()); }, + [&proto](const TValueSinceUnixEpochModeSettings& expr) { expr.SerializeTo(*proto.mutable_value_since_unix_epoch()); }, + }, + Tiers_.front().GetExpression()); + } else { + for (const auto& tier : Tiers_) { + tier.SerializeTo(*proto.mutable_tiered_ttl()->add_tiers()); + } + } + + if (RunInterval_) { + proto.set_run_interval_seconds(RunInterval_.Seconds()); + } +} + +TTtlSettings::EMode TTtlSettings::GetMode() const { + return static_cast(Tiers_.front().GetExpression().index()); +} + +TTtlSettings& TTtlSettings::SetRunInterval(const TDuration& value) { + RunInterval_ = value; + return *this; +} + +const TDuration& TTtlSettings::GetRunInterval() const { + return RunInterval_; +} + +const std::vector& TTtlSettings::GetTiers() const { + return Tiers_; +} + +TAlterTtlSettings::EAction TAlterTtlSettings::GetAction() const { + return static_cast(Action_.index()); +} + +const TTtlSettings& TAlterTtlSettings::GetTtlSettings() const { + return std::get(Action_); +} + +class TAlterTtlSettingsBuilder::TImpl { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TImpl() { } + + void Drop() { + AlterTtlSettings_ = TAlterTtlSettings::Drop(); + } + + void Set(TTtlSettings&& settings) { + AlterTtlSettings_ = TAlterTtlSettings::Set(std::move(settings)); + } + + void Set(const TTtlSettings& settings) { + AlterTtlSettings_ = TAlterTtlSettings::Set(settings); + } + + const std::optional& GetAlterTtlSettings() const { + return AlterTtlSettings_; + } + +private: + std::optional AlterTtlSettings_; +}; + +TAlterTtlSettingsBuilder::TAlterTtlSettingsBuilder(TAlterTableSettings& parent) + : Parent_(parent) + , Impl_(std::make_shared()) +{ } + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Drop() { + Impl_->Drop(); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(TTtlSettings&& settings) { + Impl_->Set(std::move(settings)); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TTtlSettings& settings) { + Impl_->Set(settings); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const std::string& columnName, const TDuration& expireAfter) { + return Set(TTtlSettings(columnName, expireAfter)); +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const std::string& columnName, EUnit columnUnit, const TDuration& expireAfter) { + return Set(TTtlSettings(columnName, columnUnit, expireAfter)); +} + +TAlterTableSettings& TAlterTtlSettingsBuilder::EndAlterTtlSettings() { + return Parent_.AlterTtlSettings(Impl_->GetAlterTtlSettings()); +} + +class TAlterTableSettings::TImpl { +public: + TImpl() { } + + void SetAlterTtlSettings(const std::optional& value) { + AlterTtlSettings_ = value; + } + + const std::optional& GetAlterTtlSettings() const { + return AlterTtlSettings_; + } + +private: + std::optional AlterTtlSettings_; +}; + +TAlterTableSettings::TAlterTableSettings() + : Impl_(std::make_shared()) +{ } + +TAlterTableSettings& TAlterTableSettings::AlterTtlSettings(const std::optional& value) { + Impl_->SetAlterTtlSettings(value); + return *this; +} + +const std::optional& TAlterTableSettings::GetAlterTtlSettings() const { + return Impl_->GetAlterTtlSettings(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TReadReplicasSettings::TReadReplicasSettings(EMode mode, uint64_t readReplicasCount) + : Mode_(mode) + , ReadReplicasCount_(readReplicasCount) +{} + +TReadReplicasSettings::EMode TReadReplicasSettings::GetMode() const { + return Mode_; +} + +uint64_t TReadReplicasSettings::GetReadReplicasCount() const { + return ReadReplicasCount_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TBulkUpsertResult::TBulkUpsertResult(TStatus&& status) + : TStatus(std::move(status)) +{} + +TReadRowsResult::TReadRowsResult(TStatus&& status, TResultSet&& resultSet) + : TStatus(std::move(status)) + , ResultSet(std::move(resultSet)) +{} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/table/ya.make b/ydb/public/sdk/cpp/src/client/table/ya.make new file mode 100644 index 000000000000..d9726f9428f6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/table/ya.make @@ -0,0 +1,29 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + out.cpp + proto_accessor.cpp + table.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table_enum.h) + +PEERDIR( + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/impl/ydb_internal/kqp_session_common + ydb/public/sdk/cpp/src/client/impl/ydb_internal/retry + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/params + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table/impl + ydb/public/sdk/cpp/src/client/table/query_stats + ydb/public/sdk/cpp/src/client/types/operation + ydb/public/sdk/cpp/src/client/value +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/topic/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/topic/CMakeLists.txt new file mode 100644 index 000000000000..e18f83ff29f4 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/CMakeLists.txt @@ -0,0 +1,45 @@ +add_subdirectory(common) +add_subdirectory(impl) +add_subdirectory(codecs) + +_ydb_sdk_add_library(client-ydb_topic) + +target_link_libraries(client-ydb_topic PUBLIC + yutil + enum_serialization_runtime + client-ydb_topic-codecs + retry + client-ydb_topic-common + client-ydb_topic-impl + client-ydb_proto + client-ydb_driver + api-grpc + api-grpc-draft + api-protos + client-ydb_table +) + +target_sources(client-ydb_topic PRIVATE + proto_accessor.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_topic + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/topic/control_plane.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/topic/control_plane.h +) + +generate_enum_serilization(client-ydb_topic + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/topic/read_events.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/topic/read_events.h +) + +generate_enum_serilization(client-ydb_topic + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/topic/write_events.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/topic/write_events.h +) + +_ydb_sdk_make_client_component(Topic client-ydb_topic) diff --git a/ydb/public/sdk/cpp/src/client/topic/codecs/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/topic/codecs/CMakeLists.txt new file mode 100644 index 000000000000..e8069c8debd9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/codecs/CMakeLists.txt @@ -0,0 +1,21 @@ +_ydb_sdk_add_library(client-ydb_topic-codecs) + +target_link_libraries(client-ydb_topic-codecs PUBLIC + yutil + streams-zstd + api-grpc-draft + api-grpc + api-protos +) + +target_sources(client-ydb_topic-codecs PRIVATE + codecs.cpp +) + +generate_enum_serilization(client-ydb_topic-codecs + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/topic/codecs.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/topic/codecs.h +) + +_ydb_sdk_install_targets(TARGETS client-ydb_topic-codecs) diff --git a/ydb/public/sdk/cpp/src/client/topic/codecs/codecs.cpp b/ydb/public/sdk/cpp/src/client/topic/codecs/codecs.cpp new file mode 100644 index 000000000000..14c5eddead6e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/codecs/codecs.cpp @@ -0,0 +1,66 @@ +#include + +#include + +#include +#include + +namespace NYdb::inline V3::NTopic { + +namespace { + +class TZLibToStringCompressor: private TEmbedPolicy, public TZLibCompress { +public: + TZLibToStringCompressor(TBuffer& dst, ZLib::StreamType type, size_t quality) + : TEmbedPolicy(dst) + , TZLibCompress(TEmbedPolicy::Ptr(), type, quality) + { + } +}; + +class TZstdToStringCompressor: private TEmbedPolicy, public TZstdCompress { +public: + TZstdToStringCompressor(TBuffer& dst, int quality) + : TEmbedPolicy(dst) + , TZstdCompress(TEmbedPolicy::Ptr(), quality) + { + } +}; + +} + +std::string TGzipCodec::Decompress(const std::string& data) const { + TMemoryInput input(data.data(), data.size()); + TString result; + TStringOutput resultOutput(result); + TZLibDecompress inputStreamStorage(&input); + TransferData(&inputStreamStorage, &resultOutput); + return result; +} + +std::unique_ptr TGzipCodec::CreateCoder(TBuffer& result, int quality) const { + return std::make_unique(result, ZLib::GZip, quality >= 0 ? quality : 6); +} + +std::string TZstdCodec::Decompress(const std::string& data) const { + TMemoryInput input(data.data(), data.size()); + TString result; + TStringOutput resultOutput(result); + TZstdDecompress inputStreamStorage(&input); + TransferData(&inputStreamStorage, &resultOutput); + return result; +} + +std::unique_ptr TZstdCodec::CreateCoder(TBuffer& result, int quality) const { + return std::make_unique(result, quality); +} + +std::string TUnsupportedCodec::Decompress(const std::string&) const { + throw yexception() << "use of unsupported codec"; +} + +std::unique_ptr TUnsupportedCodec::CreateCoder(TBuffer&, int) const { + throw yexception() << "use of unsupported codec"; +} + +}; // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/codecs/ya.make b/ydb/public/sdk/cpp/src/client/topic/codecs/ya.make new file mode 100644 index 000000000000..b4c67bdb2536 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/codecs/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + codecs.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/topic/common/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/topic/common/CMakeLists.txt new file mode 100644 index 000000000000..7605cf26c02a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/CMakeLists.txt @@ -0,0 +1,15 @@ +_ydb_sdk_add_library(client-ydb_topic-common) + +target_link_libraries(client-ydb_topic-common PUBLIC + client-ydb_common_client-impl + client-ydb_types + monlib-dynamic_counters + retry +) + +target_sources(client-ydb_topic-common PRIVATE + executor_impl.cpp + retry_policy.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_topic-common) diff --git a/ydb/public/sdk/cpp/src/client/topic/common/callback_context.h b/ydb/public/sdk/cpp/src/client/topic/common/callback_context.h new file mode 100644 index 000000000000..a6789ee27f5f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/callback_context.h @@ -0,0 +1,202 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace NYdb::inline V3::NTopic { + +template +class TContextOwner; + +template +class TCallbackContext { + friend class TContextOwner; + + // thread_id -> number of LockShared calls from this thread + using TSharedLockCounter = std::map; + using TSharedLockCounterPtr = std::shared_ptr; + using TSpinLockPtr = std::shared_ptr; + +public: + using TMutexPtr = std::shared_ptr; + + class TBorrowed { + public: + explicit TBorrowed(const TCallbackContext& parent) + : Mutex(parent.Mutex) + , SharedLockCounterMutex(parent.SharedLockCounterMutex) + , SharedLockCounter(parent.SharedLockCounter) + { + // "Recursive shared lock". + // + // https://en.cppreference.com/w/cpp/thread/shared_mutex/lock_shared says: + // If lock_shared is called by a thread that already owns the mutex + // in any mode (exclusive or shared), the behavior is UNDEFINED. + // + // So if a thread calls LockShared more than once without releasing the lock, + // we should call lock_shared only on the first call. + + bool takeLock = false; + + with_lock(*SharedLockCounterMutex) { + auto& counter = SharedLockCounter->emplace(std::this_thread::get_id(), 0).first->second; + ++counter; + takeLock = counter == 1; + } + + if (takeLock) { + Mutex->lock_shared(); + } + + Ptr = parent.GuardedObjectPtr.get(); + } + + ~TBorrowed() { + bool releaseLock = false; + + with_lock(*SharedLockCounterMutex) { + auto it = SharedLockCounter->find(std::this_thread::get_id()); + Y_ABORT_UNLESS(it != SharedLockCounter->end()); + auto& counter = it->second; + --counter; + if (counter == 0) { + releaseLock = true; + SharedLockCounter->erase(it); + } + } + + if (releaseLock) { + Mutex->unlock_shared(); + } + } + + TGuardedObject* operator->() { + return Ptr; + } + + const TGuardedObject* operator->() const { + return Ptr; + } + + operator bool() { + return Ptr; + } + + private: + TMutexPtr Mutex; + TGuardedObject* Ptr = nullptr; + + TSpinLockPtr SharedLockCounterMutex; + TSharedLockCounterPtr SharedLockCounter; + }; + +public: + explicit TCallbackContext(std::shared_ptr ptr) + : Mutex(std::make_shared()) + , GuardedObjectPtr(std::move(ptr)) + , SharedLockCounterMutex(std::make_shared()) + , SharedLockCounter(std::make_shared()) + {} + + TBorrowed LockShared() { + return TBorrowed(*this); + } + +// TODO change section below to private after removing pqv1 read session implementation +// (relation of 1 owner : n impls) +public: + void Cancel() { + std::shared_ptr waste; + std::lock_guard lock(*Mutex); + std::swap(waste, GuardedObjectPtr); + } + + std::shared_ptr TryGet() const { + if (!GuardedObjectPtr) { + ythrow yexception() << "TryGet failed, empty GuardedObjectPtr"; + } + return GuardedObjectPtr; + } + +private: + + TMutexPtr Mutex; + std::shared_ptr GuardedObjectPtr; + + TSpinLockPtr SharedLockCounterMutex; + TSharedLockCounterPtr SharedLockCounter; +}; + +template +class TEnableSelfContext { + template + friend std::shared_ptr> MakeWithCallbackContext(Args&&... args); + +public: + TEnableSelfContext() = default; + ~TEnableSelfContext() = default; + + // non-moveable for simplicity, use only via shared pointers + TEnableSelfContext(const TEnableSelfContext&) = delete; + TEnableSelfContext(TEnableSelfContext&&) = delete; + TEnableSelfContext& operator=(const TEnableSelfContext&) = delete; + TEnableSelfContext& operator=(TEnableSelfContext&&) = delete; + +protected: + void SetSelfContext(std::shared_ptr ptr) { + SelfContext = std::make_shared>(std::move(ptr)); + } + +protected: + std::shared_ptr> SelfContext; +}; + +template +std::shared_ptr> MakeWithCallbackContext(Args&&... args) { + static_assert(std::is_base_of_v, T>, "Expected object derived from TEnableSelfContext"); + auto pObject = std::make_shared(std::forward(args)...); + pObject->SetSelfContext(pObject); + return pObject->SelfContext; +} + +template +class TContextOwner { +public: + template + TContextOwner(Args&&... args) + : ImplContext(MakeWithCallbackContext(std::forward(args)...)) { + } + + // may block + ~TContextOwner() { + CancelImpl(); + } + + TContextOwner(const TContextOwner&) = delete; + TContextOwner(TContextOwner&&) = default; + TContextOwner& operator=(const TContextOwner&) = delete; + TContextOwner& operator=(TContextOwner&&) = default; + +protected: + std::shared_ptr TryGetImpl() const { + return ImplContext->TryGet(); + } + + void CancelImpl() { + if (ImplContext) { + ImplContext->Cancel(); + } + } + +protected: + std::shared_ptr> ImplContext; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.cpp b/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.cpp new file mode 100644 index 000000000000..39c56acb3295 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.cpp @@ -0,0 +1,98 @@ +#include "executor_impl.h" + +namespace NYdb::inline V3::NTopic { + +void IAsyncExecutor::Post(TFunction&& f) { + PostImpl(std::move(f)); +} + +IAsyncExecutor::TPtr CreateDefaultExecutor() { + return CreateThreadPoolExecutor(1); +} + +void TThreadPoolExecutor::PostImpl(std::vector&& fs) { + for (auto& f : fs) { + ThreadPool->SafeAddFunc(std::move(f)); + } +} + +void TThreadPoolExecutor::PostImpl(TFunction&& f) { + ThreadPool->SafeAddFunc(std::move(f)); +} + +TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor) + : Executor(executor) +{ + Y_ABORT_UNLESS(executor); +} + +void TSerialExecutor::PostImpl(std::vector&& fs) { + for (auto& f : fs) { + PostImpl(std::move(f)); + } +} + +void TSerialExecutor::PostImpl(TFunction&& f) { + { + std::lock_guard guard(Mutex); + ExecutionQueue.push(std::move(f)); + if (Busy) { + return; + } + PostNext(); + } +} + +void TSerialExecutor::PostNext() { + Y_ABORT_UNLESS(!Busy); + + if (ExecutionQueue.empty()) { + return; + } + + auto weakThis = weak_from_this(); + Executor->Post([weakThis, f = std::move(ExecutionQueue.front())]() { + if (auto sharedThis = weakThis.lock()) { + f(); + { + std::lock_guard guard(sharedThis->Mutex); + sharedThis->Busy = false; + sharedThis->PostNext(); + } + } + }); + ExecutionQueue.pop(); + Busy = true; +} + +IExecutor::TPtr CreateThreadPoolExecutor(size_t threads) { + return MakeIntrusive(threads); +} + +IExecutor::TPtr CreateGenericExecutor() { + return CreateThreadPoolExecutor(1); +} + +IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr threadPool) { + return MakeIntrusive(std::move(threadPool)); +} + +TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr threadPool) + : ThreadPool(std::move(threadPool)) +{ + IsFakeThreadPool = dynamic_cast(ThreadPool.get()) != nullptr; +} + +TThreadPoolExecutor::TThreadPoolExecutor(size_t threadsCount) + : TThreadPoolExecutor(CreateThreadPool(threadsCount)) +{ + Y_ABORT_UNLESS(threadsCount > 0); + ThreadsCount = threadsCount; +} + +IExecutor::TPtr CreateSyncExecutor() +{ + return MakeIntrusive(); +} + +} // namespace NYdb::NTopic \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.h b/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.h new file mode 100644 index 000000000000..cd06370e6675 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/executor_impl.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NTopic { + +class IAsyncExecutor : public IExecutor { +private: + virtual void PostImpl(std::vector>&&) = 0; + virtual void PostImpl(std::function&&) = 0; + +public: + bool IsAsync() const override { + return true; + } + // Post Implementation MUST NOT run f before it returns + void Post(TFunction&& f) final; +}; + +IExecutor::TPtr CreateDefaultExecutor(); + + +class TThreadPoolExecutor : public IAsyncExecutor { +private: + std::shared_ptr ThreadPool; + +public: + TThreadPoolExecutor(std::shared_ptr threadPool); + TThreadPoolExecutor(size_t threadsCount); + ~TThreadPoolExecutor() = default; + + bool IsAsync() const override { + return !IsFakeThreadPool; + } + + void DoStart() override { + if (ThreadsCount) { + ThreadPool->Start(ThreadsCount); + } + } + +private: + void PostImpl(std::vector&& fs) override; + void PostImpl(TFunction&& f) override; + +private: + bool IsFakeThreadPool = false; + size_t ThreadsCount = 0; +}; + +class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this { +private: + IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job + bool Busy = false; //!< Set if some closure was scheduled for execution and did not finish yet + std::mutex Mutex = {}; + std::queue ExecutionQueue = {}; + +public: + TSerialExecutor(IAsyncExecutor::TPtr executor); + ~TSerialExecutor() = default; + +private: + void PostImpl(std::vector&& fs) override; + void PostImpl(TFunction&& f) override; + void PostNext(); +}; + +class TSyncExecutor : public IExecutor { +public: + void Post(TFunction&& f) final { + return f(); + } + bool IsAsync() const final { + return false; + } + void DoStart() override { + } +}; + +IExecutor::TPtr CreateGenericExecutor(); + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/common/log_lazy.h b/ydb/public/sdk/cpp/src/client/topic/common/log_lazy.h new file mode 100644 index 000000000000..0635ef2cc89b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/log_lazy.h @@ -0,0 +1,10 @@ +#pragma once + +#ifdef LOG_LAZY +#error log macro redefinition +#endif + +#define LOG_LAZY(log, priority, message) \ + if (log.IsOpen() && log.FiltrationLevel() >= priority) { \ + log.Write(priority, message); \ + } diff --git a/ydb/public/sdk/cpp/src/client/topic/common/retry_policy.cpp b/ydb/public/sdk/cpp/src/client/topic/common/retry_policy.cpp new file mode 100644 index 000000000000..a2bac4767a24 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/retry_policy.cpp @@ -0,0 +1,32 @@ +#include +#include + +namespace NYdb::inline V3::NTopic { + +IRetryPolicy::TPtr IRetryPolicy::GetDefaultPolicy() { + static IRetryPolicy::TPtr policy = GetExponentialBackoffPolicy(); + return policy; +} + +IRetryPolicy::TPtr IRetryPolicy::GetNoRetryPolicy() { + return ::IRetryPolicy::GetNoRetryPolicy(); +} + +IRetryPolicy::TPtr +IRetryPolicy::GetExponentialBackoffPolicy(TDuration minDelay, TDuration minLongRetryDelay, TDuration maxDelay, + size_t maxRetries, TDuration maxTime, double scaleFactor, + std::function customRetryClassFunction) { + return ::IRetryPolicy::GetExponentialBackoffPolicy( + customRetryClassFunction ? customRetryClassFunction : GetRetryErrorClass, minDelay, + minLongRetryDelay, maxDelay, maxRetries, maxTime, scaleFactor); +} + +IRetryPolicy::TPtr +IRetryPolicy::GetFixedIntervalPolicy(TDuration delay, TDuration longRetryDelay, size_t maxRetries, TDuration maxTime, + std::function customRetryClassFunction) { + return ::IRetryPolicy::GetFixedIntervalPolicy( + customRetryClassFunction ? customRetryClassFunction : GetRetryErrorClass, delay, + longRetryDelay, maxRetries, maxTime); +} + +} // namespace NYdb::NTopic \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/topic/common/trace_lazy.h b/ydb/public/sdk/cpp/src/client/topic/common/trace_lazy.h new file mode 100644 index 000000000000..c082e1d43245 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/trace_lazy.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#ifdef TRACE_LAZY +#error log macro redefinition +#endif + +const std::string TRACE_EVENT_MARKER = "TRACE_EVENT"; + +#define TRACE_LAZY(log, eventName, ...) \ + if (log.IsOpen() && log.FiltrationLevel() >= TLOG_RESOURCES) { \ + TStringBuilder message; \ + message << TRACE_EVENT_MARKER << ' ' << eventName << ' '; \ + __VA_ARGS__; \ + log.Write(TLOG_RESOURCES, message); \ + } + + +#define TRACE_KV(key, value) (message << " " << (key) << "=" << (value)) +#define TRACE_KV_IF(condition, key, value) (condition ? (message << " " << (key) << "=" << (value)) : message) +#define TRACE_IF(condition, ...) (condition ? __VA_ARGS__ : message) diff --git a/ydb/public/sdk/cpp/src/client/topic/common/ya.make b/ydb/public/sdk/cpp/src/client/topic/common/ya.make new file mode 100644 index 000000000000..084d12ffa5ef --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/common/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + callback_context.h + executor_impl.h + executor_impl.cpp + log_lazy.h + retry_policy.cpp + trace_lazy.h +) + +PEERDIR( + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic + + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/types + + + library/cpp/monlib/dynamic_counters +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/topic/impl/CMakeLists.txt new file mode 100644 index 000000000000..299c0dbe2691 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/CMakeLists.txt @@ -0,0 +1,40 @@ +_ydb_sdk_add_library(client-ydb_topic-impl) + +target_link_libraries(client-ydb_topic-impl PUBLIC + yutil + grpc-client + monlib-dynamic_counters + monlib-metrics + string_utils-url + persqueue-obfuscate + api-grpc-draft + api-grpc + impl-ydb_internal-make_request + client-ydb_common_client-impl + client-ydb_driver + client-ydb_topic-common + client-ydb_proto + proto_output +) + +target_compile_options(client-ydb_topic-impl + PRIVATE + -Wno-deprecated +) + +target_sources(client-ydb_topic-impl + PRIVATE + common.cpp + deferred_commit.cpp + event_handlers.cpp + offsets_collector.cpp + read_session_event.cpp + read_session.cpp + topic_impl.cpp + topic.cpp + transaction.cpp + write_session_impl.cpp + write_session.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_topic-impl) diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/common.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/common.cpp new file mode 100644 index 000000000000..5b3be0554b1a --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/common.cpp @@ -0,0 +1,109 @@ +#include "common.h" + +#include + +namespace NYdb::inline V3::NTopic { + +ERetryErrorClass GetRetryErrorClass(EStatus status) { + switch (status) { + case EStatus::SUCCESS: // NoRetry? + case EStatus::INTERNAL_ERROR: // NoRetry? + case EStatus::ABORTED: + case EStatus::UNAVAILABLE: + case EStatus::GENERIC_ERROR: // NoRetry? + case EStatus::BAD_SESSION: // NoRetry? + case EStatus::SESSION_EXPIRED: + case EStatus::CANCELLED: + case EStatus::UNDETERMINED: + case EStatus::SESSION_BUSY: + case EStatus::CLIENT_INTERNAL_ERROR: + case EStatus::CLIENT_CANCELLED: + case EStatus::CLIENT_OUT_OF_RANGE: + return ERetryErrorClass::ShortRetry; + + case EStatus::OVERLOADED: + case EStatus::TIMEOUT: + case EStatus::TRANSPORT_UNAVAILABLE: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::CLIENT_DEADLINE_EXCEEDED: + case EStatus::CLIENT_LIMITS_REACHED: + case EStatus::CLIENT_DISCOVERY_FAILED: + return ERetryErrorClass::LongRetry; + + case EStatus::SCHEME_ERROR: + case EStatus::STATUS_UNDEFINED: + case EStatus::BAD_REQUEST: + case EStatus::UNAUTHORIZED: + case EStatus::PRECONDITION_FAILED: + case EStatus::UNSUPPORTED: + case EStatus::ALREADY_EXISTS: + case EStatus::NOT_FOUND: + case EStatus::EXTERNAL_ERROR: + case EStatus::CLIENT_UNAUTHENTICATED: + case EStatus::CLIENT_CALL_UNIMPLEMENTED: + return ERetryErrorClass::NoRetry; + } +} + +ERetryErrorClass GetRetryErrorClassV2(EStatus status) { + switch (status) { + case EStatus::SCHEME_ERROR: + return ERetryErrorClass::NoRetry; + default: + return GetRetryErrorClass(status); + + } +} + +std::string IssuesSingleLineString(const NYdb::NIssue::TIssues& issues) { + return SubstGlobalCopy(issues.ToString(), '\n', ' '); +} + +void Cancel(NYdbGrpc::IQueueClientContextPtr& context) { + if (context) { + context->Cancel(); + } +} + +NYdb::NIssue::TIssues MakeIssueWithSubIssues(const std::string& description, const NYdb::NIssue::TIssues& subissues) { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::TIssue issue(description); + for (const auto& i : subissues) { + issue.AddSubIssue(MakeIntrusive(i)); + } + issues.AddIssue(std::move(issue)); + return issues; +} + +static std::string_view SplitPort(std::string_view endpoint) { + for (int i = endpoint.size() - 1; i >= 0; --i) { + if (endpoint[i] == ':') { + return endpoint.substr(i + 1, std::string_view::npos); + } + if (!IsDigit(endpoint[i])) { + return std::string_view(); // empty + } + } + return std::string_view(); // empty +} + +std::string ApplyClusterEndpoint(std::string_view driverEndpoint, const std::string& clusterDiscoveryEndpoint) { + const std::string_view clusterDiscoveryPort = SplitPort(clusterDiscoveryEndpoint); + if (!clusterDiscoveryPort.empty()) { + return clusterDiscoveryEndpoint; + } + + const std::string_view driverPort = SplitPort(driverEndpoint); + if (driverPort.empty()) { + return clusterDiscoveryEndpoint; + } + + const bool hasColon = clusterDiscoveryEndpoint.find(':') != std::string::npos; + if (hasColon) { + return TStringBuilder() << '[' << clusterDiscoveryEndpoint << "]:" << driverPort; + } else { + return TStringBuilder() << clusterDiscoveryEndpoint << ':' << driverPort; + } +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/common.h b/ydb/public/sdk/cpp/src/client/topic/impl/common.h new file mode 100644 index 000000000000..0445589cbda0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/common.h @@ -0,0 +1,426 @@ +#pragma once + +#include +#include + +#include + +#include + +#include +#include + +namespace NYdb::inline V3::NTopic { + +ERetryErrorClass GetRetryErrorClass(EStatus status); +ERetryErrorClass GetRetryErrorClassV2(EStatus status); + +void Cancel(NYdbGrpc::IQueueClientContextPtr& context); + +NYdb::NIssue::TIssues MakeIssueWithSubIssues(const std::string& description, const NYdb::NIssue::TIssues& subissues); + +std::string IssuesSingleLineString(const NYdb::NIssue::TIssues& issues); + +template +size_t CalcDataSize(const typename TEvent::TEvent& event) { + constexpr bool UseMigrationProtocol = !std::is_same_v; + + if (const typename TEvent::TDataReceivedEvent* dataEvent = + std::get_if(&event)) { + size_t len = 0; + + bool hasCompressedMsgs = [&dataEvent](){ + if constexpr (UseMigrationProtocol) { + return dataEvent->IsCompressedMessages(); + } else { + return dataEvent->HasCompressedMessages(); + } + }(); + + if (hasCompressedMsgs) { + for (const auto& msg : dataEvent->GetCompressedMessages()) { + len += msg.GetData().size(); + } + } else { + for (const auto& msg : dataEvent->GetMessages()) { + if (!msg.HasException()) { + len += msg.GetData().size(); + } + } + } + return len; + } + return 0; +} + +template +bool IsErrorMessage(const TMessage& serverMessage) { + const Ydb::StatusIds::StatusCode status = serverMessage.status(); + return status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; +} + +template +TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(serverMessage.issues(), issues); + return TPlainStatus(static_cast(serverMessage.status()), std::move(issues)); +} + +// Gets source endpoint for the whole driver (or persqueue client) +// and endpoint that was given us by the cluster discovery service +// and gives endpoint for the current LB cluster. +// For examples see tests. +std::string ApplyClusterEndpoint(std::string_view driverEndpoint, const std::string& clusterDiscoveryEndpoint); + +// Factory for IStreamRequestReadWriteProcessor +// It is created in order to separate grpc transport logic from +// the logic of session. +// So there is grpc factory implementation to use in SDK +// and another one to use in tests for testing only session logic +// without transport stuff. +template +struct ISessionConnectionProcessorFactory { + using IProcessor = NYdbGrpc::IStreamRequestReadWriteProcessor; + using TConnectedCallback = std::function; + using TConnectTimeoutCallback = std::function; + + virtual ~ISessionConnectionProcessorFactory() = default; + + // Creates processor + virtual void CreateProcessor( + // Params for connect. + TConnectedCallback callback, + const TRpcRequestSettings& requestSettings, + NYdbGrpc::IQueueClientContextPtr connectContext, + // Params for timeout and its cancellation. + TDuration connectTimeout, + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext, + TConnectTimeoutCallback connectTimeoutCallback, + // Params for delay before reconnect and its cancellation. + TDuration connectDelay = TDuration::Zero(), + NYdbGrpc::IQueueClientContextPtr connectDelayOperationContext = nullptr) = 0; +}; + +template +class TSessionConnectionProcessorFactory : public ISessionConnectionProcessorFactory, + public std::enable_shared_from_this> +{ +public: + using TConnectedCallback = typename ISessionConnectionProcessorFactory::TConnectedCallback; + using TConnectTimeoutCallback = typename ISessionConnectionProcessorFactory::TConnectTimeoutCallback; + TSessionConnectionProcessorFactory( + TGRpcConnectionsImpl::TStreamRpc rpc, + std::shared_ptr connections, + TDbDriverStatePtr dbState + ) + : Rpc(rpc) + , Connections(std::move(connections)) + , DbDriverState(dbState) + { + } + + void CreateProcessor( + TConnectedCallback callback, + const TRpcRequestSettings& requestSettings, + NYdbGrpc::IQueueClientContextPtr connectContext, + TDuration connectTimeout, + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext, + TConnectTimeoutCallback connectTimeoutCallback, + TDuration connectDelay, + NYdbGrpc::IQueueClientContextPtr connectDelayOperationContext) override + { + Y_ASSERT(connectContext); + Y_ASSERT(connectTimeoutContext); + Y_ASSERT((connectDelay == TDuration::Zero()) == !connectDelayOperationContext); + if (connectDelay == TDuration::Zero()) { + Connect(std::move(callback), + requestSettings, + std::move(connectContext), + connectTimeout, + std::move(connectTimeoutContext), + std::move(connectTimeoutCallback)); + } else { + auto connect = [ + weakThis = this->weak_from_this(), + callback = std::move(callback), + requestSettings, + connectContext = std::move(connectContext), + connectTimeout, + connectTimeoutContext = std::move(connectTimeoutContext), + connectTimeoutCallback = std::move(connectTimeoutCallback) + ] (bool ok) + { + if (!ok) { + return; + } + + if (auto sharedThis = weakThis.lock()) { + sharedThis->Connect( + std::move(callback), + requestSettings, + std::move(connectContext), + connectTimeout, + std::move(connectTimeoutContext), + std::move(connectTimeoutCallback) + ); + } + }; + + Connections->ScheduleCallback( + connectDelay, + std::move(connect), + std::move(connectDelayOperationContext) + ); + } + } + +private: + void Connect( + TConnectedCallback callback, + const TRpcRequestSettings& requestSettings, + NYdbGrpc::IQueueClientContextPtr connectContext, + TDuration connectTimeout, + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext, + TConnectTimeoutCallback connectTimeoutCallback) + { + Connections->StartBidirectionalStream( + std::move(callback), + Rpc, + DbDriverState, + requestSettings, + std::move(connectContext) + ); + + Connections->ScheduleCallback( + connectTimeout, + std::move(connectTimeoutCallback), + std::move(connectTimeoutContext) + ); + } + +private: + TGRpcConnectionsImpl::TStreamRpc Rpc; + std::shared_ptr Connections; + TDbDriverStatePtr DbDriverState; +}; + +template +std::shared_ptr> + CreateConnectionProcessorFactory( + TGRpcConnectionsImpl::TStreamRpc rpc, + std::shared_ptr connections, + TDbDriverStatePtr dbState + ) +{ + return std::make_shared>(rpc, std::move(connections), std::move(dbState)); +} + + + +template +struct TBaseEventInfo { + using TEvent = TEvent_; + + TEvent Event; + + TEvent& GetEvent() { + return Event; + } + + void OnUserRetrievedEvent() { + } + + template + TBaseEventInfo(T&& event) + : Event(std::forward(event)) + {} +}; + + +class ISignalable { +public: + ISignalable() = default; + virtual ~ISignalable() {} + virtual void Signal() = 0; +}; + +// Waiter on queue. +// Future or GetEvent call +class TWaiter { +public: + TWaiter() = default; + + TWaiter(const TWaiter&) = delete; + TWaiter& operator=(const TWaiter&) = delete; + TWaiter(TWaiter&&) = default; + TWaiter& operator=(TWaiter&&) = default; + + TWaiter(NThreading::TPromise&& promise, ISignalable* self) + : Promise(promise) + , Future(promise.Initialized() ? Promise.GetFuture() : NThreading::TFuture()) + , Self(self) + { + } + + void Signal() { + if (Self) { + Self->Signal(); + } + if (Promise.Initialized() && !Promise.HasValue()) { + Promise.SetValue(); + } + } + + bool Valid() const { + if (!Future.Initialized()) return false; + return !Promise.Initialized() || Promise.GetFuture().StateId() == Future.StateId(); + } + + NThreading::TPromise ExtractPromise() { + NThreading::TPromise promise; + Y_ABORT_UNLESS(!promise.Initialized()); + std::swap(Promise, promise); + return promise; + } + + NThreading::TFuture GetFuture() { + Y_ABORT_UNLESS(Future.Initialized()); + return Future; + } + +private: + NThreading::TPromise Promise; + NThreading::TFuture Future; + ISignalable* Self = nullptr; +}; + + + +// Class that is responsible for: +// - events queue; +// - signalling futures that wait for events; +// - packing events for waiters; +// - waking up waiters. +// Thread safe. +template > +class TBaseSessionEventsQueue : public ISignalable { +protected: + using TSelf = TBaseSessionEventsQueue; + using TSettings = TSettings_; + using TEvent = TEvent_; + using TEventInfo = TEventInfo_; + using TClosedEvent = TClosedEvent_; + using TExecutor = TExecutor_; + + // Template for visitor implementation. + struct TBaseHandlersVisitor { + TBaseHandlersVisitor(const TSettings& settings, TEvent& event) + : Settings(settings) + , Event(event) + {} + + template + bool PushHandler(TEvent&& event, const TFunc& specific, const TCommonFunc& common) { + if (specific) { + PushSpecificHandler(std::move(event), specific); + return true; + } + if (common) { + PushCommonHandler(std::move(event), common); + return true; + } + return false; + } + + template + void PushSpecificHandler(TEvent&& event, const TFunc& f) { + Post(Settings.EventHandlers_.HandlersExecutor_, + [func = f, event = std::move(event)]() mutable { + func(std::get(event)); + }); + } + + template + void PushCommonHandler(TEvent&& event, const TFunc& f) { + Post(Settings.EventHandlers_.HandlersExecutor_, + [func = f, event = std::move(event)]() mutable { func(event); }); + } + + virtual void Post(const typename TExecutor::TPtr& executor, typename TExecutor::TFunction&& f) { + executor->Post(std::move(f)); + } + + const TSettings& Settings; + TEvent& Event; + }; + + +public: + TBaseSessionEventsQueue(const TSettings& settings) + : Settings(settings) + , Waiter(NThreading::NewPromise(), this) + {} + + virtual ~TBaseSessionEventsQueue() = default; + + + void Signal() override { + CondVar.notify_one(); + } + +protected: + virtual bool HasEventsImpl() const { // Assumes that we're under lock. + return !Events.empty() || CloseEvent; + } + + TWaiter PopWaiterImpl() { // Assumes that we're under lock. + TWaiter waiter(Waiter.ExtractPromise(), this); + WaiterWillBeSignaled = true; + return std::move(waiter); + } + + void WaitEventsImpl() { // Assumes that we're under lock. Posteffect: HasEventsImpl() is true. + while (!HasEventsImpl()) { + std::unique_lock lk(Mutex, std::adopt_lock); + CondVar.wait(lk); + lk.release(); + } + } + + void RenewWaiterImpl() { + if (Events.empty() && WaiterWillBeSignaled) { + Waiter = TWaiter(NThreading::NewPromise(), this); + WaiterWillBeSignaled = false; + } + } + +public: + NThreading::TFuture WaitEvent() { + { + std::lock_guard guard (Mutex); + if (HasEventsImpl()) { + return NThreading::MakeFuture(); // Signalled + } else { + Y_ABORT_UNLESS(Waiter.Valid()); + auto res = Waiter.GetFuture(); + return res; + } + } + } + + bool IsClosed() { + return Closed; + } + +protected: + const TSettings& Settings; + TWaiter Waiter; + bool WaiterWillBeSignaled = false; + std::queue Events; + std::condition_variable CondVar; + std::mutex Mutex; + std::optional CloseEvent; + std::atomic Closed = false; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/counters_logger.h b/ydb/public/sdk/cpp/src/client/topic/impl/counters_logger.h new file mode 100644 index 000000000000..9edc26a6b201 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/counters_logger.h @@ -0,0 +1,151 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NTopic { + +template +class TSingleClusterReadSessionImpl; + +template +using TCallbackContextPtr = std::shared_ptr>>; + +template +class TCountersLogger : public std::enable_shared_from_this> { +public: + static constexpr auto UPDATE_PERIOD = TDuration::Seconds(1); + +public: + explicit TCountersLogger(std::shared_ptr connections, + std::vector> sessions, + TReaderCounters::TPtr counters, const TLog& log, std::string prefix, TInstant startSessionTime) + : Connections(std::move(connections)) + , SessionsContexts(std::move(sessions)) + , Counters(std::move(counters)) + , Log(log) + , Prefix(std::move(prefix)) + , StartSessionTime(startSessionTime) { + } + + std::shared_ptr>> MakeCallbackContext() { + SelfContext = std::make_shared>>(this->shared_from_this()); + return SelfContext; + } + + void Start() { + Y_ABORT_UNLESS(SelfContext); + ScheduleDumpCountersToLog(); + } + + void Stop() { + Y_ABORT_UNLESS(SelfContext); + + { + std::lock_guard guard(Lock); + Stopping = true; + } + + // Log final counters. + DumpCountersToLog(); + } + +private: + void ScheduleDumpCountersToLog(size_t timeNumber = 0) { + std::lock_guard guard(Lock); + if (Stopping) { + return; + } + + DumpCountersContext = Connections->CreateContext(); + if (DumpCountersContext) { + auto callback = [ctx = SelfContext, timeNumber](bool ok) { + if (ok) { + if (auto borrowedSelf = ctx->LockShared()) { + borrowedSelf->DumpCountersToLog(timeNumber); + } + } + }; + Connections->ScheduleCallback(UPDATE_PERIOD, + std::move(callback), + DumpCountersContext); + } + } + + void DumpCountersToLog(size_t timeNumber = 0) { + const bool logCounters = timeNumber % 60 == 0; // Every 1 minute. + const bool dumpSessionsStatistics = timeNumber % 600 == 0; // Every 10 minutes. + + *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - StartSessionTime).MilliSeconds(); + + { + std::optional log; + bool dumpHeader = true; + + for (auto& sessionCtx : SessionsContexts) { + if (auto borrowedSession = sessionCtx->LockShared()) { + borrowedSession->UpdateMemoryUsageStatistics(); + if (dumpSessionsStatistics) { + if (dumpHeader) { + log.emplace(&Log, TLOG_INFO); + (*log) << "Read/commit by partition streams (cluster:topic:partition:stream-id:read-offset:committed-offset):"; + dumpHeader = false; + } + borrowedSession->DumpStatisticsToLog(*log); + } + } + } + } + +#define C(counter) \ + << " " Y_STRINGIZE(counter) ": " \ + << Counters->counter->Val() \ + /**/ + + if (logCounters) { + LOG_LAZY(Log, TLOG_INFO, + TStringBuilder() << Prefix << "Counters: {" + C(Errors) + C(CurrentSessionLifetimeMs) + C(BytesRead) + C(MessagesRead) + C(BytesReadCompressed) + C(BytesInflightUncompressed) + C(BytesInflightCompressed) + C(BytesInflightTotal) + C(MessagesInflight) + << " }" + ); + } + +#undef C + + ScheduleDumpCountersToLog(timeNumber + 1); + } + + +private: + std::shared_ptr>> SelfContext; + + TSpinLock Lock; + + std::shared_ptr Connections; + std::vector> SessionsContexts; + + IQueueClientContextPtr DumpCountersContext; + + TReaderCounters::TPtr Counters; + + TLog Log; + const std::string Prefix; + const TInstant StartSessionTime; + + bool Stopping = false; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/deferred_commit.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/deferred_commit.cpp new file mode 100644 index 000000000000..53e10bfd156b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/deferred_commit.cpp @@ -0,0 +1,131 @@ +#include "read_session_impl.ipp" + +#include + +#include + +namespace NYdb::inline V3::NTopic { + +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TDeferredCommit + +class TDeferredCommit::TImpl { +public: + + void Add(const TPartitionSession::TPtr& partitionStream, ui64 startOffset, ui64 endOffset); + void Add(const TPartitionSession::TPtr& partitionStream, ui64 offset); + + void Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message); + void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); + + void Commit(); + +private: + static void Add(const TPartitionSession::TPtr& partitionStream, TDisjointIntervalTree& offsetSet, ui64 startOffset, ui64 endOffset); + +private: + // Partition stream -> offsets set. + std::unordered_map, THash> Offsets; +}; + +TDeferredCommit::TDeferredCommit() { +} + +TDeferredCommit::TDeferredCommit(TDeferredCommit&&) = default; + +TDeferredCommit& TDeferredCommit::operator=(TDeferredCommit&&) = default; + +TDeferredCommit::~TDeferredCommit() { +} + +#define GET_IMPL() \ + if (!Impl) { \ + Impl = std::make_unique(); \ + } \ + Impl + +void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, uint64_t startOffset, uint64_t endOffset) { + GET_IMPL()->Add(partitionStream, startOffset, endOffset); +} + +void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, uint64_t offset) { + GET_IMPL()->Add(partitionStream, offset); +} + +void TDeferredCommit::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) { + GET_IMPL()->Add(message); +} + +void TDeferredCommit::Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent) { + GET_IMPL()->Add(dataReceivedEvent); +} + +#undef GET_IMPL + +void TDeferredCommit::Commit() { + if (Impl) { + Impl->Commit(); + } +} + +void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) { + Y_ASSERT(message.GetPartitionSession()); + Add(message.GetPartitionSession(), message.GetOffset()); +} + +void TDeferredCommit::TImpl::Add(const TPartitionSession::TPtr& partitionStream, TDisjointIntervalTree& offsetSet, ui64 startOffset, ui64 endOffset) { + if (offsetSet.Intersects(startOffset, endOffset)) { + ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval [" + << startOffset << "; " << endOffset + << ") for partition stream with id " << partitionStream->GetPartitionSessionId()); + } else { + offsetSet.InsertInterval(startOffset, endOffset); + } +} + +void TDeferredCommit::TImpl::Add(const TPartitionSession::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) { + Y_ASSERT(partitionStream); + Add(partitionStream, Offsets[partitionStream], startOffset, endOffset); +} + +void TDeferredCommit::TImpl::Add(const TPartitionSession::TPtr& partitionStream, ui64 offset) { + Y_ASSERT(partitionStream); + auto& offsetSet = Offsets[partitionStream]; + if (offsetSet.Has(offset)) { + ThrowFatalError(TStringBuilder() << "Commit set already has offset " << offset + << " for partition stream with id " << partitionStream->GetPartitionSessionId()); + } else { + offsetSet.Insert(offset); + } +} + +void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent) { + const TPartitionSession::TPtr& partitionStream = dataReceivedEvent.GetPartitionSession(); + Y_ASSERT(partitionStream); + auto& offsetSet = Offsets[partitionStream]; + auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0); + for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) { + auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i); + if (msgOffsetRange.first == endOffset) { + endOffset= msgOffsetRange.second; + } else { + Add(partitionStream, offsetSet, startOffset, endOffset); + startOffset = msgOffsetRange.first; + endOffset = msgOffsetRange.second; + } + } + Add(partitionStream, offsetSet, startOffset, endOffset); +} + +void TDeferredCommit::TImpl::Commit() { + for (auto&& [partitionStream, offsetRanges] : Offsets) { + for (auto&& [startOffset, endOffset] : offsetRanges) { + static_cast*>(partitionStream.Get())->Commit(startOffset, endOffset); + } + } + Offsets.clear(); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/event_handlers.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/event_handlers.cpp new file mode 100644 index 000000000000..1a4fad20fa00 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/event_handlers.cpp @@ -0,0 +1,151 @@ +#include + +#include + +namespace NYdb::inline V3::NTopic { + +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEventHandlers + +class TGracefulReleasingSimpleDataHandlers : public TThrRefBase { +public: + explicit TGracefulReleasingSimpleDataHandlers(std::function dataHandler, bool commitAfterProcessing) + : DataHandler(std::move(dataHandler)) + , CommitAfterProcessing(commitAfterProcessing) + { + } + + void OnDataReceived(TReadSessionEvent::TDataReceivedEvent& event) { + Y_ASSERT(event.GetMessagesCount()); + TDeferredCommit deferredCommit; + { + std::lock_guard guard(Lock); + auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionSession()->GetPartitionSessionId()]; + // Messages could contain holes in offset, but later commit ack will tell us right border. + // So we can easily insert the whole interval with holes included. + // It will be removed from set by specifying proper right border. + auto firstMessageOffsets = GetMessageOffsetRange(event, 0); + auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1); + + offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second); + + if (CommitAfterProcessing) { + deferredCommit.Add(event); + } + } + DataHandler(event); + deferredCommit.Commit(); + } + + void OnCommitAcknowledgement(TReadSessionEvent::TCommitOffsetAcknowledgementEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionSession()->GetPartitionSessionId(); + auto& offsetSet = PartitionStreamToUncommittedOffsets[partitionStreamId]; + if (offsetSet.EraseInterval(0, event.GetCommittedOffset() + 1)) { // Remove some offsets. + if (offsetSet.Empty()) { // No offsets left. + auto unconfirmedDestroyIt = UnconfirmedDestroys.find(partitionStreamId); + if (unconfirmedDestroyIt != UnconfirmedDestroys.end()) { + // Confirm and forget about this partition stream. + unconfirmedDestroyIt->second.Confirm(); + UnconfirmedDestroys.erase(unconfirmedDestroyIt); + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + } + } + } + } + + void OnCreatePartitionStream(TReadSessionEvent::TStartPartitionSessionEvent& event) { + { + std::lock_guard guard(Lock); + Y_ABORT_UNLESS(PartitionStreamToUncommittedOffsets[event.GetPartitionSession()->GetPartitionSessionId()].Empty()); + } + event.Confirm(); + } + + void OnDestroyPartitionStream(TReadSessionEvent::TStopPartitionSessionEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionSession()->GetPartitionSessionId(); + Y_ABORT_UNLESS(UnconfirmedDestroys.find(partitionStreamId) == UnconfirmedDestroys.end()); + if (PartitionStreamToUncommittedOffsets[partitionStreamId].Empty()) { + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + event.Confirm(); + } else { + UnconfirmedDestroys.emplace(partitionStreamId, std::move(event)); + } + } + + void OnEndPartitionStream(TReadSessionEvent::TEndPartitionSessionEvent& event) { + event.Confirm(); + } + + void OnPartitionStreamClosed(TReadSessionEvent::TPartitionSessionClosedEvent& event) { + std::lock_guard guard(Lock); + const ui64 partitionStreamId = event.GetPartitionSession()->GetPartitionSessionId(); + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + UnconfirmedDestroys.erase(partitionStreamId); + } + +private: + TAdaptiveLock Lock; // For the case when user gave us multithreaded executor. + const std::function DataHandler; + const bool CommitAfterProcessing; + std::unordered_map> PartitionStreamToUncommittedOffsets; // Partition stream id -> set of offsets. + std::unordered_map UnconfirmedDestroys; // Partition stream id -> destroy events. +}; + +TReadSessionSettings::TEventHandlers& TReadSessionSettings::TEventHandlers::SimpleDataHandlers(std::function dataHandler, + bool commitDataAfterProcessing, + bool gracefulReleaseAfterCommit) { + Y_ASSERT(dataHandler); + + PartitionSessionStatusHandler([](TReadSessionEvent::TPartitionSessionStatusEvent&){}); + + if (gracefulReleaseAfterCommit) { + auto handlers = MakeIntrusive(std::move(dataHandler), commitDataAfterProcessing); + DataReceivedHandler([handlers](TReadSessionEvent::TDataReceivedEvent& event) { + handlers->OnDataReceived(event); + }); + StartPartitionSessionHandler([handlers](TReadSessionEvent::TStartPartitionSessionEvent& event) { + handlers->OnCreatePartitionStream(event); + }); + StopPartitionSessionHandler([handlers](TReadSessionEvent::TStopPartitionSessionEvent& event) { + handlers->OnDestroyPartitionStream(event); + }); + EndPartitionSessionHandler([handlers](TReadSessionEvent::TEndPartitionSessionEvent& event) { + handlers->OnEndPartitionStream(event); + }); + CommitOffsetAcknowledgementHandler([handlers](TReadSessionEvent::TCommitOffsetAcknowledgementEvent& event) { + handlers->OnCommitAcknowledgement(event); + }); + PartitionSessionClosedHandler([handlers](TReadSessionEvent::TPartitionSessionClosedEvent& event) { + handlers->OnPartitionStreamClosed(event); + }); + } else { + if (commitDataAfterProcessing) { + DataReceivedHandler([dataHandler = std::move(dataHandler)](TReadSessionEvent::TDataReceivedEvent& event) { + TDeferredCommit deferredCommit; + deferredCommit.Add(event); + dataHandler(event); + deferredCommit.Commit(); + }); + } else { + DataReceivedHandler(std::move(dataHandler)); + } + StartPartitionSessionHandler([](TReadSessionEvent::TStartPartitionSessionEvent& event) { + event.Confirm(); + }); + StopPartitionSessionHandler([](TReadSessionEvent::TStopPartitionSessionEvent& event) { + event.Confirm(); + }); + EndPartitionSessionHandler([](TReadSessionEvent::TEndPartitionSessionEvent& event) { + event.Confirm(); + }); + CommitOffsetAcknowledgementHandler([](TReadSessionEvent::TCommitOffsetAcknowledgementEvent&){}); + PartitionSessionClosedHandler([](TReadSessionEvent::TPartitionSessionClosedEvent&){}); + } + return *this; +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.cpp new file mode 100644 index 000000000000..b846ccf703c0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.cpp @@ -0,0 +1,72 @@ +#include "offsets_collector.h" + +namespace NYdb::inline V3::NTopic { + +std::vector TOffsetsCollector::GetOffsets() const +{ + std::vector topics; + + for (auto& [path, partitions] : Ranges) { + TTopicOffsets topic; + topic.Path = path; + + topics.push_back(std::move(topic)); + + for (auto& [id, ranges] : partitions) { + TPartitionOffsets partition; + partition.PartitionId = id; + + TTopicOffsets& t = topics.back(); + t.Partitions.push_back(std::move(partition)); + + for (auto& range : ranges) { + TPartitionOffsets& p = t.Partitions.back(); + + TOffsetsRange r; + r.Start = range.first; + r.End = range.second; + + p.Offsets.push_back(r); + } + } + } + + return topics; +} + +void TOffsetsCollector::CollectOffsets(const std::vector& events) +{ + for (auto& event : events) { + if (auto* e = std::get_if(&event)) { + CollectOffsets(*e); + } + } +} + +void TOffsetsCollector::CollectOffsets(const TReadSessionEvent::TEvent& event) +{ + if (auto* e = std::get_if(&event)) { + CollectOffsets(*e); + } +} + +void TOffsetsCollector::CollectOffsets(const TReadSessionEvent::TDataReceivedEvent& event) +{ + const auto& session = *event.GetPartitionSession(); + const std::string& topicPath = session.GetTopicPath(); + uint32_t partitionId = session.GetPartitionId(); + + if (event.HasCompressedMessages()) { + for (auto& message : event.GetCompressedMessages()) { + uint64_t offset = message.GetOffset(); + Ranges[topicPath][partitionId].InsertInterval(offset, offset + 1); + } + } else { + for (auto& message : event.GetMessages()) { + uint64_t offset = message.GetOffset(); + Ranges[topicPath][partitionId].InsertInterval(offset, offset + 1); + } + } +} + +} \ No newline at end of file diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.h b/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.h new file mode 100644 index 000000000000..3e18087051e1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/offsets_collector.h @@ -0,0 +1,35 @@ +#pragma once + +#include "topic_impl.h" +#include "transaction.h" + +#include +#include + +#include + +#include +#include +#include + +#include + +namespace NYdb::inline V3::NTopic { + +class TOffsetsCollector { +public: + std::vector GetOffsets() const; + + void CollectOffsets(const std::vector& events); + void CollectOffsets(const TReadSessionEvent::TEvent& event); + +private: + // topic -> partition -> (begin, end) + using TOffsetRanges = std::unordered_map>>; + + void CollectOffsets(const TReadSessionEvent::TDataReceivedEvent& event); + + TOffsetRanges Ranges; +}; + +} diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/read_session.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/read_session.cpp new file mode 100644 index 000000000000..e73ce3f40339 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/read_session.cpp @@ -0,0 +1,327 @@ +#include "read_session.h" + +#include +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +namespace NYdb::inline V3::NTopic { + +static const std::string DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; + +void SetReadInTransaction(TReadSessionEvent::TEvent& event) +{ + if (auto* e = std::get_if(&event)) { + e->SetReadInTransaction(); + } +} + +TReadSession::TReadSession(const TReadSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : Settings(settings) + , SessionId(CreateGuidAsString()) + , Log(settings.Log_.value_or(dbDriverState->Log)) + , Client(std::move(client)) + , Connections(std::move(connections)) + , DbDriverState(std::move(dbDriverState)) +{ + if (!Settings.RetryPolicy_) { + Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); + } + + MakeCountersIfNeeded(); +} + +TReadSession::~TReadSession() { + Close(TDuration::Zero()); + + Abort(EStatus::ABORTED, "Aborted"); + ClearAllEvents(); + + if (CbContext) { + CbContext->Cancel(); + } + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + } +} + +void TReadSession::Start() { + EventsQueue = std::make_shared>(Settings); + + if (!ValidateSettings()) { + return; + } + + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Starting read session"); + + TDeferredActions deferred; + with_lock(Lock) { + if (Aborting) { + return; + } + Topics = Settings.Topics_; + CreateClusterSessionsImpl(deferred); + } + SetupCountersLogger(); +} + +void TReadSession::CreateClusterSessionsImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + // Create cluster sessions. + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Starting single session" + ); + auto context = Client->CreateContext(); + if (!context) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return; + } + + CbContext = MakeWithCallbackContext>( + Settings, + DbDriverState->Database, + SessionId, + "", + Log, + Client->CreateReadSessionConnectionProcessorFactory(), + EventsQueue, + context, + 1, + 1 + ); + + deferred.DeferStartSession(CbContext); +} + +bool TReadSession::ValidateSettings() { + NYdb::NIssue::TIssues issues; + if (Settings.Topics_.empty()) { + issues.AddIssue("Empty topics list."); + } + + if (Settings.ConsumerName_.empty() && !Settings.WithoutConsumer_) { + issues.AddIssue("No consumer specified."); + } + + if (!Settings.ConsumerName_.empty() && Settings.WithoutConsumer_) { + issues.AddIssue("No need to specify a consumer when reading without a consumer."); + } + + if (Settings.MaxMemoryUsageBytes_ < 1_MB) { + issues.AddIssue("Too small max memory usage. Valid values start from 1 megabyte."); + } + + if (issues) { + Abort(EStatus::BAD_REQUEST, MakeIssueWithSubIssues("Invalid read session settings", issues)); + return false; + } else { + return true; + } +} + +NThreading::TFuture TReadSession::WaitEvent() { + return EventsQueue->WaitEvent(); +} + +std::vector TReadSession::GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) { + auto res = EventsQueue->GetEvents(block, maxEventsCount, maxByteSize); + if (EventsQueue->IsClosed()) { + Abort(EStatus::ABORTED, "Aborted"); + } + return res; +} + +std::vector TReadSession::GetEvents(const TReadSessionGetEventSettings& settings) +{ + auto events = GetEvents(settings.Block_, settings.MaxEventsCount_, settings.MaxByteSize_); + if (!events.empty() && settings.Tx_) { + auto& tx = settings.Tx_->get(); + CbContext->TryGet()->CollectOffsets(tx, events, Client); + for (auto& event : events) { + SetReadInTransaction(event); + } + } + return events; +} + +std::optional TReadSession::GetEvent(bool block, size_t maxByteSize) { + auto res = EventsQueue->GetEvent(block, maxByteSize); + if (EventsQueue->IsClosed()) { + Abort(EStatus::ABORTED, "Aborted"); + } + return res; +} + +std::optional TReadSession::GetEvent(const TReadSessionGetEventSettings& settings) +{ + auto event = GetEvent(settings.Block_, settings.MaxByteSize_); + if (event && settings.Tx_) { + auto& tx = settings.Tx_->get(); + CbContext->TryGet()->CollectOffsets(tx, *event, Client); + SetReadInTransaction(*event); + } + return event; +} + +bool TReadSession::Close(TDuration timeout) { + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Closing read session. Close timeout: " << timeout); + // Log final counters. + if (CountersLogger) { + CountersLogger->Stop(); + } + with_lock(Lock) { + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + } + } + + TSingleClusterReadSessionImpl::TPtr session; + NThreading::TPromise promise = NThreading::NewPromise(); + auto callback = [=]() mutable { + promise.TrySetValue(true); + }; + + TDeferredActions deferred; + with_lock(Lock) { + if (Closing || Aborting) { + return false; + } + + if (!timeout) { + AbortImpl(EStatus::ABORTED, "Close with zero timeout", deferred); + return false; + } + + Closing = true; + session = CbContext->TryGet(); + } + session->Close(callback); + + callback(); // For the case when there are no subsessions yet. + + auto timeoutCallback = [=](bool) mutable { + promise.TrySetValue(false); + }; + + auto timeoutContext = Connections->CreateContext(); + if (!timeoutContext) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return false; + } + Connections->ScheduleCallback(timeout, + std::move(timeoutCallback), + timeoutContext); + + // Wait. + NThreading::TFuture resultFuture = promise.GetFuture(); + const bool result = resultFuture.GetValueSync(); + if (result) { + Cancel(timeoutContext); + + NYdb::NIssue::TIssues issues; + issues.AddIssue("Session was gracefully closed"); + EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); + } else { + ++*Settings.Counters_->Errors; + session->Abort(); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); + EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); + } + { + std::lock_guard guard(Lock); + Aborting = true; // Set abort flag for doing nothing on destructor. + } + return result; +} + +void TReadSession::ClearAllEvents() { + EventsQueue->ClearAllEvents(); +} + +TStringBuilder TReadSession::GetLogPrefix() const { + return TStringBuilder() << GetDatabaseLogPrefix(DbDriverState->Database) << "[" << SessionId << "] "; +} + +void TReadSession::MakeCountersIfNeeded() { + if (!Settings.Counters_ || HasNullCounters(*Settings.Counters_)) { + TReaderCounters::TPtr counters = MakeIntrusive(); + if (Settings.Counters_) { + *counters = *Settings.Counters_; // Copy all counters that have been set by user. + } + MakeCountersNotNull(*counters); + Settings.Counters(counters); + } +} + +void TReadSession::SetupCountersLogger() { + std::lock_guard guard(Lock); + std::vector> sessions{CbContext}; + + CountersLogger = std::make_shared>(Connections, sessions, Settings.Counters_, Log, + GetLogPrefix(), StartSessionTime); + DumpCountersContext = CountersLogger->MakeCallbackContext(); + CountersLogger->Start(); +} + +void TReadSession::AbortImpl(TDeferredActions&) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Aborting) { + Aborting = true; + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + } + if (CbContext) { + CbContext->TryGet()->Abort(); + } + } +} + +void TReadSession::AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& deferred) { + LOG_LAZY(Log, TLOG_NOTICE, GetLogPrefix() << "Aborting read session. Description: " << closeEvent.DebugString()); + + EventsQueue->Close(std::move(closeEvent), deferred); + AbortImpl(deferred); +} + +void TReadSession::AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + AbortImpl(TSessionClosedEvent(statusCode, std::move(issues)), deferred); +} + +void TReadSession::AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + AbortImpl(statusCode, std::move(issues), deferred); +} + +void TReadSession::Abort(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + Abort(TSessionClosedEvent(statusCode, std::move(issues))); +} + +void TReadSession::Abort(EStatus statusCode, const std::string& message) { + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + Abort(statusCode, std::move(issues)); +} + +void TReadSession::Abort(TSessionClosedEvent&& closeEvent) { + TDeferredActions deferred; + with_lock(Lock) { + AbortImpl(std::move(closeEvent), deferred); + } +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/read_session.h b/ydb/public/sdk/cpp/src/client/topic/impl/read_session.h new file mode 100644 index 000000000000..21cf15479cfe --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/read_session.h @@ -0,0 +1,88 @@ +#pragma once + +#include "counters_logger.h" +#include "read_session_impl.ipp" +#include "topic_impl.h" + +#include + +namespace NYdb::inline V3::NTopic { + +class TReadSession : public IReadSession { +public: + TReadSession(const TReadSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + ~TReadSession(); + + void Start(); + + NThreading::TFuture WaitEvent() override; + std::vector GetEvents(bool block, + std::optional maxEventsCount, + size_t maxByteSize) override; + std::vector GetEvents(const TReadSessionGetEventSettings& settings) override; + std::optional GetEvent(bool block, + size_t maxByteSize) override; + std::optional GetEvent(const TReadSessionGetEventSettings& settings) override; + + bool Close(TDuration timeout) override; + + inline std::string GetSessionId() const override { + return SessionId; + } + + inline TReaderCounters::TPtr GetCounters() const override { + return Settings.Counters_; // Always not nullptr. + } + + void Abort(TSessionClosedEvent&& closeEvent); + + void ClearAllEvents(); + +private: + TStringBuilder GetLogPrefix() const; + + // Start + bool ValidateSettings(); + + void CreateClusterSessionsImpl(TDeferredActions& deferred); + + void MakeCountersIfNeeded(); + void SetupCountersLogger(); + + // Shutdown. + void Abort(EStatus statusCode, NYdb::NIssue::TIssues&& issues); + void Abort(EStatus statusCode, const std::string& message); + + void AbortImpl(TDeferredActions& deferred); + void AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred); + +private: + + TReadSessionSettings Settings; + const std::string SessionId; + const TInstant StartSessionTime = TInstant::Now(); + TLog Log; + std::shared_ptr Client; + std::shared_ptr Connections; + TDbDriverStatePtr DbDriverState; + TAdaptiveLock Lock; + std::shared_ptr> EventsQueue; + + std::shared_ptr>> CbContext; + std::vector Topics; + + std::shared_ptr> CountersLogger; + std::shared_ptr>> DumpCountersContext; + + // Exiting. + bool Aborting = false; + bool Closing = false; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/read_session_event.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_event.cpp new file mode 100644 index 000000000000..8eb8858e84b2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_event.cpp @@ -0,0 +1,446 @@ +#include "common.h" +#include "read_session_impl.ipp" + +#include + +namespace NYdb::inline V3::NTopic { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Aliases for event types + +using TDataReceivedEvent = TReadSessionEvent::TDataReceivedEvent; +using TMessageInformation = TDataReceivedEvent::TMessageInformation; +using TMessageBase = TDataReceivedEvent::TMessageBase; +using TMessage = TDataReceivedEvent::TMessage; +using TCompressedMessage = TDataReceivedEvent::TCompressedMessage; +using TCommitOffsetAcknowledgementEvent = TReadSessionEvent::TCommitOffsetAcknowledgementEvent; +using TStartPartitionSessionEvent = TReadSessionEvent::TStartPartitionSessionEvent; +using TStopPartitionSessionEvent = TReadSessionEvent::TStopPartitionSessionEvent; +using TEndPartitionSessionEvent = TReadSessionEvent::TEndPartitionSessionEvent; +using TPartitionSessionStatusEvent = TReadSessionEvent::TPartitionSessionStatusEvent; +using TPartitionSessionClosedEvent = TReadSessionEvent::TPartitionSessionClosedEvent; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Helpers + +std::pair GetMessageOffsetRange(const TDataReceivedEvent& dataReceivedEvent, uint64_t index) { + if (dataReceivedEvent.HasCompressedMessages()) { + const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; + return {msg.GetOffset(), msg.GetOffset() + 1}; + } + const auto& msg = dataReceivedEvent.GetMessages()[index]; + return {msg.GetOffset(), msg.GetOffset() + 1}; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent::TMessageInformation + +TMessageInformation::TMessageInformation( + uint64_t offset, + std::string producerId, + uint64_t seqNo, + TInstant createTime, + TInstant writeTime, + TWriteSessionMeta::TPtr meta, + TMessageMeta::TPtr messageMeta, + uint64_t uncompressedSize, + std::string messageGroupId +) + : Offset(offset) + , ProducerId(producerId) + , SeqNo(seqNo) + , CreateTime(createTime) + , WriteTime(writeTime) + , Meta(meta) + , MessageMeta(messageMeta) + , UncompressedSize(uncompressedSize) + , MessageGroupId(messageGroupId) +{} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent::TPartitionSessionAccessor + +TReadSessionEvent::TPartitionSessionAccessor::TPartitionSessionAccessor(TPartitionSession::TPtr partitionSession) + : PartitionSession(std::move(partitionSession)) +{} + +const TPartitionSession::TPtr& TReadSessionEvent::TPartitionSessionAccessor::GetPartitionSession() const { + return PartitionSession; +} + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const { + const auto* self = static_cast(this); + res << " Partition session id: " << self->GetPartitionSessionId() + << " Topic: \"" << self->GetTopicPath() << "\"" + << " Partition: " << self->GetPartitionId(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent::TMessageBase + +TMessageBase::TMessageBase(const std::string& data, TMessageInformation info) + : Data(data) + , Information(std::move(info)) +{} + +const std::string& TMessageBase::GetData() const { + return Data; +} + +uint64_t TMessageBase::GetOffset() const { + return Information.Offset; +} + +const std::string& TMessageBase::GetProducerId() const { + return Information.ProducerId; +} + +const std::string& TMessageBase::GetMessageGroupId() const { + return Information.MessageGroupId; +} + +uint64_t TMessageBase::GetSeqNo() const { + return Information.SeqNo; +} + +TInstant TMessageBase::GetCreateTime() const { + return Information.CreateTime; +} + +TInstant TMessageBase::GetWriteTime() const { + return Information.WriteTime; +} + +const TWriteSessionMeta::TPtr& TMessageBase::GetMeta() const { + return Information.Meta; +} + +const TMessageMeta::TPtr& TMessageBase::GetMessageMeta() const { + return Information.MessageMeta; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + try { + const std::string& data = self->GetData(); + if (printData) { + ret << " Data: \"" << data << "\""; + } else { + ret << " Data: .." << data.size() << " bytes.."; + } + } catch (...) { + ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\""; + } + ret << " Information: {" + << " Offset: " << self->GetOffset() + << " ProducerId: \"" << self->GetProducerId() << "\"" + << " SeqNo: " << self->GetSeqNo() + << " CreateTime: " << self->GetCreateTime() + << " WriteTime: " << self->GetWriteTime() + << " MessageGroupId: \"" << self->GetMessageGroupId() << "\""; + ret << " Meta: {"; + bool firstKey = true; + for (const auto& [k, v] : self->GetMeta()->Fields) { + ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\""; + firstKey = false; + } + ret << " }"; + ret << " MessageMeta: {"; + firstKey = true; + for (const auto& [k, v] : self->GetMessageMeta()->Fields) { + ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\""; + firstKey = false; + } + ret << " } }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage + +TMessage::TMessage(const std::string& data, + std::exception_ptr decompressionException, + TMessageInformation information, + TPartitionSession::TPtr partitionSession) + : TMessageBase(data, std::move(information)) + , TPartitionSessionAccessor(std::move(partitionSession)) + , DecompressionException(std::move(decompressionException)) { +} + +const std::string& TMessage::GetData() const { + if (DecompressionException) { + std::rethrow_exception(DecompressionException); + } + return TMessageBase::GetData(); +} + +bool TMessage::HasException() const { + return DecompressionException != nullptr; +} + +void TMessage::Commit() { + static_cast*>(PartitionSession.Get()) + ->Commit(Information.Offset, Information.Offset + 1); +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "Message {"; + static_cast(self)->DebugString(ret, printData); + self->GetPartitionSession()->DebugString(ret); + ret << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage + +TCompressedMessage::TCompressedMessage(ECodec codec, + const std::string& data, + TMessageInformation information, + TPartitionSession::TPtr partitionSession) + : TMessageBase(data, std::move(information)) + , TPartitionSessionAccessor(std::move(partitionSession)) + , Codec(codec) { +} + +ECodec TCompressedMessage::GetCodec() const { + return Codec; +} + +uint64_t TCompressedMessage::GetUncompressedSize() const { + return Information.UncompressedSize; +} + +void TCompressedMessage::Commit() { + static_cast*>(PartitionSession.Get()) + ->Commit(Information.Offset, Information.Offset + 1); +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "CompressedMessage {"; + static_cast(self)->DebugString(ret, printData); + self->GetPartitionSession()->DebugString(ret); + ret << " Codec: " << self->GetCodec() + << " Uncompressed size: " << self->GetUncompressedSize() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TDataReceivedEvent + +TDataReceivedEvent::TDataReceivedEvent(std::vector messages, std::vector compressedMessages, + TPartitionSession::TPtr partitionSession) + : TPartitionSessionAccessor(std::move(partitionSession)) + , Messages(std::move(messages)) + , CompressedMessages(std::move(compressedMessages)) { + for (size_t i = 0; i < GetMessagesCount(); ++i) { + auto [from, to] = GetMessageOffsetRange(*this, i); + if (OffsetRanges.empty() || OffsetRanges.back().second != from) { + OffsetRanges.emplace_back(from, to); + } else { + OffsetRanges.back().second = to; + } + } +} + +void TDataReceivedEvent::Commit() { + if (ReadInTransaction) { + ythrow yexception() << "Offsets cannot be commited explicitly when reading in a transaction"; + } + + for (auto [from, to] : OffsetRanges) { + static_cast*>(PartitionSession.Get())->Commit(from, to); + } +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool printData) const { + const auto* self = static_cast(this); + ret << "DataReceived {"; + self->GetPartitionSession()->DebugString(ret); + if (self->HasCompressedMessages()) { + for (const auto& message : self->GetCompressedMessages()) { + ret << " "; + message.DebugString(ret, printData); + } + } else { + for (const auto& message : self->GetMessages()) { + ret << " "; + message.DebugString(ret, printData); + } + } + ret << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TCommitOffsetAcknowledgementEvent + +TCommitOffsetAcknowledgementEvent::TCommitOffsetAcknowledgementEvent(TPartitionSession::TPtr partitionSession, + uint64_t committedOffset) + : TPartitionSessionAccessor(std::move(partitionSession)) + , CommittedOffset(committedOffset) { +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "CommitAcknowledgement {"; + self->GetPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TStartPartitionSessionEvent + +TStartPartitionSessionEvent::TStartPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset, + uint64_t endOffset) + : TPartitionSessionAccessor(std::move(partitionSession)) + , CommittedOffset(committedOffset) + , EndOffset(endOffset) { +} + +void TStartPartitionSessionEvent::Confirm(std::optional readOffset, std::optional commitOffset) { + if (PartitionSession) { + static_cast*>(PartitionSession.Get()) + ->ConfirmCreate(readOffset, commitOffset); + } +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "StartPartitionSession {"; + self->GetPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " EndOffset: " << self->GetEndOffset() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TStopPartitionSessionEvent + +TStopPartitionSessionEvent::TStopPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset) + : TPartitionSessionAccessor(std::move(partitionSession)) + , CommittedOffset(committedOffset) { +} + +void TStopPartitionSessionEvent::Confirm() { + if (PartitionSession) { + static_cast*>(PartitionSession.Get())->ConfirmDestroy(); + } +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "StopPartitionSession {"; + self->GetPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TEndPartitionSessionEvent + +TEndPartitionSessionEvent::TEndPartitionSessionEvent(TPartitionSession::TPtr partitionSession, std::vector&& adjacentPartitionIds, std::vector&& childPartitionIds) + : TPartitionSessionAccessor(std::move(partitionSession)) + , AdjacentPartitionIds(std::move(adjacentPartitionIds)) + , ChildPartitionIds(std::move(childPartitionIds)) { +} + +void TEndPartitionSessionEvent::Confirm() { + if (PartitionSession) { + static_cast*>(PartitionSession.Get())->ConfirmEnd(GetChildPartitionIds()); + } +} + +void JoinIds(TStringBuilder& ret, const std::vector ids) { + ret << "["; + for (size_t i = 0; i < ids.size(); ++i) { + if (i) { + ret << ", "; + } + ret << ids[i]; + } + ret << "]"; +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "EndPartitionSession {"; + self->GetPartitionSession()->DebugString(ret); + ret << " AdjacentPartitionIds: "; + JoinIds(ret, self->GetAdjacentPartitionIds()); + ret << " ChildPartitionIds: "; + JoinIds(ret, self->GetChildPartitionIds()); + ret << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TPartitionSessionStatusEvent + +TPartitionSessionStatusEvent::TPartitionSessionStatusEvent(TPartitionSession::TPtr partitionSession, + uint64_t committedOffset, uint64_t readOffset, uint64_t endOffset, + TInstant writeTimeHighWatermark) + : TPartitionSessionAccessor(std::move(partitionSession)) + , CommittedOffset(committedOffset) + , ReadOffset(readOffset) + , EndOffset(endOffset) + , WriteTimeHighWatermark(writeTimeHighWatermark) { +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "PartitionSessionStatus {"; + self->GetPartitionSession()->DebugString(ret); + ret << " CommittedOffset: " << self->GetCommittedOffset() + << " ReadOffset: " << self->GetReadOffset() + << " EndOffset: " << self->GetEndOffset() + << " WriteWatermark: " << self->GetWriteTimeHighWatermark() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent::TPartitionSessionClosedEvent + +TPartitionSessionClosedEvent::TPartitionSessionClosedEvent(TPartitionSession::TPtr partitionSession, EReason reason) + : TPartitionSessionAccessor(std::move(partitionSession)) + , Reason(reason) +{ +} + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "PartitionSessionClosed {"; + self->GetPartitionSession()->DebugString(ret); + ret << " Reason: " << self->GetReason() + << " }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TSessionClosedEvent + +template<> +void TPrintable::DebugString(TStringBuilder& ret, bool) const { + const auto* self = static_cast(this); + ret << "SessionClosed { Status: " << self->GetStatus() + << " Issues: \"" << IssuesSingleLineString(self->GetIssues()) + << "\" }"; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NTopic::TReadSessionEvent + +std::string DebugString(const TReadSessionEvent::TEvent& event) { + return std::visit([](const auto& ev) { return ev.DebugString(); }, event); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.h new file mode 100644 index 000000000000..f34dd9258c16 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.h @@ -0,0 +1,1383 @@ +#pragma once + +#ifndef INCLUDE_READ_SESSION_IMPL_H +#error "Do not include this file directly. Use read_session_impl.ipp instead." +#endif + +#include "common.h" +#include "counters_logger.h" +#include "offsets_collector.h" +#include "transaction.h" + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +#include + +#include +#include +#include + + +namespace NYdb::inline V3::NTopic { + +template +using TClientMessage = std::conditional_t; + +template +using TServerMessage = std::conditional_t; + +template +using IReadSessionConnectionProcessorFactory = + ISessionConnectionProcessorFactory, TServerMessage>; + +template +using IProcessor = typename IReadSessionConnectionProcessorFactory::IProcessor; + +template +using TPartitionData = std::conditional_t; + +template +using TAWriteSessionMeta = std::conditional_t; + +template +using TAMessageMeta = std::conditional_t; + +template +using TASessionClosedEvent = std::conditional_t; + +template +using TAPartitionStream = std::conditional_t; + +template +using TAReadSessionEvent = std::conditional_t; + +template +using IARetryPolicy = std::conditional_t; + +template +using IAExecutor = std::conditional_t; + +template +using TAReadSessionSettings = std::conditional_t; + +template +using TADataReceivedEvent = typename TAReadSessionEvent::TDataReceivedEvent; + +template +class TPartitionStreamImpl; + +template +class TSingleClusterReadSessionImpl; + +template +class TDeferredActions; + +template +class TReadSessionEventsQueue; + +class TReadSession; + +template +class TDataDecompressionInfo; + +template +using TDataDecompressionInfoPtr = typename TDataDecompressionInfo::TPtr; + +template +using TCallbackContextPtr = std::shared_ptr>>; + + +template +class TUserRetrievedEventsInfoAccumulator { +public: + void Add(TDataDecompressionInfoPtr info, i64 decompressedSize); + void OnUserRetrievedEvent() const; + +private: + struct TCounter { + i64 DecompressedSize = 0; + size_t MessagesCount = 0; + }; + + std::map, TCounter> Counters; +}; + +// Special class that stores actions to be done after lock will be released. +template +class TDeferredActions { +public: + ~TDeferredActions() { + DoActions(); + } + + void DeferReadFromProcessor(const typename IProcessor::TPtr& processor, TServerMessage* dst, typename IProcessor::TReadCallback callback); + void DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task); + void DeferAbortSession(TCallbackContextPtr cbContext, TASessionClosedEvent&& closeEvent); + void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYdb::NIssue::TIssues&& issues); + void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message); + void DeferAbortSession(TCallbackContextPtr cbContext, TPlainStatus&& status); + void DeferReconnection(TCallbackContextPtr cbContext, TPlainStatus&& status); + void DeferStartSession(TCallbackContextPtr cbContext); + void DeferSignalWaiter(TWaiter&& waiter); + void DeferDestroyDecompressionInfos(std::vector>&& infos); + +private: + void DoActions(); + + void Read(); + void StartExecutorTasks(); + void AbortSession(); + void Reconnect(); + void SignalWaiters(); + void StartSessions(); + +private: + // Read. + typename IProcessor::TPtr Processor; + TServerMessage* ReadDst = nullptr; + typename IProcessor::TReadCallback ReadCallback; + + // Executor tasks. + std::vector::TPtr, typename IAExecutor::TFunction>> ExecutorsTasks; + + // Abort session. + std::optional> SessionClosedEvent; + + // Waiters. + std::vector Waiters; + + // Reconnection and abort. + TCallbackContextPtr CbContext; + TPlainStatus ReconnectionStatus; + + // Contexts for sessions to start + std::vector> CbContexts; + + std::vector> DecompressionInfos; +}; + +template +class TDataDecompressionInfo : public std::enable_shared_from_this> { +public: + using TPtr = std::shared_ptr>; + + TDataDecompressionInfo(const TDataDecompressionInfo&) = default; + TDataDecompressionInfo(TDataDecompressionInfo&&) = default; + TDataDecompressionInfo( + TPartitionData&& msg, + TCallbackContextPtr cbContext, + bool doDecompress, + i64 serverBytesSize = 0 // to increment read request bytes size + ); + ~TDataDecompressionInfo(); + + i64 StartDecompressionTasks(const typename IAExecutor::TPtr& executor, + i64 availableMemory, + TDeferredActions& deferred); + void PlanDecompressionTasks(double averageCompressionRatio, + TIntrusivePtr> partitionStream); + + void OnDestroyReadSession(); + + bool IsReady() const { + return SourceDataNotProcessed == 0; + } + + bool AllDecompressionTasksStarted() const { + return Tasks.empty(); + } + + i64 GetCompressedDataSize() const { + return CompressedDataSize; + } + + const TPartitionData& GetServerMessage() const { + return ServerMessage; + } + + TPartitionData& GetServerMessage() { + return ServerMessage; + } + + bool GetDoDecompress() const { + return DoDecompress; + } + + i64 GetServerBytesSize() const { + return ServerBytesSize; + } + + std::optional> GetReadyThreshold() const { + size_t readyCount = 0; + std::pair ret; + for (auto i = ReadyThresholds.begin(), end = ReadyThresholds.end(); i != end; ++i) { + if (i->Ready) { + ret.first = i->Batch; + ret.second = i->Message; + ++readyCount; + } else { + break; + } + } + if (!readyCount) { + return std::nullopt; + } + return ret; + } + + typename TAWriteSessionMeta::TPtr GetBatchMeta(size_t batchIndex) const { + Y_ASSERT(batchIndex < BatchesMeta.size()); + return BatchesMeta[batchIndex]; + } + + template > + typename TAMessageMeta::TPtr GetMessageMeta(size_t batchIndex, size_t messageIndex) const { + Y_ASSERT(batchIndex < MessagesMeta.size()); + Y_ASSERT(messageIndex < MessagesMeta[batchIndex].size()); + return MessagesMeta[batchIndex][messageIndex]; + } + + bool HasMoreData() const { + return CurrentReadingMessage.first < static_cast(GetServerMessage().batches_size()); + } + + bool HasReadyUnreadData() const; + + void PutDecompressionError(std::exception_ptr error, size_t batch, size_t message); + std::exception_ptr GetDecompressionError(size_t batch, size_t message); + + void OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount); + void OnUserRetrievedEvent(i64 decompressedDataSize, size_t messagesCount); + +private: + // Special struct for marking (batch/message) as ready. + struct TReadyMessageThreshold { + size_t Batch = 0; // Last ready batch with message index. + size_t Message = 0; // Last ready message index. + std::atomic Ready = false; + }; + + struct TDecompressionTask { + TDecompressionTask(TDataDecompressionInfo::TPtr parent, TIntrusivePtr> partitionStream, TReadyMessageThreshold* ready); + + // Decompress and notify about memory consumption changes. + void operator()(); + + void Add(size_t batch, size_t message, size_t sourceDataSize, size_t estimatedDecompressedSize); + + size_t AddedDataSize() const { + return SourceDataSize; + } + size_t AddedMessagesCount() const { + return Messages.size(); + } + + i64 GetEstimatedDecompressedSize() const { + return EstimatedDecompressedSize; + } + + void ClearParent(); + + private: + TDataDecompressionInfo::TPtr Parent; + TIntrusivePtr> PartitionStream; + i64 SourceDataSize = 0; + i64 EstimatedDecompressedSize = 0; + i64 DecompressedSize = 0; + struct TMessageRange { + size_t Batch; + std::pair MessageRange; + }; + std::vector Messages; + TReadyMessageThreshold* Ready; + }; + + void BuildBatchesMeta(); + +private: + TPartitionData ServerMessage; + using TMetadataPtrVector = std::vector::TPtr>; + using TMessageMetaPtrVector = std::vector::TPtr>; + TMetadataPtrVector BatchesMeta; + std::vector MessagesMeta; + TCallbackContextPtr CbContext; + bool DoDecompress; + i64 ServerBytesSize = 0; + std::atomic SourceDataNotProcessed = 0; + std::pair CurrentDecompressingMessage = {0, 0}; // (Batch, Message) + std::deque ReadyThresholds; + std::pair CurrentReadingMessage = {0, 0}; // (Batch, Message) + + // Decompression exceptions. + // Optimization for rare using. + std::atomic DecompressionErrorsStructCreated = false; + TAdaptiveLock DecompressionErrorsStructLock; + std::vector> DecompressionErrors; + + std::atomic MessagesInflight = 0; + std::atomic CompressedDataSize = 0; + std::atomic DecompressedDataSize = 0; + + std::deque Tasks; +}; + +template +class TDataDecompressionEvent { +public: + TDataDecompressionEvent(size_t batch, size_t message, TDataDecompressionInfoPtr parent, std::atomic& ready) : + Batch{batch}, + Message{message}, + Parent{std::move(parent)}, + Ready{ready} + { + } + + bool IsReady() const { + return Ready; + } + + void TakeData(TIntrusivePtr> partitionStream, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + size_t& maxByteSize, + size_t& dataSize) const; + + TDataDecompressionInfoPtr GetParent() const { + return Parent; + } + +private: + size_t Batch; + size_t Message; + TDataDecompressionInfoPtr Parent; + std::atomic& Ready; +}; + +template +struct IUserRetrievedEventCallback { + virtual ~IUserRetrievedEventCallback() = default; + + virtual void OnUserRetrievedEvent(i64 decompressedSize, size_t messagesCount) = 0; +}; + +template +struct TReadSessionEventInfo { + using TEvent = typename TAReadSessionEvent::TEvent; + using TMessage = typename TADataReceivedEvent::TMessage; + using TCompressedMessage = typename TADataReceivedEvent::TCompressedMessage; + + // Event with only partition stream ref. + // Partition stream holds all its events. + TIntrusivePtr> PartitionStream; + bool HasDataEvents = false; + size_t EventsCount = 0; + std::optional Event; + TCallbackContextPtr CbContext; + + // Close event. + TReadSessionEventInfo(const TASessionClosedEvent& event, TCallbackContextPtr cbContext = {}) + : Event(TEvent(event)) + , CbContext(std::move(cbContext)) + { + } + + // Usual event. + TReadSessionEventInfo(TIntrusivePtr> partitionStream, TCallbackContextPtr cbContext, TEvent event); + + // Data event. + TReadSessionEventInfo(TIntrusivePtr> partitionStream, + TCallbackContextPtr cbContext, + bool hasDataEvents); + + bool IsEmpty() const; + bool IsDataEvent() const; + + TEvent& GetEvent() { + Y_ASSERT(Event); + return *Event; + } + + bool IsSessionClosedEvent() const { + return Event && std::holds_alternative>(*Event); + } +}; + +// Raw data with maybe uncompressed parts or other read session event. +template +struct TRawPartitionStreamEvent { + using TEvent = typename TAReadSessionEvent::TEvent; + + std::variant, TEvent> Event; + + TRawPartitionStreamEvent(const TRawPartitionStreamEvent&) = default; + TRawPartitionStreamEvent(TRawPartitionStreamEvent&&) = default; + + TRawPartitionStreamEvent(size_t batch, + size_t message, + TDataDecompressionInfoPtr parent, + std::atomic &ready) + : Event(std::in_place_type_t>(), + batch, + message, + std::move(parent), + ready) + { + } + + template + explicit TRawPartitionStreamEvent(T&& event) + : Event(std::in_place_type_t(), std::forward(event)) + { + } + + bool IsDataEvent() const { + return std::holds_alternative>(Event); + } + + const TDataDecompressionEvent& GetDataEvent() const { + Y_ASSERT(IsDataEvent()); + return std::get>(Event); + } + + TEvent& GetEvent() { + Y_ASSERT(!IsDataEvent()); + return std::get(Event); + } + + const TEvent& GetEvent() const { + Y_ASSERT(!IsDataEvent()); + return std::get(Event); + } + + bool IsReady() const { + if (!IsDataEvent()) { + return true; + } + + return std::get>(Event).IsReady(); + } +}; + +template +class TRawPartitionStreamEventQueue { +public: + TRawPartitionStreamEventQueue(TCallbackContextPtr cbContext) + : CbContext(cbContext) { + }; + + template + TRawPartitionStreamEvent& emplace_back(Ts&&... event) + { + return NotReady.emplace_back(std::forward(event)...); + } + + bool empty() const + { + return Ready.empty() && NotReady.empty(); + } + + TRawPartitionStreamEvent& front() + { + Y_ABORT_UNLESS(!empty()); + + return (Ready.empty() ? NotReady : Ready).front(); + } + + void pop_front() + { + Y_ABORT_UNLESS(!empty()); + + (Ready.empty() ? NotReady : Ready).pop_front(); + } + + void pop_back() + { + Y_ABORT_UNLESS(!empty()); + + (NotReady.empty() ? Ready : NotReady).pop_back(); + } + + void clear() noexcept { + NotReady.clear(); + Ready.clear(); + } + + void SignalReadyEvents(TIntrusivePtr> stream, + TReadSessionEventsQueue& queue, + TDeferredActions& deferred); + void DeleteNotReadyTail(TDeferredActions& deferred); + + void GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator); + +private: + static void GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator, + std::deque>& queue); + +private: + TCallbackContextPtr CbContext; + std::deque> Ready; + std::deque> NotReady; +}; + +template +class TPartitionStreamImpl : public TAPartitionStream { +public: + struct TKey { // Hash is defined later in this file. + std::string Topic; + std::string Cluster; + ui64 Partition; + + bool operator==(const TKey& other) const { + // Compare the most variable fields first. + return Partition == other.Partition + && Cluster == other.Cluster + && Topic == other.Topic; + } + }; + + template > + TPartitionStreamImpl(ui64 partitionStreamId, + std::string topicPath, + std::string cluster, + ui64 partitionGroupId, + ui64 partitionId, + ui64 assignId, + ui64 readOffset, + TCallbackContextPtr cbContext) + : Key{topicPath, cluster, partitionId} + , AssignId(assignId) + , FirstNotReadOffset(readOffset) + , CbContext(std::move(cbContext)) + , EventsQueue(CbContext) + { + TAPartitionStream::PartitionStreamId = partitionStreamId; + TAPartitionStream::TopicPath = std::move(topicPath); + TAPartitionStream::Cluster = std::move(cluster); + TAPartitionStream::PartitionGroupId = partitionGroupId; + TAPartitionStream::PartitionId = partitionId; + MaxCommittedOffset = readOffset; + } + + template > + TPartitionStreamImpl(ui64 partitionStreamId, + std::string topicPath, + i64 partitionId, + i64 assignId, + i64 readOffset, + TCallbackContextPtr cbContext) + : Key{topicPath, "", static_cast(partitionId)} + , AssignId(static_cast(assignId)) + , FirstNotReadOffset(static_cast(readOffset)) + , CbContext(std::move(cbContext)) + , EventsQueue(CbContext) + { + TAPartitionStream::PartitionSessionId = partitionStreamId; + TAPartitionStream::TopicPath = std::move(topicPath); + TAPartitionStream::PartitionId = static_cast(partitionId); + MaxCommittedOffset = static_cast(readOffset); + } + + ~TPartitionStreamImpl() = default; + + ui64 GetFirstNotReadOffset() const { + return FirstNotReadOffset; + } + + void SetFirstNotReadOffset(const ui64 offset) { + FirstNotReadOffset = offset; + } + + void Commit(ui64 startOffset, ui64 endOffset) /*override*/; + void RequestStatus() override; + + void ConfirmCreate(std::optional readOffset, std::optional commitOffset); + void ConfirmDestroy(); + void ConfirmEnd(const std::vector& childIds); + + void StopReading() /*override*/; + void ResumeReading() /*override*/; + + ui64 GetAssignId() const { + return AssignId; + } + + const TKey& GetKey() const { + return Key; + } + + template + void InsertEvent(T&& event) { + EventsQueue.emplace_back(std::forward(event)); + } + + void InsertDataEvent(size_t batch, + size_t message, + TDataDecompressionInfoPtr parent, + std::atomic &ready) + { + EventsQueue.emplace_back(batch, message, std::move(parent), ready); + } + + bool HasEvents() const { + return !EventsQueue.empty(); + } + + TRawPartitionStreamEvent& TopEvent() { + return EventsQueue.front(); + } + + void PopEvent() { + EventsQueue.pop_front(); + } + + TCallbackContextPtr GetCbContext() const { + return CbContext; + } + + TLog GetLog() const; + + static void SignalReadyEvents(TIntrusivePtr> stream, + TReadSessionEventsQueue* queue, + TDeferredActions& deferred); + + + ui64 GetMaxReadOffset() const { + return MaxReadOffset; + } + + ui64 GetMaxCommittedOffset() const { + return MaxCommittedOffset; + } + + void UpdateMaxReadOffset(ui64 offset) { + if (offset > MaxReadOffset) { + MaxReadOffset = offset; + } + } + + void UpdateMaxCommittedOffset(ui64 offset) { + if (offset > MaxCommittedOffset) { + ClientCommits.EraseInterval(MaxCommittedOffset, offset); + MaxCommittedOffset = offset; + } + } + + bool HasCommitsInflight() const { + if (ClientCommits.Empty()) + return false; + auto range = *ClientCommits.begin(); + if (range.first > MaxCommittedOffset) + return false; + // Here we got first range that can be committed by server. + // If offset to commit is from same position - then nothing is inflight. + if (!Commits.Empty() && Commits.begin()->first == range.first) + return false; + return true; + } + + bool AddToCommitRanges(const ui64 startOffset, const ui64 endOffset, bool rangesMode) { + if (ClientCommits.Intersects(startOffset, endOffset) || startOffset < MaxCommittedOffset) { + auto id = [this](){ + if constexpr (UseMigrationProtocol) { + return this->PartitionStreamId; + } else { + return this->PartitionSessionId; + } + }(); + ThrowFatalError(TStringBuilder() << "Invalid offset range [" << startOffset << ", " << endOffset << ") : range must start from " + << MaxCommittedOffset << " or has some offsets that are committed already. Partition stream id: -" << id << Endl); + return false; + } + if (rangesMode) { // Otherwise no need to send it to server. + Y_ABORT_UNLESS(!Commits.Intersects(startOffset, endOffset)); + Commits.InsertInterval(startOffset, endOffset); + } + ClientCommits.InsertInterval(startOffset, endOffset); + return true; + } + + void DeleteNotReadyTail(TDeferredActions& deferred); + + void ClearQueue() noexcept { + EventsQueue.clear(); + } + + TRawPartitionStreamEventQueue ExtractQueue() noexcept { + return std::exchange(EventsQueue, TRawPartitionStreamEventQueue(CbContext)); + } + + static void GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator); + + std::mutex& GetLock() { + return Lock; + } + +private: + const TKey Key; + ui64 AssignId; + ui64 FirstNotReadOffset; + TCallbackContextPtr CbContext; + TRawPartitionStreamEventQueue EventsQueue; + ui64 MaxReadOffset = 0; + ui64 MaxCommittedOffset = 0; + + TDisjointIntervalTree Commits; + TDisjointIntervalTree ClientCommits; + + std::mutex Lock; +}; + + +template +class TReadSessionEventsQueue: public TBaseSessionEventsQueue, + typename TAReadSessionEvent::TEvent, + TASessionClosedEvent, + IAExecutor, + TReadSessionEventInfo> { + using TParent = TBaseSessionEventsQueue, + typename TAReadSessionEvent::TEvent, + TASessionClosedEvent, + IAExecutor, + TReadSessionEventInfo>; + +public: + TReadSessionEventsQueue(const TAReadSessionSettings& settings); + + // Assumes we are under lock. + TReadSessionEventInfo + GetEventImpl(size_t& maxByteSize, + TUserRetrievedEventsInfoAccumulator& accumulator); + + std::vector::TEvent> + GetEvents(bool block = false, + std::optional maxEventsCount = std::nullopt, + size_t maxByteSize = std::numeric_limits::max()); + + std::optional::TEvent> + GetEvent(bool block = false, + size_t maxByteSize = std::numeric_limits::max()); + + bool Close(const TASessionClosedEvent& event, TDeferredActions& deferred) { + TWaiter waiter; + std::vector> deferredDelete; + { + std::lock_guard guard(TParent::Mutex); + if (TParent::Closed) { + return false; + } + deferredDelete.reserve(TParent::Events.size()); + while (!TParent::Events.empty()) { + auto& event = TParent::Events.front(); + if (!event.IsEmpty()) { + deferredDelete.push_back(event.PartitionStream->ExtractQueue()); + } + TParent::Events.pop(); + } + TParent::CloseEvent = event; + TParent::Closed = true; + waiter = TWaiter(TParent::Waiter.ExtractPromise(), this); + } + + // Delayed deletion is necessary to avoid deadlock with PushEvent + deferredDelete.clear(); + + TReadSessionEventInfo info(event); + ApplyHandler(info, deferred); + deferred.DeferSignalWaiter(std::move(waiter)); + return true; + } + + bool TryApplyCallbackToEventImpl(typename TParent::TEvent& event, + TDeferredActions& deferred, + TCallbackContextPtr& cbContext); + bool HasDataEventCallback() const; + void ApplyCallbackToEventImpl(TADataReceivedEvent& event, + TUserRetrievedEventsInfoAccumulator&& eventsInfo, + TDeferredActions& deferred); + + void GetDataEventCallbackSettings(size_t& maxMessagesBytes); + + // Push usual event. Returns false if queue is closed + bool PushEvent(TIntrusivePtr> partitionStream, + typename TAReadSessionEvent::TEvent event, + TDeferredActions& deferred); + + // Push data event. Returns false if queue is closed + bool PushDataEvent(TIntrusivePtr> stream, + size_t batch, + size_t message, + TDataDecompressionInfoPtr parent, + std::atomic &ready); + + void SignalEventImpl(TIntrusivePtr> partitionStream, + TDeferredActions& deferred, + bool isDataEvent); // Assumes that we're under lock. + + void SignalReadyEvents(TIntrusivePtr> partitionStream); + + void SignalReadyEventsImpl(TIntrusivePtr> partitionStream, + TDeferredActions& deferred); // Assumes that we're under lock. + + void SignalWaiterImpl(TDeferredActions& deferred) { + TWaiter waiter = TParent::PopWaiterImpl(); + deferred.DeferSignalWaiter(std::move(waiter)); // No effect if waiter is empty. + } + + void ClearAllEvents(); + + void SetCallbackContext(TCallbackContextPtr& ctx) { + CbContext = ctx; + } + +private: + struct THandlersVisitor : public TParent::TBaseHandlersVisitor { + THandlersVisitor(const TAReadSessionSettings& settings, + typename TParent::TEvent& event, + TDeferredActions& deferred, + TCallbackContextPtr& cbContext) + : TParent::TBaseHandlersVisitor(settings, event) + , Deferred(deferred) + , CbContext(cbContext) { + } + +#define DECLARE_HANDLER(type, handler, answer) \ + bool operator()(type&) { \ + if (this->template PushHandler( \ + std::move(TParent::TBaseHandlersVisitor::Event), \ + this->Settings.EventHandlers_.handler, \ + this->Settings.EventHandlers_.CommonHandler_)) { \ + return answer; \ + } \ + return false; \ + } \ + /**/ + +#define DECLARE_TEMPLATE_HANDLER(type_true, type_false, handler_true, handler_false) \ + bool operator()(std::conditional_t&) { \ + if (this->template PushHandler>( \ + std::move(TParent::TBaseHandlersVisitor::Event), \ + [this](){ \ + if constexpr (UseMigrationProtocol) { \ + return this->Settings.EventHandlers_.handler_true; \ + } else { \ + return this->Settings.EventHandlers_.handler_false; \ + } \ + }(), \ + this->Settings.EventHandlers_.CommonHandler_)) { \ + return true; \ + } \ + return false; \ + } \ + /**/ + + DECLARE_TEMPLATE_HANDLER(typename TAReadSessionEvent::TDataReceivedEvent, + typename TAReadSessionEvent::TDataReceivedEvent, + DataReceivedHandler_, + DataReceivedHandler_); + DECLARE_TEMPLATE_HANDLER(typename TAReadSessionEvent::TCommitAcknowledgementEvent, + typename TAReadSessionEvent::TCommitOffsetAcknowledgementEvent, + CommitAcknowledgementHandler_, + CommitOffsetAcknowledgementHandler_); + DECLARE_TEMPLATE_HANDLER(typename TAReadSessionEvent::TCreatePartitionStreamEvent, + typename TAReadSessionEvent::TStartPartitionSessionEvent, + CreatePartitionStreamHandler_, + StartPartitionSessionHandler_); + DECLARE_TEMPLATE_HANDLER(typename TAReadSessionEvent::TDestroyPartitionStreamEvent, + typename TAReadSessionEvent::TStopPartitionSessionEvent, + DestroyPartitionStreamHandler_, + StopPartitionSessionHandler_); + DECLARE_TEMPLATE_HANDLER(typename TAReadSessionEvent::TPartitionStreamStatusEvent, + typename TAReadSessionEvent::TPartitionSessionStatusEvent, + PartitionStreamStatusHandler_, + PartitionSessionStatusHandler_); + DECLARE_HANDLER(TASessionClosedEvent, SessionClosedHandler_, false); // Not applied + +#undef DECLARE_HANDLER +#undef DECLARE_TEMPLATE_HANDLER + + bool operator()(std::conditional_t::TPartitionStreamClosedEvent, typename TAReadSessionEvent::TPartitionSessionClosedEvent>&) { + auto specific = [this]() { + if constexpr (UseMigrationProtocol) { + return this->Settings.EventHandlers_.PartitionStreamClosedHandler_; + } else { + return this->Settings.EventHandlers_.PartitionSessionClosedHandler_; + } + }(); + + if (!specific && !this->Settings.EventHandlers_.CommonHandler_) { + return false; + } + + this->template PushCommonHandler<>( + std::move(TParent::TBaseHandlersVisitor::Event), + [specific = specific, + common = this->Settings.EventHandlers_.CommonHandler_, + cbContext = CbContext](auto& event) { + auto& e = std::get::TPartitionStreamClosedEvent, typename TAReadSessionEvent::TPartitionSessionClosedEvent>>(event); + if (specific) { + specific(e); + } else if (common) { + common(event); + } + if constexpr (!UseMigrationProtocol) { + if (auto session = cbContext->LockShared()) { + session->UnregisterPartition(e.GetPartitionSession()->GetPartitionId(), e.GetPartitionSession()->GetPartitionSessionId()); + } + } + }); + + return true; + } + + template + constexpr std::enable_if_t + operator()(typename TAReadSessionEvent::TEndPartitionSessionEvent&) { + if (!this->Settings.EventHandlers_.EndPartitionSessionHandler_ && !this->Settings.EventHandlers_.CommonHandler_) { + return false; + } + this->template PushCommonHandler<>( + std::move(TParent::TBaseHandlersVisitor::Event), + [specific = this->Settings.EventHandlers_.EndPartitionSessionHandler_, + common = this->Settings.EventHandlers_.CommonHandler_, + cbContext = CbContext](TReadSessionEvent::TEvent& event) { + auto& e = std::get(event); + if (specific) { + specific(e); + } else if (common) { + common(event); + } + }); + return true; + } + + bool Visit() { + return std::visit(*this, TParent::TBaseHandlersVisitor::Event); + } + + void Post(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& f) override { + Deferred.DeferStartExecutorTask(executor, std::move(f)); + } + + TDeferredActions& Deferred; + TCallbackContextPtr CbContext; + }; + + TADataReceivedEvent + GetDataEventImpl(TIntrusivePtr> stream, + size_t& maxByteSize, + TUserRetrievedEventsInfoAccumulator& accumulator); // Assumes that we're under lock. + + bool ApplyHandler(TReadSessionEventInfo& eventInfo, TDeferredActions& deferred) { + THandlersVisitor visitor(this->Settings, eventInfo.GetEvent(), deferred, CbContext); + return visitor.Visit(); + } + +private: + bool HasEventCallbacks; + TCallbackContextPtr CbContext; +}; + +} // namespace NYdb::NTopic + +template <> +struct THash::TKey> { + size_t operator()(const NYdb::NTopic::TPartitionStreamImpl::TKey& key) const { + THash strHash; + const size_t h1 = strHash(key.Topic); + const size_t h2 = NumericHash(key.Partition); + return CombineHashes(h1, h2); + } +}; + +template <> +struct THash::TKey> { + size_t operator()(const NYdb::NTopic::TPartitionStreamImpl::TKey& key) const { + THash strHash; + const size_t h1 = strHash(key.Topic); + const size_t h2 = strHash(key.Cluster); + const size_t h3 = NumericHash(key.Partition); + return CombineHashes(h1, CombineHashes(h2, h3)); + } +}; + +namespace NYdb::NTopic { + +// Read session for single cluster. +// This class holds only read session logic. +// It is parametrized with output queue for client events +// and connection factory interface to separate logic from transport. +template +class TSingleClusterReadSessionImpl : public TEnableSelfContext>, + public IUserRetrievedEventCallback { +public: + using TSelf = TSingleClusterReadSessionImpl; + using TPtr = std::shared_ptr; + using IProcessor = typename IReadSessionConnectionProcessorFactory::IProcessor; + + + friend class TPartitionStreamImpl; + + TSingleClusterReadSessionImpl( + const TAReadSessionSettings& settings, + const std::string& database, + const std::string& sessionId, + const std::string& clusterName, + const TLog& log, + std::shared_ptr> connectionFactory, + std::shared_ptr> eventsQueue, + NYdbGrpc::IQueueClientContextPtr clientContext, + ui64 partitionStreamIdStart, + ui64 partitionStreamIdStep + ) + : Settings(settings) + , Database(database) + , SessionId(sessionId) + , ClusterName(clusterName) + , Log(log) + , NextPartitionStreamId(partitionStreamIdStart) + , PartitionStreamIdStep(partitionStreamIdStep) + , ConnectionFactory(std::move(connectionFactory)) + , EventsQueue(std::move(eventsQueue)) + , ClientContext(std::move(clientContext)) + , CookieMapping() + , ReadSizeBudget(GetCompressedDataSizeLimit()) + , ReadSizeServerDelta(0) + { + } + + ~TSingleClusterReadSessionImpl(); + + void Start(); + void ConfirmPartitionStreamCreate(const TPartitionStreamImpl* partitionStream, std::optional readOffset, std::optional commitOffset); + void ConfirmPartitionStreamDestroy(TPartitionStreamImpl* partitionStream); + void ConfirmPartitionStreamEnd(TPartitionStreamImpl* partitionStream, const std::vector& childIds); + void RequestPartitionStreamStatus(const TPartitionStreamImpl* partitionStream); + void Commit(const TPartitionStreamImpl* partitionStream, ui64 startOffset, ui64 endOffset); + + void OnCreateNewDecompressionTask(); + void OnDecompressionInfoDestroy(i64 compressedSize, i64 decompressedSize, i64 messagesCount, i64 serverBytesSize); + void OnDecompressionInfoDestroyImpl(i64 compressedSize, + i64 decompressedSize, + i64 messagesCount, + i64 serverBytesSize, + TDeferredActions& deferred); + + void OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount, i64 serverBytesSize = 0); + + TReadSessionEventsQueue* GetEventsQueue() { + return EventsQueue.get(); + } + + void OnUserRetrievedEvent(i64 decompressedSize, size_t messagesCount) override; + + void Abort(); + void AbortImpl(); + void Close(std::function callback); + void AbortSession(TASessionClosedEvent&& closeEvent); + + void AbortSession(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + AbortSession(TASessionClosedEvent(statusCode, std::move(issues))); + } + + void AbortSession(EStatus statusCode, const std::string& message) { + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + AbortSession(statusCode, std::move(issues)); + } + + void AbortSession(TPlainStatus&& status) { + AbortSession(TASessionClosedEvent(std::move(status))); + } + + bool Reconnect(const TPlainStatus& status); + + void StopReadingData(); + void ResumeReadingData(); + + void DumpStatisticsToLog(TLogElement& log); + void UpdateMemoryUsageStatistics(); + + TStringBuilder GetLogPrefix() const; + + const TLog& GetLog() const { + return Log; + } + + void RegisterParentPartition(ui32 partitionId, ui32 parentPartitionId, ui64 parentPartitionSessionId); + void UnregisterPartition(ui32 partitionId, ui64 partitionSessionId); + std::vector GetParentPartitionSessions(ui32 partitionId, ui64 partitionSessionId); + bool AllParentSessionsHasBeenRead(ui32 partitionId, ui64 partitionSessionId); + + void SetSelfContext(TPtr ptr) { + TEnableSelfContext>::SetSelfContext(std::move(ptr)); + EventsQueue->SetCallbackContext(TEnableSelfContext>::SelfContext); + } + + void CollectOffsets(NTable::TTransaction& tx, + const std::vector& events, + std::shared_ptr client); + void CollectOffsets(NTable::TTransaction& tx, + const TReadSessionEvent::TEvent& event, + std::shared_ptr client); + +private: + void BreakConnectionAndReconnectImpl(TPlainStatus&& status, TDeferredActions& deferred); + + void BreakConnectionAndReconnectImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { + BreakConnectionAndReconnectImpl(TPlainStatus(statusCode, std::move(issues)), deferred); + } + + void BreakConnectionAndReconnectImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred) { + BreakConnectionAndReconnectImpl(TPlainStatus(statusCode, message), deferred); + } + + bool HasCommitsInflightImpl() const; + + void OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext); + void OnConnect(TPlainStatus&&, typename IProcessor::TPtr&&, const NYdbGrpc::IQueueClientContextPtr& connectContext); + void DestroyAllPartitionStreamsImpl(TDeferredActions& deferred); // Destroy all streams before setting new connection // Assumes that we're under lock. + + // Initing. + inline void InitImpl(TDeferredActions& deferred); // Assumes that we're under lock. + + // Working logic. + void ContinueReadingDataImpl(); // Assumes that we're under lock. + bool IsActualPartitionStreamImpl(const TPartitionStreamImpl* partitionStream); // Assumes that we're under lock. + + // Read/Write. + void ReadFromProcessorImpl(TDeferredActions& deferred); // Assumes that we're under lock. + void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock. + void OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration); + + // Assumes that we're under lock. + template + inline void OnReadDoneImpl(TMessage&& msg, TDeferredActions& deferred); + + void StartDecompressionTasksImpl(TDeferredActions& deferred); // Assumes that we're under lock. + + i64 GetCompressedDataSizeLimit() const { + const double overallLimit = static_cast(Settings.MaxMemoryUsageBytes_); + // CompressedDataSize + CompressedDataSize * AverageCompressionRatio <= Settings.MaxMemoryUsageBytes_ + return Max(1l, static_cast(overallLimit / (1.0 + AverageCompressionRatio))); + } + + i64 GetDecompressedDataSizeLimit() const { + return Max(1l, static_cast(Settings.MaxMemoryUsageBytes_) - GetCompressedDataSizeLimit()); + } + + bool GetRangesMode() const; + + void CallCloseCallbackImpl(); + + void UpdateMemoryUsageStatisticsImpl(); + +private: + struct TPartitionCookieMapping { + struct TCookie : public TThrRefBase { + struct TKey { + const ui64 AssignId; + const ui64 CookieId; + + TKey(ui64 assignId, ui64 cookieId) + : AssignId(assignId) + , CookieId(cookieId) + { + } + + bool operator==(const TKey& k) const { + return AssignId == k.AssignId && CookieId == k.CookieId; + } + + struct THash { + size_t operator()(const TKey& k) const { + ::THash> h; + return h(std::make_pair(k.AssignId, k.CookieId)); + } + }; + + }; + + using TPtr = TIntrusivePtr; + + explicit TCookie(ui64 cookie, TIntrusivePtr> partitionStream) + : Cookie(cookie) + , PartitionStream(std::move(partitionStream)) + { + } + + // Sets reverse mapping for max offset in this cookie. + void SetOffsetRange(const std::pair& offsetRange) { + OffsetRange = offsetRange; + UncommittedMessagesLeft = offsetRange.second - offsetRange.first; + } + + TKey GetKey() const { + return TKey(PartitionStream->GetAssignId(), Cookie); + } + + ui64 Cookie = 0; + TIntrusivePtr> PartitionStream; + std::pair OffsetRange; + size_t UncommittedMessagesLeft = 0; + }; + + explicit TPartitionCookieMapping() + { + } + + bool AddMapping(const typename TCookie::TPtr& cookie); + + // Removes (partition stream, offset) from mapping. + // Returns cookie ptr if this was the last message, otherwise nullptr. + typename TSingleClusterReadSessionImpl::TPartitionCookieMapping::TCookie::TPtr CommitOffset(ui64 partitionStreamId, ui64 offset); + + // Gets and then removes committed cookie from mapping. + typename TSingleClusterReadSessionImpl::TPartitionCookieMapping::TCookie::TPtr RetrieveCommittedCookie(const Ydb::PersQueue::V1::CommitCookie& cookieProto); + + // Removes mapping on partition stream. + void RemoveMapping(ui64 partitionStreamId); + + // Clear all mapping before reconnect. + void ClearMapping(); + + bool HasUnacknowledgedCookies() const; + + private: + std::unordered_map Cookies; + std::unordered_map, typename TCookie::TPtr, THash>> UncommittedOffsetToCookie; // (Partition stream id, Offset) -> Cookie. + std::unordered_multimap PartitionStreamIdToCookie; + size_t CommitInflight = 0; // Commit inflight to server. + }; + + struct TDecompressionQueueItem { + TDecompressionQueueItem(TDataDecompressionInfoPtr batchInfo, + TIntrusivePtr> partitionStream) + : BatchInfo(std::move(batchInfo)) + , PartitionStream(std::move(partitionStream)) + { + } + + void OnDestroyReadSession(); + + TDataDecompressionInfoPtr BatchInfo; + TIntrusivePtr> PartitionStream; + }; + +private: + struct TTransactionInfo { + TSpinLock Lock; + bool IsActive = false; + bool Subscribed = false; + bool CommitCalled = false; + TOffsetsCollector OffsetsCollector; + }; + + using TTransactionInfoPtr = std::shared_ptr; + using TTransactionMap = std::unordered_map>; + + void TrySubscribeOnTransactionCommit(NTable::TTransaction& tx, + std::shared_ptr client); + TTransactionInfoPtr GetOrCreateTxInfo(const TTransactionId& txId); + void DeleteTx(const TTransactionId& txId); + + const TAReadSessionSettings Settings; + const std::string Database; + const std::string SessionId; + const std::string ClusterName; + TLog Log; + ui64 NextPartitionStreamId; + ui64 PartitionStreamIdStep; + std::shared_ptr> ConnectionFactory; + std::shared_ptr> EventsQueue; + NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. + NYdbGrpc::IQueueClientContextPtr ConnectContext; + NYdbGrpc::IQueueClientContextPtr ConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr ConnectDelayContext; + size_t ConnectionGeneration = 0; + TAdaptiveLock Lock; + typename IProcessor::TPtr Processor; + typename IARetryPolicy::IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). + size_t ConnectionAttemptsDone = 0; + + // Memory usage. + i64 CompressedDataSize = 0; + i64 DecompressedDataSize = 0; + double AverageCompressionRatio = 1.0; // Weighted average for compression memory usage estimate. + TInstant UsageStatisticsLastUpdateTime = TInstant::Now(); + + bool WaitingReadResponse = false; + std::shared_ptr> ServerMessage; // Server message to write server response to. + std::unordered_map>> PartitionStreams; // assignId -> Partition stream. + TPartitionCookieMapping CookieMapping; + std::deque DecompressionQueue; + bool DataReadingSuspended = false; + + // Exiting. + bool Aborting = false; + bool Closing = false; + std::function CloseCallback; + std::atomic DecompressionTasksInflight = 0; + i64 ReadSizeBudget; + i64 ReadSizeServerDelta = 0; + + struct TParentInfo { + ui32 PartitionId; + ui64 PartitionSessionId; + }; + + std::unordered_map> HierarchyData; + std::unordered_set ReadingFinishedData; + + TTransactionMap Txs; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.ipp b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.ipp new file mode 100644 index 000000000000..bf2b2e770954 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/read_session_impl.ipp @@ -0,0 +1,3014 @@ +#pragma once + +#define INCLUDE_READ_SESSION_IMPL_H +#include "read_session_impl.h" +#undef INCLUDE_READ_SESSION_IMPL_H + +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + + +namespace NYdb::inline V3::NTopic { + +static const bool RangesMode = !std::string{std::getenv("PQ_OFFSET_RANGES_MODE") ? std::getenv("PQ_OFFSET_RANGES_MODE") : ""}.empty(); + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TPartitionStreamImpl + +template +TLog TPartitionStreamImpl::GetLog() const { + if (auto session = CbContext->LockShared()) { + return session->GetLog(); + } + return {}; +} + +template +void TPartitionStreamImpl::Commit(ui64 startOffset, ui64 endOffset) { + std::vector> toCommit; + if (auto sessionShared = CbContext->LockShared()) { + Y_ABORT_UNLESS(endOffset > startOffset); + { + std::lock_guard guard(sessionShared->Lock); + if (!AddToCommitRanges(startOffset, endOffset, true)) // Add range for real commit always. + return; + + Y_ABORT_UNLESS(!Commits.Empty()); + for (auto c : Commits) { + if (c.first >= endOffset) break; // Commit only gaps before client range. + toCommit.emplace_back(c); + } + Commits.EraseInterval(0, endOffset); // Drop only committed ranges; + } + for (auto range: toCommit) { + sessionShared->Commit(this, range.first, Min(range.second, endOffset)); + } + } +} + +template +void TPartitionStreamImpl::RequestStatus() { + if (auto sessionShared = CbContext->LockShared()) { + sessionShared->RequestPartitionStreamStatus(this); + } +} + +template +void TPartitionStreamImpl::ConfirmCreate(std::optional readOffset, std::optional commitOffset) { + if (auto sessionShared = CbContext->LockShared()) { + sessionShared->ConfirmPartitionStreamCreate(this, readOffset, commitOffset); + } +} + +template +void TPartitionStreamImpl::ConfirmDestroy() { + if (auto sessionShared = CbContext->LockShared()) { + sessionShared->ConfirmPartitionStreamDestroy(this); + } +} + +template +void TPartitionStreamImpl::ConfirmEnd(const std::vector& childIds) { + if (auto sessionShared = CbContext->LockShared()) { + sessionShared->ConfirmPartitionStreamEnd(this, childIds); + } +} + +template +void TPartitionStreamImpl::StopReading() { + Y_ABORT("Not implemented"); // TODO +} + +template +void TPartitionStreamImpl::ResumeReading() { + Y_ABORT("Not implemented"); // TODO +} + +template +void TPartitionStreamImpl::SignalReadyEvents(TIntrusivePtr> stream, + TReadSessionEventsQueue* queue, + TDeferredActions& deferred) +{ + Y_ABORT_UNLESS(queue); + + stream->EventsQueue.SignalReadyEvents(stream, *queue, deferred); +} + +template +void TPartitionStreamImpl::DeleteNotReadyTail(TDeferredActions& deferred) +{ + EventsQueue.DeleteNotReadyTail(deferred); +} + +template +void TPartitionStreamImpl::GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator) +{ + partitionStream->EventsQueue.GetDataEventImpl(partitionStream, + maxEventsCount, + maxByteSize, + messages, + compressedMessages, + accumulator); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TRawPartitionStreamEventQueue + +template +void TRawPartitionStreamEventQueue::SignalReadyEvents(TIntrusivePtr> stream, + TReadSessionEventsQueue& queue, + TDeferredActions& deferred) +{ + if constexpr (!UseMigrationProtocol) { + if (auto session = CbContext->LockShared()) { + if (!session->AllParentSessionsHasBeenRead(stream->GetPartitionId(), stream->GetPartitionSessionId())) { + return; + } + } + } + + auto moveToReadyQueue = [&](TRawPartitionStreamEvent &&event) { + queue.SignalEventImpl(stream, deferred, event.IsDataEvent()); + + Ready.push_back(std::move(event)); + NotReady.pop_front(); + }; + + while (!NotReady.empty() && NotReady.front().IsReady()) { + auto& front = NotReady.front(); + + if (front.IsDataEvent()) { + if (queue.HasDataEventCallback()) { + std::vector::TMessage> messages; + std::vector::TCompressedMessage> compressedMessages; + TUserRetrievedEventsInfoAccumulator accumulator; + auto maxEventsCount = Max(); + auto maxByteSize = Max(); + + queue.GetDataEventCallbackSettings(maxByteSize); + + TRawPartitionStreamEventQueue::GetDataEventImpl(stream, + maxEventsCount, + maxByteSize, + messages, + compressedMessages, + accumulator, + NotReady); + + TADataReceivedEvent data(std::move(messages), + std::move(compressedMessages), + stream); + + queue.ApplyCallbackToEventImpl(data, std::move(accumulator), deferred); + } else { + moveToReadyQueue(std::move(front)); + } + } else { + if (queue.TryApplyCallbackToEventImpl(front.GetEvent(), deferred, CbContext)) { + NotReady.pop_front(); + } else { + moveToReadyQueue(std::move(front)); + } + } + } +} + +template +void TRawPartitionStreamEventQueue::DeleteNotReadyTail(TDeferredActions& deferred) +{ + std::deque> ready; + + auto i = NotReady.begin(); + for (; (i != NotReady.end()) && i->IsReady(); ++i) { + ready.push_back(std::move(*i)); + } + + std::vector> infos; + + for (; i != NotReady.end(); ++i) { + if (i->IsDataEvent()) { + infos.push_back(i->GetDataEvent().GetParent()); + } + } + + deferred.DeferDestroyDecompressionInfos(std::move(infos)); + + swap(ready, NotReady); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TDecompressionQueueItem + +template +void TSingleClusterReadSessionImpl::TDecompressionQueueItem::OnDestroyReadSession() +{ + BatchInfo->OnDestroyReadSession(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TSingleClusterReadSessionImpl + +template +TSingleClusterReadSessionImpl::~TSingleClusterReadSessionImpl() { + for (auto&& [_, partitionStream] : PartitionStreams) { + partitionStream->ClearQueue(); + } + + for (auto& e : DecompressionQueue) { + e.OnDestroyReadSession(); + } +} + +template +TStringBuilder TSingleClusterReadSessionImpl::GetLogPrefix() const { + return TStringBuilder() << GetDatabaseLogPrefix(Database) << "[" << SessionId << "] [" << ClusterName << "] "; +} + +template +void TSingleClusterReadSessionImpl::Start() { + Y_ABORT_UNLESS(this->SelfContext); + Settings.DecompressionExecutor_->Start(); + Settings.EventHandlers_.HandlersExecutor_->Start(); + if (!Reconnect(TPlainStatus())) { + AbortSession(EStatus::ABORTED, "Driver is stopping"); + } +} + +template +bool TSingleClusterReadSessionImpl::Reconnect(const TPlainStatus& status) { + TDuration delay = TDuration::Zero(); + + // Previous operations contexts. + NYdbGrpc::IQueueClientContextPtr prevConnectContext; + NYdbGrpc::IQueueClientContextPtr prevConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr prevConnectDelayContext; + + // Callbacks + std::function connectCallback; + std::function connectTimeoutCallback; + + if (!status.Ok()) { + LOG_LAZY(Log, TLOG_ERR, GetLogPrefix() << "Got error. Status: " << status.Status + << ". Description: " << IssuesSingleLineString(status.Issues)); + } + + NYdbGrpc::IQueueClientContextPtr delayContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr; + + TDeferredActions deferred; + { + std::lock_guard guard(Lock); + connectContext = ClientContext->CreateContext(); + connectTimeoutContext = ClientContext->CreateContext(); + if (!connectContext || !connectTimeoutContext) { + return false; + } + + if (Aborting) { + Cancel(connectContext); + Cancel(connectTimeoutContext); + return false; + } + if (Processor) { + Processor->Cancel(); + } + Processor = nullptr; + WaitingReadResponse = false; + ServerMessage = std::make_shared>(); + ++ConnectionGeneration; + + LOG_LAZY(Log, TLOG_DEBUG, + GetLogPrefix() << "In Reconnect, ReadSizeBudget = " << ReadSizeBudget + << ", ReadSizeServerDelta = " << ReadSizeServerDelta); + + ReadSizeBudget += ReadSizeServerDelta; + ReadSizeServerDelta = 0; + + LOG_LAZY(Log, TLOG_DEBUG, + GetLogPrefix() << "New values: ReadSizeBudget = " << ReadSizeBudget + << ", ReadSizeServerDelta = " << ReadSizeServerDelta); + + if (!RetryState) { + RetryState = Settings.RetryPolicy_->CreateRetryState(); + } + if (!status.Ok()) { + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); + + if (!nextDelay) { + return false; + } + delay = *nextDelay; + delayContext = ClientContext->CreateContext(); + if (!delayContext) { + return false; + } + } + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Reconnecting session to cluster " << ClusterName << " in " << delay); + + ++ConnectionAttemptsDone; + + // Set new context + prevConnectContext = std::exchange(ConnectContext, connectContext); + prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); + prevConnectDelayContext = std::exchange(ConnectDelayContext, delayContext); + + Y_ASSERT(ConnectContext); + Y_ASSERT(ConnectTimeoutContext); + Y_ASSERT((delay == TDuration::Zero()) == !ConnectDelayContext); + + // Destroy all partition streams before connecting. + DestroyAllPartitionStreamsImpl(deferred); + + Y_ABORT_UNLESS(this->SelfContext); + + connectCallback = [cbContext = this->SelfContext, + connectContext = connectContext](TPlainStatus&& st, typename IProcessor::TPtr&& processor) { + if (auto borrowedSelf = cbContext->LockShared()) { + borrowedSelf->OnConnect(std::move(st), std::move(processor), connectContext); // OnConnect could be called inplace! + } + }; + + connectTimeoutCallback = [cbContext = this->SelfContext, + connectTimeoutContext = connectTimeoutContext](bool ok) { + if (ok) { + if (auto borrowedSelf = cbContext->LockShared()) { + borrowedSelf->OnConnectTimeout(connectTimeoutContext); + } + } + }; + } + + // Cancel previous operations. + Cancel(prevConnectContext); + Cancel(prevConnectTimeoutContext); + Cancel(prevConnectDelayContext); + + Y_ASSERT(connectContext); + Y_ASSERT(connectTimeoutContext); + Y_ASSERT((delay == TDuration::Zero()) == !delayContext); + ConnectionFactory->CreateProcessor( + std::move(connectCallback), + TRpcRequestSettings::Make(Settings), + std::move(connectContext), + TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting. + std::move(connectTimeoutContext), + std::move(connectTimeoutCallback), + delay, + std::move(delayContext)); + return true; +} + +template +void TSingleClusterReadSessionImpl::BreakConnectionAndReconnectImpl( + TPlainStatus&& status, TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + LOG_LAZY(Log, TLOG_INFO, + GetLogPrefix() << "Break connection due to unexpected message from server. Status: " << status.Status + << ", Issues: \"" << IssuesSingleLineString(status.Issues) << "\""); + + Processor->Cancel(); + Processor = nullptr; + RetryState = Settings.RetryPolicy_->CreateRetryState(); // Explicitly create retry state to determine whether we should connect to server again. + + deferred.DeferReconnection(this->SelfContext, std::move(status)); +} + +template +void TSingleClusterReadSessionImpl::OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext) { + { + std::lock_guard guard(Lock); + if (ConnectTimeoutContext == connectTimeoutContext) { + Cancel(ConnectContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + + if (Closing || Aborting) { + CallCloseCallbackImpl(); + return; + } + } else { + return; + } + } + + ++*Settings.Counters_->Errors; + TStringBuilder description; + description << "Failed to establish connection to server. Attempts done: " << ConnectionAttemptsDone; + if (!Reconnect(TPlainStatus(EStatus::TIMEOUT, description))) { + AbortSession(EStatus::TIMEOUT, description); + } +} + +template +void TSingleClusterReadSessionImpl::OnConnect( + TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NYdbGrpc::IQueueClientContextPtr& connectContext) { + TDeferredActions deferred; + { + std::lock_guard guard(Lock); + if (ConnectContext == connectContext) { + Cancel(ConnectTimeoutContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + + if (Closing || Aborting) { + CallCloseCallbackImpl(); + return; + } + + if (st.Ok()) { + Processor = std::move(processor); + ConnectionAttemptsDone = 0; + InitImpl(deferred); + return; + } + } else { + return; + } + } + + if (!st.Ok()) { + ++*Settings.Counters_->Errors; + if (!Reconnect(st)) { + AbortSession( + st.Status, MakeIssueWithSubIssues(TStringBuilder() << "Failed to establish connection to server \"" + << st.Endpoint << "\" ( cluster " << ClusterName + << "). Attempts done: " << ConnectionAttemptsDone, + st.Issues)); + } + } +} + +template<> +inline void TSingleClusterReadSessionImpl::InitImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Successfully connected. Initializing session"); + TClientMessage req; + auto& init = *req.mutable_init_request(); + init.set_ranges_mode(GetRangesMode()); + for (const NPersQueue::TTopicReadSettings& topic : Settings.Topics_) { + auto* topicSettings = init.add_topics_read_settings(); + topicSettings->set_topic(TStringType{topic.Path_}); + if (topic.StartingMessageTimestamp_) { + topicSettings->set_start_from_written_at_ms(topic.StartingMessageTimestamp_->MilliSeconds()); + } + for (ui64 groupId : topic.PartitionGroupIds_) { + topicSettings->add_partition_group_ids(groupId); + } + } + init.set_consumer(TStringType{Settings.ConsumerName_}); + init.set_read_only_original(Settings.ReadOnlyOriginal_); + init.mutable_read_params()->set_max_read_size(Settings.MaxMemoryUsageBytes_); + if (Settings.MaxTimeLag_) { + init.set_max_lag_duration_ms(Settings.MaxTimeLag_->MilliSeconds()); + } + if (Settings.StartingMessageTimestamp_) { + init.set_start_from_written_at_ms(Settings.StartingMessageTimestamp_->MilliSeconds()); + } + + WriteToProcessorImpl(std::move(req)); + ReadFromProcessorImpl(deferred); +} + +template<> +inline void TSingleClusterReadSessionImpl::InitImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Successfully connected. Initializing session"); + TClientMessage req; + auto& init = *req.mutable_init_request(); + + init.set_consumer(TStringType{Settings.ConsumerName_}); + init.set_auto_partitioning_support(Settings.AutoPartitioningSupport_); + + for (const TTopicReadSettings& topic : Settings.Topics_) { + auto* topicSettings = init.add_topics_read_settings(); + topicSettings->set_path(TStringType{topic.Path_}); + for (ui64 partitionId : topic.PartitionIds_) { + topicSettings->add_partition_ids(partitionId); + } + + if (topic.ReadFromTimestamp_) { + *topicSettings->mutable_read_from() = + ::google::protobuf::util::TimeUtil::MillisecondsToTimestamp(topic.ReadFromTimestamp_->MilliSeconds()); + } else if (Settings.ReadFromTimestamp_) { + *topicSettings->mutable_read_from() = + ::google::protobuf::util::TimeUtil::MillisecondsToTimestamp(Settings.ReadFromTimestamp_->MilliSeconds()); + } + + if (topic.MaxLag_) { + *topicSettings->mutable_max_lag() = + ::google::protobuf::util::TimeUtil::MillisecondsToDuration(topic.MaxLag_->MilliSeconds()); + } else if (Settings.MaxLag_) { + *topicSettings->mutable_max_lag() = + ::google::protobuf::util::TimeUtil::MillisecondsToDuration(Settings.MaxLag_->MilliSeconds()); + } + } + + WriteToProcessorImpl(std::move(req)); + ReadFromProcessorImpl(deferred); +} + +template +void TSingleClusterReadSessionImpl::ContinueReadingDataImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Closing + && !Aborting + && !WaitingReadResponse + && !DataReadingSuspended + && Processor + && CompressedDataSize < GetCompressedDataSizeLimit() + && static_cast(CompressedDataSize + DecompressedDataSize) < Settings.MaxMemoryUsageBytes_) + { + TClientMessage req; + if constexpr (UseMigrationProtocol) { + req.mutable_read(); + } else { + LOG_LAZY(Log, TLOG_DEBUG, + GetLogPrefix() << "In ContinueReadingDataImpl, ReadSizeBudget = " << ReadSizeBudget + << ", ReadSizeServerDelta = " << ReadSizeServerDelta); + + if (ReadSizeBudget <= 0 || ReadSizeServerDelta + ReadSizeBudget <= 0) { + return; + } + req.mutable_read_request()->set_bytes_size(ReadSizeBudget); + ReadSizeServerDelta += ReadSizeBudget; + ReadSizeBudget = 0; + } + + WriteToProcessorImpl(std::move(req)); + LOG_LAZY(Log, TLOG_DEBUG, + GetLogPrefix() << "After sending read request: ReadSizeBudget = " << ReadSizeBudget + << ", ReadSizeServerDelta = " << ReadSizeServerDelta); + WaitingReadResponse = true; + } +} + +template +ui64 GetPartitionStreamId(const TPartitionStreamImpl* partitionStream) { + if constexpr (UseMigrationProtocol) { + return partitionStream->GetPartitionStreamId(); + } else { + return partitionStream->GetPartitionSessionId(); + } +} + +template +std::string GetCluster(const TPartitionStreamImpl* partitionStream) { + if constexpr (UseMigrationProtocol) { + return partitionStream->GetCluster(); + } else { + return "-"; + } +} + +template +bool TSingleClusterReadSessionImpl::IsActualPartitionStreamImpl(const TPartitionStreamImpl* partitionStream) { + Y_ABORT_UNLESS(Lock.IsLocked()); + auto actualPartitionStreamIt = PartitionStreams.find(partitionStream->GetAssignId()); + return actualPartitionStreamIt != PartitionStreams.end() + && GetPartitionStreamId(actualPartitionStreamIt->second.Get()) == GetPartitionStreamId(partitionStream); +} + +template +void TSingleClusterReadSessionImpl::ConfirmPartitionStreamCreate(const TPartitionStreamImpl* partitionStream, std::optional readOffset, std::optional commitOffset) { + TStringBuilder commitOffsetLogStr; + if (commitOffset) { + commitOffsetLogStr << ". Commit offset: " << *commitOffset; + } + LOG_LAZY(Log, + TLOG_INFO, + GetLogPrefix() << "Confirm partition stream create. Partition stream id: " << GetPartitionStreamId(partitionStream) + << ". Cluster: \"" << GetCluster(partitionStream) << "\". Topic: \"" << partitionStream->GetTopicPath() + << "\". Partition: " << partitionStream->GetPartitionId() + << ". Read offset: " << readOffset << commitOffsetLogStr + ); + + std::lock_guard guard(Lock); + if (Aborting || Closing || !IsActualPartitionStreamImpl(partitionStream)) { // Got previous incarnation. + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Skip partition stream create confirm. Partition stream id: " + << GetPartitionStreamId(partitionStream) + ); + return; + } + + TClientMessage req; + + if constexpr (UseMigrationProtocol) { + auto& startRead = *req.mutable_start_read(); + startRead.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + startRead.set_cluster(TStringType{partitionStream->GetCluster()}); + startRead.set_partition(partitionStream->GetPartitionId()); + startRead.set_assign_id(partitionStream->GetAssignId()); + if (readOffset) { + startRead.set_read_offset(*readOffset); + } + if (commitOffset) { + startRead.set_commit_offset(*commitOffset); + } + } else { + auto& startRead = *req.mutable_start_partition_session_response(); + startRead.set_partition_session_id(partitionStream->GetAssignId()); + if (readOffset) { + startRead.set_read_offset(*readOffset); + } + if (commitOffset) { + startRead.set_commit_offset(*commitOffset); + } + } + + WriteToProcessorImpl(std::move(req)); +} + +template +void TSingleClusterReadSessionImpl::ConfirmPartitionStreamDestroy(TPartitionStreamImpl* partitionStream) { + LOG_LAZY(Log, + TLOG_INFO, + GetLogPrefix() << "Confirm partition stream destroy. Partition stream id: " + << GetPartitionStreamId(partitionStream) + << ". Cluster: \"" << GetCluster(partitionStream) << "\". Topic: \"" << partitionStream->GetTopicPath() + << "\". Partition: " << partitionStream->GetPartitionId() + ); + + TDeferredActions deferred; + std::lock_guard guard(Lock); + if (Aborting || Closing || !IsActualPartitionStreamImpl(partitionStream)) { // Got previous incarnation. + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Skip partition stream destroy confirm. Partition stream id: " + << GetPartitionStreamId(partitionStream) + ); + return; + } + + using TClosedEvent = std::conditional_t< + UseMigrationProtocol, + NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent, + NTopic::TReadSessionEvent::TPartitionSessionClosedEvent + >; + + CookieMapping.RemoveMapping(GetPartitionStreamId(partitionStream)); + PartitionStreams.erase(partitionStream->GetAssignId()); + + bool pushRes = true; + if constexpr (UseMigrationProtocol) { + pushRes = EventsQueue->PushEvent(partitionStream, + TClosedEvent(partitionStream, TClosedEvent::EReason::DestroyConfirmedByUser), + deferred); + } else { + pushRes = EventsQueue->PushEvent(partitionStream, + TClosedEvent(partitionStream, TClosedEvent::EReason::StopConfirmedByUser), + deferred); + } + if (!pushRes) { + AbortImpl(); + return; + } + TClientMessage req; + + if constexpr (UseMigrationProtocol) { + auto& released = *req.mutable_released(); + released.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + released.set_cluster(TStringType{partitionStream->GetCluster()}); + released.set_partition(partitionStream->GetPartitionId()); + released.set_assign_id(partitionStream->GetAssignId()); + } else { + auto& released = *req.mutable_stop_partition_session_response(); + released.set_partition_session_id(partitionStream->GetAssignId()); + } + + WriteToProcessorImpl(std::move(req)); +} + +template +void TSingleClusterReadSessionImpl::Commit(const TPartitionStreamImpl* partitionStream, ui64 startOffset, ui64 endOffset) { + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Commit offsets [" << startOffset << ", " << endOffset + << "). Partition stream id: " << GetPartitionStreamId(partitionStream) + ); + std::lock_guard guard(Lock); + if (Aborting || Closing || !IsActualPartitionStreamImpl(partitionStream)) { // Got previous incarnation. + return; + } + TClientMessage req; + bool hasSomethingToCommit = false; + + if constexpr (UseMigrationProtocol) { + if (GetRangesMode()) { + hasSomethingToCommit = true; + auto* range = req.mutable_commit()->add_offset_ranges(); + range->set_assign_id(partitionStream->GetAssignId()); + range->set_start_offset(startOffset); + range->set_end_offset(endOffset); + } else { + for (ui64 offset = startOffset; offset < endOffset; ++offset) { + typename TPartitionCookieMapping::TCookie::TPtr cookie = CookieMapping.CommitOffset(GetPartitionStreamId(partitionStream), offset); + if (cookie) { + hasSomethingToCommit = true; + auto* cookieInfo = req.mutable_commit()->add_cookies(); + cookieInfo->set_assign_id(partitionStream->GetAssignId()); + cookieInfo->set_partition_cookie(cookie->Cookie); + } + } + } + } else { + hasSomethingToCommit = true; + auto* part_commit = req.mutable_commit_offset_request()->add_commit_offsets(); + part_commit->set_partition_session_id(partitionStream->GetAssignId()); + auto* range = part_commit->add_offsets(); + range->set_start(startOffset); + range->set_end(endOffset); + } + + if (hasSomethingToCommit) { + WriteToProcessorImpl(std::move(req)); + } +} + +template +void TSingleClusterReadSessionImpl::RequestPartitionStreamStatus(const TPartitionStreamImpl* partitionStream) { + LOG_LAZY(Log, + TLOG_DEBUG, + GetLogPrefix() << "Requesting status for partition stream id: " << GetPartitionStreamId(partitionStream) + ); + std::lock_guard guard(Lock); + if (Aborting || Closing || !IsActualPartitionStreamImpl(partitionStream)) { // Got previous incarnation. + return; + } + + TClientMessage req; + + if constexpr (UseMigrationProtocol) { + auto& status = *req.mutable_status(); + status.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + status.set_cluster(TStringType{partitionStream->GetCluster()}); + status.set_partition(partitionStream->GetPartitionId()); + status.set_assign_id(partitionStream->GetAssignId()); + } else { + auto& status = *req.mutable_partition_session_status_request(); + status.set_partition_session_id(partitionStream->GetAssignId()); + } + + WriteToProcessorImpl(std::move(req)); +} + +template +void TSingleClusterReadSessionImpl::OnUserRetrievedEvent(i64 decompressedSize, size_t messagesCount) +{ + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() + << "The application data is transferred to the client. Number of messages " + << messagesCount + << ", size " + << decompressedSize + << " bytes"); + + *Settings.Counters_->MessagesInflight -= messagesCount; + *Settings.Counters_->BytesInflightTotal -= decompressedSize; + *Settings.Counters_->BytesInflightUncompressed -= decompressedSize; + + TDeferredActions deferred; + std::lock_guard guard(Lock); + UpdateMemoryUsageStatisticsImpl(); + + Y_ABORT_UNLESS(decompressedSize <= DecompressedDataSize); + DecompressedDataSize -= decompressedSize; + + ContinueReadingDataImpl(); + StartDecompressionTasksImpl(deferred); +} + +template +void TSingleClusterReadSessionImpl::WriteToProcessorImpl( + TClientMessage&& req) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (Processor) { + Processor->Write(std::move(req)); + } +} + +template +bool TSingleClusterReadSessionImpl::HasCommitsInflightImpl() const { + Y_ABORT_UNLESS(Lock.IsLocked()); + for (const auto& [id, partitionStream] : PartitionStreams) { + if (partitionStream->HasCommitsInflight()) + return true; + } + return false; +} + +template +void TSingleClusterReadSessionImpl::ReadFromProcessorImpl( + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (Aborting) { + return; + } + if (Closing && !HasCommitsInflightImpl()) { + Processor->Cancel(); + CallCloseCallbackImpl(); + return; + } + + if (Processor && !Closing) { + ServerMessage->Clear(); + + Y_ABORT_UNLESS(this->SelfContext); + + auto callback = [cbContext = this->SelfContext, + connectionGeneration = ConnectionGeneration, + // Capture message & processor not to read in freed memory. + serverMessage = ServerMessage, + processor = Processor](NYdbGrpc::TGrpcStatus&& grpcStatus) { + if (auto borrowedSelf = cbContext->LockShared()) { + borrowedSelf->OnReadDone(std::move(grpcStatus), connectionGeneration); + } + }; + + deferred.DeferReadFromProcessor(Processor, ServerMessage.get(), std::move(callback)); + } +} + +template +void TSingleClusterReadSessionImpl::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) { + TPlainStatus errorStatus; + if (!grpcStatus.Ok()) { + errorStatus = TPlainStatus(std::move(grpcStatus)); + } + + TDeferredActions deferred; + { + std::lock_guard guard(Lock); + if (Aborting) { + return; + } + + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (errorStatus.Ok()) { + if (IsErrorMessage(*ServerMessage)) { + errorStatus = MakeErrorFromProto(*ServerMessage); + } else { + + if constexpr (UseMigrationProtocol) { + switch (ServerMessage->response_case()) { + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kInitResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_init_response()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kDataBatch: + OnReadDoneImpl(std::move(*ServerMessage->mutable_data_batch()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kAssigned: + OnReadDoneImpl(std::move(*ServerMessage->mutable_assigned()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kRelease: + OnReadDoneImpl(std::move(*ServerMessage->mutable_release()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kCommitted: + OnReadDoneImpl(std::move(*ServerMessage->mutable_committed()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::kPartitionStatus: + OnReadDoneImpl(std::move(*ServerMessage->mutable_partition_status()), deferred); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::RESPONSE_NOT_SET: + errorStatus = TPlainStatus::Internal("Unexpected response from server"); + break; + } + } else { + switch (ServerMessage->server_message_case()) { + case TServerMessage::kInitResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_init_response()), deferred); + break; + case TServerMessage::kReadResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_read_response()), deferred); + break; + case TServerMessage::kStartPartitionSessionRequest: + OnReadDoneImpl(std::move(*ServerMessage->mutable_start_partition_session_request()), deferred); + break; + case TServerMessage::kUpdatePartitionSession: + OnReadDoneImpl(std::move(*ServerMessage->mutable_update_partition_session()), deferred); + break; + + case TServerMessage::kStopPartitionSessionRequest: + OnReadDoneImpl(std::move(*ServerMessage->mutable_stop_partition_session_request()), deferred); + break; + case TServerMessage::kEndPartitionSession: + OnReadDoneImpl(std::move(*ServerMessage->mutable_end_partition_session()), deferred); + break; + case TServerMessage::kCommitOffsetResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_commit_offset_response()), deferred); + break; + case TServerMessage::kPartitionSessionStatusResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_partition_session_status_response()), deferred); + break; + case TServerMessage::kUpdateTokenResponse: + OnReadDoneImpl(std::move(*ServerMessage->mutable_update_token_response()), deferred); + break; + case TServerMessage::SERVER_MESSAGE_NOT_SET: + errorStatus = TPlainStatus::Internal("Server message is not set"); + break; + default: + errorStatus = TPlainStatus::Internal("Unexpected response from server"); + break; + } + } + + if (errorStatus.Ok()) { + ReadFromProcessorImpl(deferred); // Read next. + } + } + } + } + if (!errorStatus.Ok()) { + ++*Settings.Counters_->Errors; + + if (!Reconnect(errorStatus)) { + AbortSession(std::move(errorStatus)); + } + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::InitResponse&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + Y_UNUSED(deferred); + + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Server session id: " << msg.session_id()); + + RetryState = nullptr; + + // Successful init. Do nothing. + ContinueReadingDataImpl(); +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + if (Closing || Aborting) { + return; // Don't process new data. + } + UpdateMemoryUsageStatisticsImpl(); + for (TPartitionData& partitionData : *msg.mutable_partition_data()) { + auto partitionStreamIt = PartitionStreams.find(partitionData.cookie().assign_id()); + if (partitionStreamIt == PartitionStreams.end()) { + ++*Settings.Counters_->Errors; + BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, + TStringBuilder() + << "Got unexpected partition stream data message. Topic: " + << partitionData.topic() << ". Partition: " << partitionData.partition() + << " AssignId: " << partitionData.cookie().assign_id(), + deferred); + return; + } + const TIntrusivePtr>& partitionStream = partitionStreamIt->second; + + typename TPartitionCookieMapping::TCookie::TPtr cookie = MakeIntrusive(partitionData.cookie().partition_cookie(), partitionStream); + + ui64 firstOffset = std::numeric_limits::max(); + ui64 currentOffset = std::numeric_limits::max(); + ui64 desiredOffset = partitionStream->GetFirstNotReadOffset(); + for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::Batch& batch : partitionData.batches()) { + // Validate messages. + for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::MessageData& messageData : batch.message_data()) { + // Check offsets continuity. + if (messageData.offset() != desiredOffset) { + bool res = partitionStream->AddToCommitRanges(desiredOffset, messageData.offset(), GetRangesMode()); + Y_ABORT_UNLESS(res); + } + + if (firstOffset == std::numeric_limits::max()) { + firstOffset = messageData.offset(); + } + currentOffset = messageData.offset(); + desiredOffset = currentOffset + 1; + partitionStream->UpdateMaxReadOffset(currentOffset); + const i64 messageSize = static_cast(messageData.data().size()); + CompressedDataSize += messageSize; + *Settings.Counters_->BytesInflightTotal += messageSize; + *Settings.Counters_->BytesInflightCompressed += messageSize; + ++*Settings.Counters_->MessagesInflight; + } + } + if (firstOffset == std::numeric_limits::max()) { + BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, + TStringBuilder() << "Got empty data message. Topic: " + << partitionData.topic() + << ". Partition: " << partitionData.partition() + << " message: " << msg, + deferred); + return; + } + cookie->SetOffsetRange(std::make_pair(firstOffset, desiredOffset)); + partitionStream->SetFirstNotReadOffset(desiredOffset); + if (!CookieMapping.AddMapping(cookie)) { + BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, + TStringBuilder() << "Got unexpected data message. Topic: " + << partitionData.topic() + << ". Partition: " << partitionData.partition() + << ". Cookie mapping already has such cookie", + deferred); + return; + } + + auto decompressionInfo = std::make_shared>(std::move(partitionData), + SelfContext, + Settings.Decompress_); + Y_ABORT_UNLESS(decompressionInfo); + + decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, + partitionStream); + + DecompressionQueue.emplace_back(decompressionInfo, partitionStream); + StartDecompressionTasksImpl(deferred); + } + + WaitingReadResponse = false; + ContinueReadingDataImpl(); +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::Assigned&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStream = MakeIntrusive>( + NextPartitionStreamId, msg.topic().path(), msg.cluster(), + msg.partition() + 1, // Group. + msg.partition(), // Partition. + msg.assign_id(), msg.read_offset(), SelfContext); + NextPartitionStreamId += PartitionStreamIdStep; + + // Renew partition stream. + TIntrusivePtr>& currentPartitionStream = + PartitionStreams[partitionStream->GetAssignId()]; + if (currentPartitionStream) { + CookieMapping.RemoveMapping(currentPartitionStream->GetPartitionStreamId()); + + bool pushRes = EventsQueue->PushEvent( + currentPartitionStream, + NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent( + currentPartitionStream, NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::Lost), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + currentPartitionStream = partitionStream; + + // Send event to user. + bool pushRes = EventsQueue->PushEvent( + partitionStream, + NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent(partitionStream, msg.read_offset(), msg.end_offset()), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::Release&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStreamIt = PartitionStreams.find(msg.assign_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + TIntrusivePtr> partitionStream = partitionStreamIt->second; + bool pushRes = true; + if (msg.forceful_release()) { + PartitionStreams.erase(msg.assign_id()); + CookieMapping.RemoveMapping(partitionStream->GetPartitionStreamId()); + pushRes = EventsQueue->PushEvent( + partitionStream, + NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent( + partitionStream, NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::Lost), + deferred); + } else { + pushRes = EventsQueue->PushEvent( + partitionStream, + NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent(std::move(partitionStream), msg.commit_offset()), + deferred); + } + + if (!pushRes) { + AbortImpl(); + return; + } + +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::Committed&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Committed response: " << msg); + + std::map>> partitionStreams; + for (const Ydb::PersQueue::V1::CommitCookie& cookieProto : msg.cookies()) { + typename TPartitionCookieMapping::TCookie::TPtr cookie = CookieMapping.RetrieveCommittedCookie(cookieProto); + if (cookie) { + cookie->PartitionStream->UpdateMaxCommittedOffset(cookie->OffsetRange.second); + partitionStreams[cookie->PartitionStream->GetPartitionStreamId()] = cookie->PartitionStream; + } + } + for (auto& [id, partitionStream] : partitionStreams) { + bool pushRes = EventsQueue->PushEvent(partitionStream, + NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent( + partitionStream, partitionStream->GetMaxCommittedOffset()), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + + for (const auto& rangeProto : msg.offset_ranges()) { + auto partitionStreamIt = PartitionStreams.find(rangeProto.assign_id()); + if (partitionStreamIt != PartitionStreams.end()) { + auto partitionStream = partitionStreamIt->second; + partitionStream->UpdateMaxCommittedOffset(rangeProto.end_offset()); + bool pushRes = EventsQueue->PushEvent( + partitionStream, + NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent(partitionStream, rangeProto.end_offset()), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::PartitionStatus&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStreamIt = PartitionStreams.find(msg.assign_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + bool pushRes = EventsQueue->PushEvent(partitionStreamIt->second, + NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent( + partitionStreamIt->second, msg.committed_offset(), + 0, // TODO: support read offset in status + msg.end_offset(), TInstant::MilliSeconds(msg.write_watermark_ms())), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::InitResponse&& msg, + TDeferredActions& deferred) { + + Y_ABORT_UNLESS(Lock.IsLocked()); + Y_UNUSED(deferred); + + RetryState = nullptr; + + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Server session id: " << msg.session_id()); + + // Successful init. Do nothing. + ContinueReadingDataImpl(); +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::ReadResponse&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (Closing || Aborting) { + return; // Don't process new data. + } + + i64 serverBytesSize = msg.bytes_size(); + ReadSizeServerDelta -= serverBytesSize; + LOG_LAZY(Log, TLOG_DEBUG, + GetLogPrefix() << "Got ReadResponse, serverBytesSize = " << serverBytesSize << ", now ReadSizeBudget = " + << ReadSizeBudget << ", ReadSizeServerDelta = " << ReadSizeServerDelta); + + UpdateMemoryUsageStatisticsImpl(); + for (TPartitionData& partitionData : *msg.mutable_partition_data()) { + auto partitionStreamIt = PartitionStreams.find(partitionData.partition_session_id()); + if (partitionStreamIt == PartitionStreams.end()) { + ++*Settings.Counters_->Errors; + BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, + TStringBuilder() << "Got unexpected partition stream data message. " + << "PartitionSessionId: " << partitionData.partition_session_id(), + deferred); + return; + } + const TIntrusivePtr>& partitionStream = partitionStreamIt->second; + + i64 firstOffset = std::numeric_limits::max(); + i64 currentOffset = std::numeric_limits::max(); + i64 desiredOffset = partitionStream->GetFirstNotReadOffset(); + for (const auto& batch : partitionData.batches()) { + // Validate messages. + for (const auto& messageData : batch.message_data()) { + // Check offsets continuity. + if (messageData.offset() != desiredOffset) { + bool res = partitionStream->AddToCommitRanges(desiredOffset, messageData.offset(), GetRangesMode()); + Y_ABORT_UNLESS(res); + } + + if (firstOffset == std::numeric_limits::max()) { + firstOffset = messageData.offset(); + } + currentOffset = messageData.offset(); + desiredOffset = currentOffset + 1; + partitionStream->UpdateMaxReadOffset(currentOffset); + const i64 messageSize = static_cast(messageData.data().size()); + CompressedDataSize += messageSize; + *Settings.Counters_->BytesInflightTotal += messageSize; + *Settings.Counters_->BytesInflightCompressed += messageSize; + ++*Settings.Counters_->MessagesInflight; + } + } + if (firstOffset == std::numeric_limits::max()) { + BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, + TStringBuilder() << "Got empty data message. " + << "PartitionSessionId: " << partitionData.partition_session_id() + << " message: " << msg, + deferred); + return; + } + partitionStream->SetFirstNotReadOffset(desiredOffset); + + auto decompressionInfo = std::make_shared>(std::move(partitionData), + SelfContext, + Settings.Decompress_, + serverBytesSize); + // TODO (ildar-khisam@): share serverBytesSize between partitions data according to their actual sizes; + // for now whole serverBytesSize goes with first (and only) partition data. + serverBytesSize = 0; + Y_ABORT_UNLESS(decompressionInfo); + + decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, + partitionStream); + DecompressionQueue.emplace_back(decompressionInfo, partitionStream); + StartDecompressionTasksImpl(deferred); + } + + WaitingReadResponse = false; + ContinueReadingDataImpl(); +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::StartPartitionSessionRequest&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStream = MakeIntrusive>( + NextPartitionStreamId, msg.partition_session().path(), msg.partition_session().partition_id(), + msg.partition_session().partition_session_id(), msg.committed_offset(), + SelfContext); + NextPartitionStreamId += PartitionStreamIdStep; + + // Renew partition stream. + TIntrusivePtr>& currentPartitionStream = PartitionStreams[partitionStream->GetAssignId()]; + if (currentPartitionStream) { + bool pushRes = EventsQueue->PushEvent( + currentPartitionStream, + TReadSessionEvent::TPartitionSessionClosedEvent( + currentPartitionStream, TReadSessionEvent::TPartitionSessionClosedEvent::EReason::Lost), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + currentPartitionStream = partitionStream; + + // Send event to user. + bool pushRes = EventsQueue->PushEvent(partitionStream, + TReadSessionEvent::TStartPartitionSessionEvent( + partitionStream, msg.committed_offset(), msg.partition_offsets().end()), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::UpdatePartitionSession&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + Y_UNUSED(deferred); + + auto partitionStreamIt = PartitionStreams.find(msg.partition_session_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + //TODO: update generation/nodeid info +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::StopPartitionSessionRequest&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStreamIt = PartitionStreams.find(msg.partition_session_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + TIntrusivePtr> partitionStream = partitionStreamIt->second; + bool pushRes = true; + if (!msg.graceful()) { + PartitionStreams.erase(msg.partition_session_id()); + pushRes = EventsQueue->PushEvent(partitionStream, + TReadSessionEvent::TPartitionSessionClosedEvent( + partitionStream, TReadSessionEvent::TPartitionSessionClosedEvent::EReason::Lost), + deferred); + } else { + pushRes = EventsQueue->PushEvent( + partitionStream, + TReadSessionEvent::TStopPartitionSessionEvent(std::move(partitionStream), msg.committed_offset()), + deferred); + } + if (!pushRes) { + AbortImpl(); + return; + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::EndPartitionSession&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStreamIt = PartitionStreams.find(msg.partition_session_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + TIntrusivePtr> partitionStream = partitionStreamIt->second; + + std::vector adjacentPartitionIds; + adjacentPartitionIds.reserve(msg.adjacent_partition_ids_size()); + adjacentPartitionIds.insert(adjacentPartitionIds.end(), msg.adjacent_partition_ids().begin(), msg.adjacent_partition_ids().end()); + + std::vector childPartitionIds; + childPartitionIds.reserve(msg.child_partition_ids_size()); + childPartitionIds.insert(childPartitionIds.end(), msg.child_partition_ids().begin(), msg.child_partition_ids().end()); + + for (auto child : childPartitionIds) { + RegisterParentPartition(child, + partitionStream->GetPartitionId(), + partitionStream->GetPartitionSessionId()); + } + + bool pushRes = EventsQueue->PushEvent( + partitionStream, + TReadSessionEvent::TEndPartitionSessionEvent(std::move(partitionStream), std::move(adjacentPartitionIds), std::move(childPartitionIds)), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::CommitOffsetResponse&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Committed response: " << msg); + + for (const auto& rangeProto : msg.partitions_committed_offsets()) { + auto partitionStreamIt = PartitionStreams.find(rangeProto.partition_session_id()); + if (partitionStreamIt != PartitionStreams.end()) { + auto partitionStream = partitionStreamIt->second; + partitionStream->UpdateMaxCommittedOffset(rangeProto.committed_offset()); + bool pushRes = EventsQueue->PushEvent(partitionStream, + TReadSessionEvent::TCommitOffsetAcknowledgementEvent( + partitionStream, rangeProto.committed_offset()), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::StreamReadMessage::PartitionSessionStatusResponse&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto partitionStreamIt = PartitionStreams.find(msg.partition_session_id()); + if (partitionStreamIt == PartitionStreams.end()) { + return; + } + bool pushRes = EventsQueue->PushEvent(partitionStreamIt->second, + TReadSessionEvent::TPartitionSessionStatusEvent( + partitionStreamIt->second, msg.committed_offset(), + 0, // TODO: support read offset in status + msg.partition_offsets().end(), + TInstant::MilliSeconds(::google::protobuf::util::TimeUtil::TimestampToMilliseconds( + msg.write_time_high_watermark()))), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } +} + +template <> +template <> +inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( + Ydb::Topic::UpdateTokenResponse&& msg, + TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + // TODO + Y_UNUSED(msg, deferred); +} + +////////////// + +template +void TSingleClusterReadSessionImpl::StartDecompressionTasksImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (Aborting) { + return; + } + UpdateMemoryUsageStatisticsImpl(); + const i64 limit = GetDecompressedDataSizeLimit(); + Y_ABORT_UNLESS(limit > 0); + while (DecompressedDataSize < limit + && (static_cast(CompressedDataSize + DecompressedDataSize) < Settings.MaxMemoryUsageBytes_ + || DecompressedDataSize == 0 /* Allow decompression of at least one message even if memory is full. */) + && !DecompressionQueue.empty()) + { + TDecompressionQueueItem& current = DecompressionQueue.front(); + auto sentToDecompress = current.BatchInfo->StartDecompressionTasks(Settings.DecompressionExecutor_, + Max(limit - DecompressedDataSize, static_cast(1)), + deferred); + DecompressedDataSize += sentToDecompress; + if (current.BatchInfo->AllDecompressionTasksStarted()) { + DecompressionQueue.pop_front(); + } else { + break; + } + } +} + +template +void TSingleClusterReadSessionImpl::DestroyAllPartitionStreamsImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + using TClosedEvent = std::conditional_t< + UseMigrationProtocol, + NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent, + NTopic::TReadSessionEvent::TPartitionSessionClosedEvent + >; + + for (auto&& [key, partitionStream] : PartitionStreams) { + bool pushRes = EventsQueue->PushEvent(partitionStream, + TClosedEvent(std::move(partitionStream), TClosedEvent::EReason::ConnectionLost), + deferred); + if (!pushRes) { + AbortImpl(); + return; + } + } + PartitionStreams.clear(); + CookieMapping.ClearMapping(); +} + +template +void TSingleClusterReadSessionImpl::OnCreateNewDecompressionTask() { + ++DecompressionTasksInflight; +} + +template +void TSingleClusterReadSessionImpl::OnDecompressionInfoDestroy(i64 compressedSize, i64 decompressedSize, i64 messagesCount, i64 serverBytesSize) +{ + + *Settings.Counters_->MessagesInflight -= messagesCount; + *Settings.Counters_->BytesInflightUncompressed -= decompressedSize; + *Settings.Counters_->BytesInflightCompressed -= compressedSize; + *Settings.Counters_->BytesInflightTotal -= (compressedSize + decompressedSize); + + TDeferredActions deferred; + std::lock_guard guard(Lock); + UpdateMemoryUsageStatisticsImpl(); + + CompressedDataSize -= compressedSize; + DecompressedDataSize -= decompressedSize; + + if constexpr (!UseMigrationProtocol) { + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Returning serverBytesSize = " << serverBytesSize << " to budget"); + ReadSizeBudget += serverBytesSize; + } + + ContinueReadingDataImpl(); + StartDecompressionTasksImpl(deferred); +} + +template +void TSingleClusterReadSessionImpl::OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount, i64 serverBytesSize) { + + TDeferredActions deferred; + + Y_ABORT_UNLESS(DecompressionTasksInflight > 0); + --DecompressionTasksInflight; + + *Settings.Counters_->BytesRead += decompressedSize; + *Settings.Counters_->BytesReadCompressed += sourceSize; + *Settings.Counters_->MessagesRead += messagesCount; + *Settings.Counters_->BytesInflightUncompressed += decompressedSize; + *Settings.Counters_->BytesInflightCompressed -= sourceSize; + *Settings.Counters_->BytesInflightTotal += (decompressedSize - sourceSize); + + std::lock_guard guard(Lock); + UpdateMemoryUsageStatisticsImpl(); + CompressedDataSize -= sourceSize; + DecompressedDataSize += decompressedSize - estimatedDecompressedSize; + constexpr double weight = 0.6; + if (sourceSize > 0) { + AverageCompressionRatio = weight * static_cast(decompressedSize) / static_cast(sourceSize) + (1 - weight) * AverageCompressionRatio; + } + if (Aborting) { + return; + } + if constexpr (!UseMigrationProtocol) { + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Returning serverBytesSize = " << serverBytesSize << " to budget"); + ReadSizeBudget += serverBytesSize; + } + ContinueReadingDataImpl(); + StartDecompressionTasksImpl(deferred); +} + +template +void TSingleClusterReadSessionImpl::Abort() { + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Abort session to cluster"); + + std::lock_guard guard(Lock); + AbortImpl(); +} + +template +void TSingleClusterReadSessionImpl::AbortSession(TASessionClosedEvent&& closeEvent) { + TDeferredActions deferred; + LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Closing session to cluster: " << closeEvent.DebugString()); + + EventsQueue->Close(closeEvent, deferred); +} + + +template +void TSingleClusterReadSessionImpl::AbortImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Aborting) { + Aborting = true; + CallCloseCallbackImpl(); + + // Cancel(ClientContext); // Don't cancel, because this is used only as factory for other contexts. + Cancel(ConnectContext); + Cancel(ConnectTimeoutContext); + Cancel(ConnectDelayContext); + + if (Processor) { + Processor->Cancel(); + } + } +} + +template +void TSingleClusterReadSessionImpl::Close(std::function callback) { + std::lock_guard guard(Lock); + if (Aborting) { + callback(); + } + + if (!Closing) { + Closing = true; + + CloseCallback = std::move(callback); + + Cancel(ConnectContext); + Cancel(ConnectTimeoutContext); + Cancel(ConnectDelayContext); + + if (!Processor) { + CallCloseCallbackImpl(); + } else { + if (!HasCommitsInflightImpl()) { + Processor->Cancel(); + CallCloseCallbackImpl(); + } + } + } + + AbortImpl(); +} + +template +void TSingleClusterReadSessionImpl::CallCloseCallbackImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (CloseCallback) { + CloseCallback(); + CloseCallback = {}; + } + AbortImpl(); +} + +template +void TSingleClusterReadSessionImpl::StopReadingData() { + std::lock_guard guard(Lock); + DataReadingSuspended = true; +} + +template +void TSingleClusterReadSessionImpl::ResumeReadingData() { + std::lock_guard guard(Lock); + if (DataReadingSuspended) { + DataReadingSuspended = false; + ContinueReadingDataImpl(); + } +} + +template +void TSingleClusterReadSessionImpl::DumpStatisticsToLog(TLogElement& log) { + std::lock_guard guard(Lock); + // cluster:topic:partition:stream-id:read-offset:committed-offset + for (auto&& [key, partitionStream] : PartitionStreams) { + if constexpr (UseMigrationProtocol) { + log << " " + << ClusterName + << ':' << partitionStream->GetTopicPath() + << ':' << partitionStream->GetPartitionId() + << ':' << partitionStream->GetPartitionStreamId() + << ':' << partitionStream->GetMaxReadOffset() + << ':' << partitionStream->GetMaxCommittedOffset(); + } else { + log << " " + << "-" + << ':' << partitionStream->GetTopicPath() + << ':' << partitionStream->GetPartitionId() + << ':' << partitionStream->GetPartitionSessionId() + << ':' << partitionStream->GetMaxReadOffset() + << ':' << partitionStream->GetMaxCommittedOffset(); + } + } +} + +template +void TSingleClusterReadSessionImpl::UpdateMemoryUsageStatisticsImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + const TInstant now = TInstant::Now(); + const ui64 delta = (now - UsageStatisticsLastUpdateTime).MilliSeconds(); + UsageStatisticsLastUpdateTime = now; + const double percent = 100.0 / static_cast(Settings.MaxMemoryUsageBytes_); + + Settings.Counters_->TotalBytesInflightUsageByTime->Collect((DecompressedDataSize + CompressedDataSize) * percent, delta); + Settings.Counters_->UncompressedBytesInflightUsageByTime->Collect(DecompressedDataSize * percent, delta); + Settings.Counters_->CompressedBytesInflightUsageByTime->Collect(CompressedDataSize * percent, delta); +} + +template +void TSingleClusterReadSessionImpl::UpdateMemoryUsageStatistics() { + std::lock_guard guard(Lock); + UpdateMemoryUsageStatisticsImpl(); +} + +template +bool TSingleClusterReadSessionImpl::GetRangesMode() const { + if constexpr (UseMigrationProtocol) { + return Settings.RangesMode_.value_or(RangesMode); + } else { + return true; + } +} + +template +bool TSingleClusterReadSessionImpl::TPartitionCookieMapping::AddMapping(const typename TCookie::TPtr& cookie) { + if (!Cookies.emplace(cookie->GetKey(), cookie).second) { + return false; + } + for (ui64 offset = cookie->OffsetRange.first; offset < cookie->OffsetRange.second; ++offset) { + if (!UncommittedOffsetToCookie.emplace(std::make_pair(GetPartitionStreamId(cookie->PartitionStream.Get()), offset), cookie).second) { + return false; + } + } + PartitionStreamIdToCookie.emplace(GetPartitionStreamId(cookie->PartitionStream.Get()), cookie); + return true; +} + +template +typename TSingleClusterReadSessionImpl::TPartitionCookieMapping::TCookie::TPtr TSingleClusterReadSessionImpl::TPartitionCookieMapping::CommitOffset(ui64 partitionStreamId, ui64 offset) { + auto cookieIt = UncommittedOffsetToCookie.find(std::make_pair(partitionStreamId, offset)); + if (cookieIt != UncommittedOffsetToCookie.end()) { + typename TCookie::TPtr cookie; + if (!--cookieIt->second->UncommittedMessagesLeft) { + ++CommitInflight; + cookie = cookieIt->second; + } + UncommittedOffsetToCookie.erase(cookieIt); + return cookie; + } else { + ThrowFatalError(TStringBuilder() << "Invalid offset " << offset << ". Partition stream id: " << partitionStreamId << Endl); + } + // If offset wasn't found, there might be already hard released partition. + // This situation is OK. + return nullptr; +} + +template +typename TSingleClusterReadSessionImpl::TPartitionCookieMapping::TCookie::TPtr TSingleClusterReadSessionImpl::TPartitionCookieMapping::RetrieveCommittedCookie(const Ydb::PersQueue::V1::CommitCookie& cookieProto) { + typename TCookie::TPtr cookieInfo; + auto cookieIt = Cookies.find(typename TCookie::TKey(cookieProto.assign_id(), cookieProto.partition_cookie())); + if (cookieIt != Cookies.end()) { + --CommitInflight; + cookieInfo = cookieIt->second; + Cookies.erase(cookieIt); + + auto [rangeBegin, rangeEnd] = PartitionStreamIdToCookie.equal_range(GetPartitionStreamId(cookieInfo->PartitionStream.Get())); + for (auto i = rangeBegin; i != rangeEnd; ++i) { + if (i->second == cookieInfo) { + PartitionStreamIdToCookie.erase(i); + break; + } + } + } + return cookieInfo; +} + +template +void TSingleClusterReadSessionImpl::TPartitionCookieMapping::RemoveMapping(ui64 partitionStreamId) { + auto [rangeBegin, rangeEnd] = PartitionStreamIdToCookie.equal_range(partitionStreamId); + for (auto i = rangeBegin; i != rangeEnd; ++i) { + typename TCookie::TPtr cookie = i->second; + Cookies.erase(cookie->GetKey()); + for (ui64 offset = cookie->OffsetRange.first; offset < cookie->OffsetRange.second; ++offset) { + UncommittedOffsetToCookie.erase(std::make_pair(partitionStreamId, offset)); + } + } + PartitionStreamIdToCookie.erase(rangeBegin, rangeEnd); +} + +template +void TSingleClusterReadSessionImpl::TPartitionCookieMapping::ClearMapping() { + Cookies.clear(); + UncommittedOffsetToCookie.clear(); + PartitionStreamIdToCookie.clear(); + CommitInflight = 0; +} + +template +bool TSingleClusterReadSessionImpl::TPartitionCookieMapping::HasUnacknowledgedCookies() const { + return CommitInflight != 0; +} + +template +void TSingleClusterReadSessionImpl::RegisterParentPartition(ui32 partitionId, ui32 parentPartitionId, ui64 parentPartitionSessionId) { + auto& values = HierarchyData[partitionId]; + values.push_back({parentPartitionId, parentPartitionSessionId}); +} + +template +void TSingleClusterReadSessionImpl::UnregisterPartition(ui32 partitionId, ui64 partitionSessionId) { + for (auto it = HierarchyData.begin(); it != HierarchyData.end();) { + auto& values = it->second; + for (auto v = values.begin(); v != values.end();) { + if (v->PartitionId == partitionId && v->PartitionSessionId < partitionSessionId) { + v = values.erase(v); + } else { + ++v; + } + } + if (values.empty()) { + it = HierarchyData.erase(it); + } else { + ++it; + } + } +} + +template +std::vector TSingleClusterReadSessionImpl::GetParentPartitionSessions(ui32 partitionId, ui64 partitionSessionId) { + auto it = HierarchyData.find(partitionId); + if (it == HierarchyData.end()) { + return {}; + } + + auto& parents = it->second; + + std::unordered_map index; + for (auto& v : parents) { + if (v.PartitionSessionId > partitionSessionId) { + break; + } + + index[v.PartitionId] = v.PartitionSessionId; + } + + std::vector result; + for (auto [_, v] : index) { + result.push_back(v); + } + + return result; +} + +template +bool TSingleClusterReadSessionImpl::AllParentSessionsHasBeenRead(ui32 partitionId, ui64 partitionSessionId) { + for (auto id : GetParentPartitionSessions(partitionId, partitionSessionId)) { + if (!ReadingFinishedData.contains(id)) { + return false; + } + } + + return true; +} + +template +void TSingleClusterReadSessionImpl::ConfirmPartitionStreamEnd(TPartitionStreamImpl* partitionStream, const std::vector& childIds) { + ReadingFinishedData.insert(partitionStream->GetPartitionSessionId()); + for (auto& [_, s] : PartitionStreams) { + for (auto partitionId : childIds) { + if (s->GetPartitionId() == partitionId) { + EventsQueue->SignalReadyEvents(s); + break; + } + } + } +} + +template +void TSingleClusterReadSessionImpl::CollectOffsets(NTable::TTransaction& tx, + const std::vector& events, + std::shared_ptr client) +{ + auto txInfo = GetOrCreateTxInfo(MakeTransactionId(tx)); + TrySubscribeOnTransactionCommit(tx, std::move(client)); + with_lock (txInfo->Lock) { + txInfo->OffsetsCollector.CollectOffsets(events); + } +} + +template +void TSingleClusterReadSessionImpl::CollectOffsets(NTable::TTransaction& tx, + const TReadSessionEvent::TEvent& event, + std::shared_ptr client) +{ + auto txInfo = GetOrCreateTxInfo(MakeTransactionId(tx)); + TrySubscribeOnTransactionCommit(tx, std::move(client)); + with_lock (txInfo->Lock) { + txInfo->OffsetsCollector.CollectOffsets(event); + } +} + +template +void TSingleClusterReadSessionImpl::TrySubscribeOnTransactionCommit(NTable::TTransaction& tx, + std::shared_ptr client) +{ + const TTransactionId txId = MakeTransactionId(tx); + auto txInfo = GetOrCreateTxInfo(txId); + Y_ABORT_UNLESS(txInfo); + + with_lock (txInfo->Lock) { + if (txInfo->Subscribed) { + return; + } + + auto callback = [cbContext = this->SelfContext, txId, txInfo, consumer = Settings.ConsumerName_, client]() { + std::vector offsets; + + with_lock (txInfo->Lock) { + Y_ABORT_UNLESS(!txInfo->CommitCalled); + + txInfo->CommitCalled = true; + + offsets = txInfo->OffsetsCollector.GetOffsets(); + } + + if (auto self = cbContext->LockShared()) { + self->DeleteTx(txId); + } + + return client->UpdateOffsetsInTransaction(txId, + offsets, + consumer, + {}); + }; + + tx.AddPrecommitCallback(std::move(callback)); + + txInfo->IsActive = true; + txInfo->Subscribed = true; + } +} + +template +auto TSingleClusterReadSessionImpl::GetOrCreateTxInfo(const TTransactionId& txId) -> TTransactionInfoPtr +{ + with_lock (Lock) { + auto p = Txs.find(txId); + if (p == Txs.end()) { + TTransactionInfoPtr& txInfo = Txs[txId]; + txInfo = std::make_shared(); + txInfo->Subscribed = false; + txInfo->CommitCalled = false; + p = Txs.find(txId); + } + return p->second; + } +} + +template +void TSingleClusterReadSessionImpl::DeleteTx(const TTransactionId& txId) +{ + with_lock (Lock) { + Txs.erase(txId); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TReadSessionEventInfo + +template +TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr> partitionStream, + TCallbackContextPtr cbContext, + TEvent event) + : PartitionStream(std::move(partitionStream)) + , Event(std::move(event)) + , CbContext(std::move(cbContext)) +{ +} + +template +TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr> partitionStream, + TCallbackContextPtr cbContext, + bool hasDataEvents) + : PartitionStream(std::move(partitionStream)) + , HasDataEvents(hasDataEvents) + , EventsCount(1) + , CbContext(std::move(cbContext)) +{ +} + +template +bool TReadSessionEventInfo::IsEmpty() const { + return !PartitionStream || !PartitionStream->HasEvents(); +} + +template +bool TReadSessionEventInfo::IsDataEvent() const { + return !IsEmpty() && PartitionStream->TopEvent().IsDataEvent(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TReadSessionEventsQueue + +template +TReadSessionEventsQueue::TReadSessionEventsQueue( + const TAReadSessionSettings& settings) + : TParent(settings) { + const auto& h = TParent::Settings.EventHandlers_; + + if constexpr (UseMigrationProtocol) { + HasEventCallbacks = (h.CommonHandler_ + || h.DataReceivedHandler_ + || h.CommitAcknowledgementHandler_ + || h.CreatePartitionStreamHandler_ + || h.DestroyPartitionStreamHandler_ + || h.PartitionStreamStatusHandler_ + || h.PartitionStreamClosedHandler_ + || h.SessionClosedHandler_); + } else { + HasEventCallbacks = (h.CommonHandler_ + || h.DataReceivedHandler_ + || h.CommitOffsetAcknowledgementHandler_ + || h.StartPartitionSessionHandler_ + || h.StopPartitionSessionHandler_ + || h.EndPartitionSessionHandler_ + || h.PartitionSessionStatusHandler_ + || h.PartitionSessionClosedHandler_ + || h.SessionClosedHandler_); + } +} + +template +bool TReadSessionEventsQueue::PushEvent(TIntrusivePtr> stream, + typename TAReadSessionEvent::TEvent event, + TDeferredActions& deferred) +{ + std::lock_guard guard(TParent::Mutex); + if (TParent::Closed) { + return false; + } + //TODO: check session closed event and return false + using TClosedEvent = std::conditional_t< + UseMigrationProtocol, + NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent, + NTopic::TReadSessionEvent::TPartitionSessionClosedEvent + >; + + if (std::holds_alternative(event)) { + stream->DeleteNotReadyTail(deferred); + } + + if (!HasDataEventCallback() && !std::holds_alternative>(event)) { + // Call non-dataEvent callbacks immediately. + if (TryApplyCallbackToEventImpl(event, deferred, CbContext)) { + return true; + } + } + + stream->InsertEvent(std::move(event)); + Y_ASSERT(stream->HasEvents()); + + SignalReadyEventsImpl(stream, deferred); + return true; +} + +template +void TReadSessionEventsQueue::SignalEventImpl( + TIntrusivePtr> partitionStream, + TDeferredActions& deferred, + bool isDataEvent) +{ + if (TParent::Closed) { + return; + } + + auto cbContext = partitionStream->GetCbContext(); + + if (TParent::Events.empty()) { + TParent::Events.emplace(std::move(partitionStream), std::move(cbContext), isDataEvent); + } else { + auto& event = TParent::Events.back(); + if (event.HasDataEvents + && isDataEvent + && (event.PartitionStream == partitionStream)) { + ++event.EventsCount; + } else { + TParent::Events.emplace(std::move(partitionStream), std::move(cbContext), isDataEvent); + } + } + + SignalWaiterImpl(deferred); +} + +template +bool TReadSessionEventsQueue::PushDataEvent(TIntrusivePtr> partitionStream, + size_t batch, + size_t message, + TDataDecompressionInfoPtr parent, + std::atomic& ready) +{ + + std::lock_guard guard(TParent::Mutex); + if (this->Closed) { + return false; + } + partitionStream->InsertDataEvent(batch, message, parent, ready); + return true; +} + +template +void TRawPartitionStreamEventQueue::GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator) +{ + GetDataEventImpl(partitionStream, + maxEventsCount, + maxByteSize, + messages, + compressedMessages, + accumulator, + Ready); +} + +template +void TRawPartitionStreamEventQueue::GetDataEventImpl(TIntrusivePtr> partitionStream, + size_t& maxEventsCount, + size_t& maxByteSize, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + TUserRetrievedEventsInfoAccumulator& accumulator, + std::deque>& queue) +{ + auto readyDataInTheHead = [&queue]() { + if (queue.empty()) { + return false; + } + + auto& front = queue.front(); + + return front.IsDataEvent() && front.IsReady(); + }; + + Y_ABORT_UNLESS(readyDataInTheHead()); + + for (; readyDataInTheHead() && (maxEventsCount > 0) && (maxByteSize > 0); --maxEventsCount) { + auto& event = queue.front().GetDataEvent(); + + TDataDecompressionInfoPtr parent = event.GetParent(); + size_t size = 0; + + event.TakeData(partitionStream, messages, compressedMessages, maxByteSize, size); + queue.pop_front(); + + accumulator.Add(parent, size); + } +} + +template +TADataReceivedEvent +TReadSessionEventsQueue::GetDataEventImpl(TIntrusivePtr> stream, + size_t& maxByteSize, + TUserRetrievedEventsInfoAccumulator& accumulator) // Assumes that we're under lock. +{ + std::vector::TMessage> messages; + std::vector::TCompressedMessage> compressedMessages; + + Y_ABORT_UNLESS(!TParent::Events.empty()); + + auto& event = TParent::Events.front(); + + Y_ABORT_UNLESS(event.PartitionStream == stream); + Y_ABORT_UNLESS(event.EventsCount > 0); + + TPartitionStreamImpl::GetDataEventImpl(stream, + event.EventsCount, + maxByteSize, + messages, + compressedMessages, + accumulator); + + if (event.EventsCount == 0) { + TParent::Events.pop(); + } + + Y_ABORT_UNLESS(!messages.empty() || !compressedMessages.empty()); + + return {std::move(messages), std::move(compressedMessages), stream}; +} + +template +TReadSessionEventInfo +TReadSessionEventsQueue::GetEventImpl(size_t& maxByteSize, + TUserRetrievedEventsInfoAccumulator& accumulator) // Assumes that we're under lock. +{ + Y_ASSERT(TParent::HasEventsImpl()); + + if (!TParent::Events.empty()) { + TReadSessionEventInfo& front = TParent::Events.front(); + auto partitionStream = front.PartitionStream; + + if (!partitionStream->HasEvents()) { + Y_ABORT("can't be here - got events in global queue, but nothing in partition queue"); + } + + std::optional::TEvent> event; + auto frontCbContext = front.CbContext; + if (partitionStream->TopEvent().IsDataEvent()) { + event = GetDataEventImpl(partitionStream, maxByteSize, accumulator); + } else { + event = std::move(partitionStream->TopEvent().GetEvent()); + partitionStream->PopEvent(); + + TParent::Events.pop(); + + if constexpr (!UseMigrationProtocol) { + if (std::holds_alternative(*event)) { + auto& e = std::get(*event); + if (auto session = frontCbContext->LockShared()) { + session->UnregisterPartition(e.GetPartitionSession()->GetPartitionId(), e.GetPartitionSession()->GetPartitionSessionId()); + } + } + } + } + + TParent::RenewWaiterImpl(); + + return {partitionStream, std::move(frontCbContext), std::move(*event)}; + } + + Y_ASSERT(TParent::CloseEvent); + + return {*TParent::CloseEvent}; +} + +template +std::vector::TEvent> +TReadSessionEventsQueue::GetEvents(bool block, std::optional maxEventsCount, size_t maxByteSize) +{ + if (!maxByteSize) { + ThrowFatalError("the maxByteSize value must be greater than 0"); + } + + std::vector> eventInfos; + const size_t maxCount = maxEventsCount ? *maxEventsCount : std::numeric_limits::max(); + TUserRetrievedEventsInfoAccumulator accumulator; + { + std::lock_guard guard(TParent::Mutex); + eventInfos.reserve(Min(TParent::Events.size() + TParent::CloseEvent.has_value(), maxCount)); + do { + if (block) { + TParent::WaitEventsImpl(); + } + + while (TParent::HasEventsImpl() && eventInfos.size() < maxCount && maxByteSize > 0) { + TReadSessionEventInfo event = GetEventImpl(maxByteSize, accumulator); + eventInfos.emplace_back(std::move(event)); + if (eventInfos.back().IsSessionClosedEvent()) { + break; + } + } + } while (block && eventInfos.empty()); + } + + accumulator.OnUserRetrievedEvent(); + + std::vector::TEvent> result; + result.reserve(eventInfos.size()); + for (TReadSessionEventInfo& eventInfo : eventInfos) { + result.emplace_back(std::move(eventInfo.GetEvent())); + } + + return result; +} + +template +std::optional::TEvent> +TReadSessionEventsQueue::GetEvent(bool block, size_t maxByteSize) +{ + if (!maxByteSize) { + ThrowFatalError("the maxByteSize value must be greater than 0"); + } + + std::optional> eventInfo; + TUserRetrievedEventsInfoAccumulator accumulator; + { + std::lock_guard guard(TParent::Mutex); + do { + if (block) { + TParent::WaitEventsImpl(); + } + + if (TParent::HasEventsImpl()) { + eventInfo = GetEventImpl(maxByteSize, accumulator); + } + + } while (block && !eventInfo); + } + + accumulator.OnUserRetrievedEvent(); + + if (eventInfo) { + return std::move(eventInfo->Event); + } + + return std::nullopt; +} + +template +void TReadSessionEventsQueue::SignalReadyEvents( + TIntrusivePtr> partitionStream) { + Y_ASSERT(partitionStream); + + std::lock_guard guard1(partitionStream->GetLock()); + TDeferredActions deferred; + { + std::lock_guard g(TParent::Mutex); + SignalReadyEventsImpl(partitionStream, deferred); + } +} + +template +void TReadSessionEventsQueue::SignalReadyEventsImpl( + TIntrusivePtr> partitionStream, + TDeferredActions& deferred) { + TPartitionStreamImpl::SignalReadyEvents(partitionStream, this, deferred); +} + +template +bool TReadSessionEventsQueue::TryApplyCallbackToEventImpl(typename TParent::TEvent& event, + TDeferredActions& deferred, + TCallbackContextPtr& cbContext) +{ + THandlersVisitor visitor(TParent::Settings, event, deferred, cbContext); + return visitor.Visit(); +} + +template +bool TReadSessionEventsQueue::HasDataEventCallback() const +{ + if (!HasEventCallbacks) { + return false; + } + + if (TParent::Settings.EventHandlers_.DataReceivedHandler_) { + return true; + } + if (TParent::Settings.EventHandlers_.CommonHandler_) { + return true; + } + + return false; +} + +template +void TReadSessionEventsQueue::ApplyCallbackToEventImpl(TADataReceivedEvent& data, + TUserRetrievedEventsInfoAccumulator&& eventsInfo, + TDeferredActions& deferred) +{ + Y_ABORT_UNLESS(HasEventCallbacks); + + if (TParent::Settings.EventHandlers_.DataReceivedHandler_) { + auto action = [func = TParent::Settings.EventHandlers_.DataReceivedHandler_, + data = std::move(data), + eventsInfo = std::move(eventsInfo)]() mutable { + func(data); + eventsInfo.OnUserRetrievedEvent(); + }; + + deferred.DeferStartExecutorTask(TParent::Settings.EventHandlers_.HandlersExecutor_, std::move(action)); + } else if (TParent::Settings.EventHandlers_.CommonHandler_) { + auto action = [func = TParent::Settings.EventHandlers_.CommonHandler_, + data = std::move(data), + eventsInfo = std::move(eventsInfo)]() mutable { + typename TParent::TEvent event(std::move(data)); + + func(event); + eventsInfo.OnUserRetrievedEvent(); + }; + + deferred.DeferStartExecutorTask(TParent::Settings.EventHandlers_.HandlersExecutor_, std::move(action)); + } else { + Y_ABORT_UNLESS(false); + } +} + +template +void TReadSessionEventsQueue::GetDataEventCallbackSettings(size_t& maxMessagesBytes) +{ + maxMessagesBytes = TParent::Settings.EventHandlers_.MaxMessagesBytes_; +} + +template +void TReadSessionEventsQueue::ClearAllEvents() { + std::lock_guard guard(TParent::Mutex); + while (!TParent::Events.empty()) { + auto& event = TParent::Events.front(); + if (event.PartitionStream && event.PartitionStream->HasEvents()) { + event.PartitionStream->PopEvent(); + } + TParent::Events.pop(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TDataDecompressionInfo + +template +TDataDecompressionInfo::TDataDecompressionInfo( + TPartitionData&& msg, + TCallbackContextPtr cbContext, + bool doDecompress, + i64 serverBytesSize +) + : ServerMessage(std::move(msg)) + , CbContext(std::move(cbContext)) + , DoDecompress(doDecompress) + , ServerBytesSize(serverBytesSize) +{ + i64 compressedSize = 0; + i64 messagesCount = 0; + + for (const auto& batch : ServerMessage.batches()) { + for (const auto& messageData : batch.message_data()) { + compressedSize += messageData.data().size(); + ++messagesCount; + } + } + + MessagesInflight = messagesCount; + SourceDataNotProcessed = compressedSize; + CompressedDataSize = compressedSize; + + BuildBatchesMeta(); +} + +template +TDataDecompressionInfo::~TDataDecompressionInfo() +{ + if (auto session = CbContext->LockShared()) { + session->OnDecompressionInfoDestroy(CompressedDataSize, DecompressedDataSize, MessagesInflight, ServerBytesSize); + } +} + +template +void TDataDecompressionInfo::BuildBatchesMeta() { + BatchesMeta.reserve(ServerMessage.batches_size()); + if constexpr (!UseMigrationProtocol) { + MessagesMeta.reserve(ServerMessage.batches_size()); + } + for (const auto& batch : ServerMessage.batches()) { + // Extra fields. + typename TAWriteSessionMeta::TPtr meta = MakeIntrusive>(); + + if constexpr (UseMigrationProtocol) { + meta->Fields.reserve(batch.extra_fields_size()); + for (const Ydb::PersQueue::V1::KeyValue& kv : batch.extra_fields()) { + meta->Fields.emplace(kv.key(), kv.value()); + } + } else { + meta->Fields.reserve(batch.write_session_meta_size()); + for (const auto& [key, value] : batch.write_session_meta()) { + meta->Fields.emplace(key, value); + } + MessagesMeta.emplace_back(TMessageMetaPtrVector{}); + auto& currBatchMessagesMeta = MessagesMeta.back(); + for (const auto& messageData: batch.message_data()) { + typename TAMessageMeta::TPtr msgMeta = MakeIntrusive>(); + msgMeta->Fields.reserve(messageData.metadata_items_size()); + for (const auto& metaPair: messageData.metadata_items()) { + msgMeta->Fields.emplace_back(std::make_pair(metaPair.key(), metaPair.value())); + } + currBatchMessagesMeta.emplace_back(std::move(msgMeta)); + } + } + + BatchesMeta.emplace_back(std::move(meta)); + } +} + +template +void TDataDecompressionInfo::PutDecompressionError(std::exception_ptr error, size_t batch, size_t message) { + if (!DecompressionErrorsStructCreated) { + std::lock_guard guard(DecompressionErrorsStructLock); + DecompressionErrors.resize(ServerMessage.batches_size()); + for (size_t batch = 0; batch < static_cast(ServerMessage.batches_size()); ++batch) { + DecompressionErrors[batch].resize(static_cast(ServerMessage.batches(batch).message_data_size())); + } + + // Set barrier. + DecompressionErrorsStructCreated = true; + } + Y_ASSERT(batch < DecompressionErrors.size()); + Y_ASSERT(message < DecompressionErrors[batch].size()); + DecompressionErrors[batch][message] = std::move(error); +} + +template +std::exception_ptr TDataDecompressionInfo::GetDecompressionError(size_t batch, size_t message) { + if (!DecompressionErrorsStructCreated) { + return {}; + } + Y_ASSERT(batch < DecompressionErrors.size()); + Y_ASSERT(message < DecompressionErrors[batch].size()); + return DecompressionErrors[batch][message]; +} + +template +i64 TDataDecompressionInfo::StartDecompressionTasks( + const typename IAExecutor::TPtr& executor, i64 availableMemory, + TDeferredActions& deferred) +{ + auto session = CbContext->LockShared(); + Y_ASSERT(session); + + i64 used = 0; + + while (availableMemory > 0 && !Tasks.empty()) { + auto& task = Tasks.front(); + + used += task.GetEstimatedDecompressedSize(); + availableMemory -= task.GetEstimatedDecompressedSize(); + + session->OnCreateNewDecompressionTask(); + + deferred.DeferStartExecutorTask(executor, std::move(task)); + Tasks.pop_front(); + } + + return used; +} + +template +void TDataDecompressionInfo::PlanDecompressionTasks(double averageCompressionRatio, + TIntrusivePtr> partitionStream) { + constexpr size_t TASK_LIMIT = 512_KB; + + auto session = CbContext->LockShared(); + Y_ASSERT(session); + + ReadyThresholds.emplace_back(); + + TDecompressionTask task(TDataDecompressionInfo::shared_from_this(), partitionStream, &ReadyThresholds.back()); + + while (CurrentDecompressingMessage.first < static_cast(ServerMessage.batches_size())) { + const auto& batch = ServerMessage.batches(CurrentDecompressingMessage.first); + + if (CurrentDecompressingMessage.second < static_cast(batch.message_data_size())) { + const auto& messageData = batch.message_data(CurrentDecompressingMessage.second); + const i64 size = static_cast(messageData.data().size()); + const i64 estimatedDecompressedSize = messageData.uncompressed_size() + ? static_cast(messageData.uncompressed_size()) + : static_cast(size * averageCompressionRatio); + Y_ABORT_UNLESS(estimatedDecompressedSize >= 0); + + task.Add(CurrentDecompressingMessage.first, CurrentDecompressingMessage.second, size, estimatedDecompressedSize); + + bool pushRes = session->GetEventsQueue()->PushDataEvent(partitionStream, + CurrentDecompressingMessage.first, + CurrentDecompressingMessage.second, + TDataDecompressionInfo::shared_from_this(), + ReadyThresholds.back().Ready); + if (!pushRes) { + session->AbortImpl(); + return; + } + } + + ++CurrentDecompressingMessage.second; + + if (CurrentDecompressingMessage.second >= static_cast(batch.message_data_size())) { // next batch + ++CurrentDecompressingMessage.first; + CurrentDecompressingMessage.second = 0; + } + + if (task.AddedDataSize() >= TASK_LIMIT) { + Tasks.push_back(std::move(task)); + + ReadyThresholds.emplace_back(); + task = TDecompressionTask(TDataDecompressionInfo::shared_from_this(), partitionStream, &ReadyThresholds.back()); + } + } + + if (task.AddedMessagesCount() > 0) { + Tasks.push_back(std::move(task)); + } else { + ReadyThresholds.pop_back(); // Revert. + } +} + +template +void TDataDecompressionInfo::OnDestroyReadSession() +{ + for (auto& task : Tasks) { + task.ClearParent(); + } +} + +template +void TDataDecompressionEvent::TakeData(TIntrusivePtr> partitionStream, + std::vector::TMessage>& messages, + std::vector::TCompressedMessage>& compressedMessages, + size_t& maxByteSize, + size_t& dataSize) const +{ + auto& msg = Parent->GetServerMessage(); + i64 minOffset = Max(); + i64 maxOffset = 0; + auto& batch = *msg.mutable_batches(Batch); + const auto& meta = Parent->GetBatchMeta(Batch); + const TInstant batchWriteTimestamp = [&batch](){ + if constexpr (UseMigrationProtocol) { + return TInstant::MilliSeconds(batch.write_timestamp_ms()); + } else { + return TInstant::MilliSeconds(::google::protobuf::util::TimeUtil::TimestampToMilliseconds(batch.written_at())); + } + }(); + auto& messageData = *batch.mutable_message_data(Message); + + minOffset = Min(minOffset, static_cast(messageData.offset())); + maxOffset = Max(maxOffset, static_cast(messageData.offset())); + + if constexpr (UseMigrationProtocol) { + using TMessageInformation = NPersQueue::TReadSessionEvent::TDataReceivedEvent::TMessageInformation; + + TMessageInformation messageInfo(messageData.offset(), + batch.source_id(), + messageData.seq_no(), + TInstant::MilliSeconds(messageData.create_timestamp_ms()), + batchWriteTimestamp, + batch.ip(), + meta, + messageData.uncompressed_size()); + + if (Parent->GetDoDecompress()) { + messages.emplace_back(messageData.data(), + Parent->GetDecompressionError(Batch, Message), + messageInfo, + partitionStream, + messageData.partition_key(), + messageData.explicit_hash()); + } else { + compressedMessages.emplace_back(static_cast(messageData.codec()), + messageData.data(), + std::vector{messageInfo}, + partitionStream, + messageData.partition_key(), + messageData.explicit_hash()); + } + } else { + const auto& messageMeta = Parent->GetMessageMeta(Batch, Message); + TReadSessionEvent::TDataReceivedEvent::TMessageInformation messageInfo( + messageData.offset(), + batch.producer_id(), + messageData.seq_no(), + TInstant::MilliSeconds(::google::protobuf::util::TimeUtil::TimestampToMilliseconds(messageData.created_at())), + batchWriteTimestamp, + meta, + messageMeta, + messageData.uncompressed_size(), + messageData.message_group_id() + ); + + if (Parent->GetDoDecompress()) { + messages.emplace_back(messageData.data(), + Parent->GetDecompressionError(Batch, Message), + messageInfo, + partitionStream); + } else { + compressedMessages.emplace_back(static_cast(batch.codec()), + messageData.data(), + messageInfo, + partitionStream); + } + } + + maxByteSize -= Min(maxByteSize, messageData.data().size()); + + dataSize += messageData.data().size(); + + // Clear data to free internal session's memory. + messageData.clear_data(); + + LOG_LAZY(partitionStream->GetLog(), TLOG_DEBUG, TStringBuilder() + << "Take Data. Partition " << partitionStream->GetPartitionId() + << ". Read: {" << Batch << ", " << Message << "} (" + << minOffset << "-" << maxOffset << ")"); +} + +template +bool TDataDecompressionInfo::HasReadyUnreadData() const { + std::optional> threshold = GetReadyThreshold(); + if (!threshold) { + return false; + } + return CurrentReadingMessage <= *threshold; +} + +template +void TDataDecompressionInfo::OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount) +{ + CompressedDataSize -= sourceSize; + DecompressedDataSize += decompressedSize; + + if (auto session = CbContext->LockShared()) { + // TODO (ildar-khisam@): distribute total ServerBytesSize in proportion of source size + // Use CompressedDataSize, sourceSize, ServerBytesSize + session->OnDataDecompressed(sourceSize, estimatedDecompressedSize, decompressedSize, messagesCount, std::exchange(ServerBytesSize, 0)); + } +} + +template +void TDataDecompressionInfo::OnUserRetrievedEvent(i64 decompressedSize, size_t messagesCount) +{ + MessagesInflight -= messagesCount; + DecompressedDataSize -= decompressedSize; + + if (auto session = CbContext->LockShared()) { + session->OnUserRetrievedEvent(decompressedSize, messagesCount); + } +} + +template +void TDataDecompressionInfo::TDecompressionTask::Add(size_t batch, size_t message, + size_t sourceDataSize, + size_t estimatedDecompressedSize) { + if (Messages.empty() || Messages.back().Batch != batch) { + Messages.push_back({ batch, { message, message + 1 } }); + } + Messages.back().MessageRange.second = message + 1; + SourceDataSize += sourceDataSize; + EstimatedDecompressedSize += estimatedDecompressedSize; + Ready->Batch = batch; + Ready->Message = message; +} + +template +TDataDecompressionInfo::TDecompressionTask::TDecompressionTask( + TDataDecompressionInfo::TPtr parent, TIntrusivePtr> partitionStream, + TReadyMessageThreshold* ready) + : Parent(std::move(parent)) + , PartitionStream(std::move(partitionStream)) + , Ready(ready) { +} + +template +void TDataDecompressionInfo::TDecompressionTask::operator()() { + auto parent = Parent; + if (!parent) { + return; + } + i64 minOffset = Max(); + i64 maxOffset = 0; + const i64 partition_id = [parent](){ + if constexpr (UseMigrationProtocol) { + return parent->ServerMessage.partition(); + } else { + return parent->ServerMessage.partition_session_id(); + } + }(); + i64 dataProcessed = 0; + size_t messagesProcessed = 0; + for (const TMessageRange& messages : Messages) { + auto& batch = *parent->ServerMessage.mutable_batches(messages.Batch); + for (size_t i = messages.MessageRange.first; i < messages.MessageRange.second; ++i) { + auto& data = *batch.mutable_message_data(i); + + ++messagesProcessed; + dataProcessed += static_cast(data.data().size()); + minOffset = Min(minOffset, static_cast(data.offset())); + maxOffset = Max(maxOffset, static_cast(data.offset())); + + try { + if constexpr (UseMigrationProtocol) { + if (parent->DoDecompress + && data.codec() != Ydb::PersQueue::V1::CODEC_RAW + && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED + ) { + const ICodec* codecImpl = TCodecMap::GetTheCodecMap().GetOrThrow(static_cast(data.codec())); + std::string decompressed = codecImpl->Decompress(data.data()); + data.set_data(TStringType{decompressed}); + data.set_codec(Ydb::PersQueue::V1::CODEC_RAW); + } + } else { + if (parent->DoDecompress + && static_cast(batch.codec()) != Ydb::Topic::CODEC_RAW + && static_cast(batch.codec()) != Ydb::Topic::CODEC_UNSPECIFIED + ) { + const ICodec* codecImpl = TCodecMap::GetTheCodecMap().GetOrThrow(static_cast(batch.codec())); + std::string decompressed = codecImpl->Decompress(data.data()); + data.set_data(TStringType{decompressed}); + } + } + + DecompressedSize += data.data().size(); + } catch (...) { + parent->PutDecompressionError(std::current_exception(), messages.Batch, i); + data.clear_data(); // Free memory, because we don't count it. + + if (auto session = parent->CbContext->LockShared()) { + session->GetLog() << TLOG_INFO << "Error decompressing data: " << CurrentExceptionMessage(); + } + } + } + } + if (auto session = parent->CbContext->LockShared()) { + LOG_LAZY(session->GetLog(), TLOG_DEBUG, TStringBuilder() << "Decompression task done. Partition/PartitionSessionId: " + << partition_id << " (" << minOffset << "-" + << maxOffset << ")"); + } + Y_ASSERT(dataProcessed == SourceDataSize); + + parent->OnDataDecompressed(SourceDataSize, EstimatedDecompressedSize, DecompressedSize, messagesProcessed); + + parent->SourceDataNotProcessed -= dataProcessed; + Ready->Ready = true; + + if (auto session = parent->CbContext->LockShared()) { + session->GetEventsQueue()->SignalReadyEvents(PartitionStream); + } +} + +template +void TDataDecompressionInfo::TDecompressionTask::ClearParent() +{ + Parent = nullptr; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TUserRetrievedEventsInfoAccumulator + +template +void TUserRetrievedEventsInfoAccumulator::Add(TDataDecompressionInfoPtr info, i64 decompressedSize) +{ + auto& counter = Counters[info]; + + counter.DecompressedSize += decompressedSize; + ++counter.MessagesCount; +} + +template +void TUserRetrievedEventsInfoAccumulator::OnUserRetrievedEvent() const +{ + for (auto& [parent, counter] : Counters) { + parent->OnUserRetrievedEvent(counter.DecompressedSize, counter.MessagesCount); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TDeferredActions + +template +void TDeferredActions::DeferReadFromProcessor(const typename IProcessor::TPtr& processor, + TServerMessage* dst, + typename IProcessor::TReadCallback callback) +{ + Y_ASSERT(!Processor); + Y_ASSERT(!ReadDst); + Y_ASSERT(!ReadCallback); + Processor = processor; + ReadDst = dst; + ReadCallback = std::move(callback); +} + +template +void TDeferredActions::DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task) { + ExecutorsTasks.emplace_back(executor, std::move(task)); +} + +template +void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, TASessionClosedEvent&& closeEvent) { + CbContext = std::move(cbContext); + SessionClosedEvent.ConstructInPlace(std::move(closeEvent)); +} + +template +void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + DeferAbortSession(std::move(cbContext), TASessionClosedEvent(statusCode, std::move(issues))); +} + +template +void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message) { + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + DeferAbortSession(std::move(cbContext), statusCode, std::move(issues)); +} + +template +void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, TPlainStatus&& status) { + DeferAbortSession(std::move(cbContext), TASessionClosedEvent(std::move(status))); +} + +template +void TDeferredActions::DeferReconnection(TCallbackContextPtr cbContext, TPlainStatus&& status) { + CbContext = std::move(cbContext); + ReconnectionStatus = std::move(status); +} + +template +void TDeferredActions::DeferStartSession(TCallbackContextPtr cbContext) { + CbContexts.push_back(std::move(cbContext)); +} + +template +void TDeferredActions::DeferSignalWaiter(TWaiter&& waiter) { + Waiters.emplace_back(std::move(waiter)); +} + +template +void TDeferredActions::DeferDestroyDecompressionInfos(std::vector>&& infos) +{ + DecompressionInfos = std::move(infos); +} + +template +void TDeferredActions::DoActions() { + Read(); + StartExecutorTasks(); + AbortSession(); + Reconnect(); + SignalWaiters(); + StartSessions(); +} + +template +void TDeferredActions::StartSessions() { + for (auto& ctx : CbContexts) { + if (auto session = ctx->LockShared()) { + session->Start(); + } + } +} + +template +void TDeferredActions::Read() { + if (ReadDst) { + Y_ASSERT(Processor); + Y_ASSERT(ReadCallback); + Processor->Read(ReadDst, std::move(ReadCallback)); + } +} + +template +void TDeferredActions::StartExecutorTasks() { + for (auto&& [executor, task] : ExecutorsTasks) { + executor->Post(std::move(task)); + } +} + +template +void TDeferredActions::AbortSession() { + if (SessionClosedEvent) { + Y_ABORT_UNLESS(CbContext); + if (auto session = CbContext->LockShared()) { + session->AbortSession(std::move(*SessionClosedEvent)); + } + } +} + +template +void TDeferredActions::Reconnect() { + if (CbContext) { + if (auto session = CbContext->LockShared()) { + if (!session->Reconnect(ReconnectionStatus)) { + session->AbortSession(std::move(ReconnectionStatus)); + } + } + } +} + +template +void TDeferredActions::SignalWaiters() { + for (auto& w : Waiters) { + w.Signal(); + } +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp new file mode 100644 index 000000000000..ea3d0b59ee6d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp @@ -0,0 +1,538 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +namespace NYdb::inline V3::NTopic { + +class TCommonCodecsProvider { +public: + TCommonCodecsProvider() { + TCodecMap::GetTheCodecMap().Set((uint32_t)ECodec::GZIP, std::make_unique()); + TCodecMap::GetTheCodecMap().Set((uint32_t)ECodec::ZSTD, std::make_unique()); + } +}; +TCommonCodecsProvider COMMON_CODECS_PROVIDER; + +TDescribeTopicResult::TDescribeTopicResult(TStatus&& status, Ydb::Topic::DescribeTopicResult&& result) + : TStatus(std::move(status)) + , TopicDescription_(std::move(result)) +{ +} + +const TTopicDescription& TDescribeTopicResult::GetTopicDescription() const { + return TopicDescription_; +} + +TDescribeConsumerResult::TDescribeConsumerResult(TStatus&& status, Ydb::Topic::DescribeConsumerResult&& result) + : TStatus(std::move(status)) + , ConsumerDescription_(std::move(result)) +{ +} + +const TConsumerDescription& TDescribeConsumerResult::GetConsumerDescription() const { + return ConsumerDescription_; +} + +TDescribePartitionResult::TDescribePartitionResult(TStatus&& status, Ydb::Topic::DescribePartitionResult&& result) + : TStatus(std::move(status)) + , PartitionDescription_(std::move(result)) +{ +} + +const TPartitionDescription& TDescribePartitionResult::GetPartitionDescription() const { + return PartitionDescription_; +} + +TTopicDescription::TTopicDescription(Ydb::Topic::DescribeTopicResult&& result) + : Proto_(std::move(result)) + , PartitioningSettings_(Proto_.partitioning_settings()) + , RetentionPeriod_(TDuration::Seconds(Proto_.retention_period().seconds())) + , RetentionStorageMb_(Proto_.retention_storage_mb() > 0 ? std::optional(Proto_.retention_storage_mb()) : std::nullopt) + , PartitionWriteSpeedBytesPerSecond_(Proto_.partition_write_speed_bytes_per_second()) + , PartitionWriteBurstBytes_(Proto_.partition_write_burst_bytes()) + , MeteringMode_(TProtoAccessor::FromProto(Proto_.metering_mode())) + , TopicStats_(Proto_.topic_stats()) +{ + Owner_ = Proto_.self().owner(); + CreationTimestamp_ = NScheme::TVirtualTimestamp(Proto_.self().created_at()); + PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); + PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); + + for (const auto& part : Proto_.partitions()) { + Partitions_.emplace_back(part); + } + for (const auto& codec : Proto_.supported_codecs().codecs()) { + SupportedCodecs_.push_back((ECodec)codec); + } + for (const auto& pair : Proto_.attributes()) { + Attributes_[pair.first] = pair.second; + } + for (const auto& consumer : Proto_.consumers()) { + Consumers_.emplace_back(consumer); + } +} + +TConsumerDescription::TConsumerDescription(Ydb::Topic::DescribeConsumerResult&& result) + : Proto_(std::move(result)) + , Consumer_(Proto_.consumer()) +{ + for (const auto& part : Proto_.partitions()) { + Partitions_.emplace_back(part); + } +} + +TPartitionDescription::TPartitionDescription(Ydb::Topic::DescribePartitionResult&& result) + : Proto_(std::move(result)) + , Partition_(Proto_.partition()) +{ +} + +TConsumer::TConsumer(const Ydb::Topic::Consumer& consumer) + : ConsumerName_(consumer.name()) + , Important_(consumer.important()) + , ReadFrom_(TInstant::Seconds(consumer.read_from().seconds())) +{ + for (const auto& codec : consumer.supported_codecs().codecs()) { + SupportedCodecs_.push_back((ECodec)codec); + } + for (const auto& pair : consumer.attributes()) { + Attributes_[pair.first] = pair.second; + } +} + +const std::string& TConsumer::GetConsumerName() const { + return ConsumerName_; +} + +bool TConsumer::GetImportant() const { + return Important_; +} + +const TInstant& TConsumer::GetReadFrom() const { + return ReadFrom_; +} + +const std::vector& TConsumer::GetSupportedCodecs() const { + return SupportedCodecs_; +} + +const std::map& TConsumer::GetAttributes() const { + return Attributes_; +} + +const TPartitioningSettings& TTopicDescription::GetPartitioningSettings() const { + return PartitioningSettings_; +} + +uint32_t TTopicDescription::GetTotalPartitionsCount() const { + return Partitions_.size(); +} + +const std::vector& TTopicDescription::GetPartitions() const { + return Partitions_; +} + +const std::vector& TConsumerDescription::GetPartitions() const { + return Partitions_; +} + +const TPartitionInfo& TPartitionDescription::GetPartition() const { + return Partition_; +} + +const TConsumer& TConsumerDescription::GetConsumer() const { + return Consumer_; +} + +const std::vector& TTopicDescription::GetSupportedCodecs() const { + return SupportedCodecs_; +} + +const TDuration& TTopicDescription::GetRetentionPeriod() const { + return RetentionPeriod_; +} + +std::optional TTopicDescription::GetRetentionStorageMb() const { + return RetentionStorageMb_; +} + +uint64_t TTopicDescription::GetPartitionWriteSpeedBytesPerSecond() const { + return PartitionWriteSpeedBytesPerSecond_; +} + +uint64_t TTopicDescription::GetPartitionWriteBurstBytes() const { + return PartitionWriteBurstBytes_; +} + +EMeteringMode TTopicDescription::GetMeteringMode() const { + return MeteringMode_; +} + +const std::map& TTopicDescription::GetAttributes() const { + return Attributes_; +} + +const std::vector& TTopicDescription::GetConsumers() const { + return Consumers_; +} + +void TTopicDescription::SerializeTo(Ydb::Topic::CreateTopicRequest& request) const { + Y_UNUSED(request); + Y_ABORT("Not implemented"); +} + +const Ydb::Topic::DescribeTopicResult& TTopicDescription::GetProto() const { + return Proto_; +} + +const Ydb::Topic::DescribeConsumerResult& TConsumerDescription::GetProto() const { + return Proto_; +} + +const Ydb::Topic::DescribePartitionResult& TPartitionDescription::GetProto() const { + return Proto_; +} + +const std::string& TTopicDescription::GetOwner() const { + return Owner_; +} + +const NScheme::TVirtualTimestamp& TTopicDescription::GetCreationTimestamp() const { + return CreationTimestamp_; +} + +const TTopicStats& TTopicDescription::GetTopicStats() const { + return TopicStats_; +} + +const std::vector& TTopicDescription::GetPermissions() const { + return Permissions_; +} + +const std::vector& TTopicDescription::GetEffectivePermissions() const { + return EffectivePermissions_; +} + +TPartitioningSettings::TPartitioningSettings(const Ydb::Topic::PartitioningSettings& settings) + : MinActivePartitions_(settings.min_active_partitions()) + , MaxActivePartitions_(settings.max_active_partitions()) + , PartitionCountLimit_(settings.partition_count_limit()) + , AutoPartitioningSettings_(settings.auto_partitioning_settings()) +{} + +uint64_t TPartitioningSettings::GetMinActivePartitions() const { + return MinActivePartitions_; +} + +uint64_t TPartitioningSettings::GetMaxActivePartitions() const { + return MaxActivePartitions_; +} + +uint64_t TPartitioningSettings::GetPartitionCountLimit() const { + return PartitionCountLimit_; +} + +TAutoPartitioningSettings TPartitioningSettings::GetAutoPartitioningSettings() const { + return AutoPartitioningSettings_; +} + +TAutoPartitioningSettings::TAutoPartitioningSettings(const Ydb::Topic::AutoPartitioningSettings& settings) + : Strategy_(static_cast(settings.strategy())) + , StabilizationWindow_(TDuration::Seconds(settings.partition_write_speed().stabilization_window().seconds())) + , DownUtilizationPercent_(settings.partition_write_speed().down_utilization_percent()) + , UpUtilizationPercent_(settings.partition_write_speed().up_utilization_percent()) +{} + +EAutoPartitioningStrategy TAutoPartitioningSettings::GetStrategy() const { + return Strategy_; +} + +TDuration TAutoPartitioningSettings::GetStabilizationWindow() const { + return StabilizationWindow_; +} + +uint32_t TAutoPartitioningSettings::GetUpUtilizationPercent() const { + return UpUtilizationPercent_; +} + +uint32_t TAutoPartitioningSettings::GetDownUtilizationPercent() const { + return DownUtilizationPercent_; +} + +TTopicStats::TTopicStats(const Ydb::Topic::DescribeTopicResult::TopicStats& topicStats) + : StoreSizeBytes_(topicStats.store_size_bytes()) + , MinLastWriteTime_(TInstant::Seconds(topicStats.min_last_write_time().seconds())) + , MaxWriteTimeLag_(TDuration::Seconds(topicStats.max_write_time_lag().seconds()) + TDuration::MicroSeconds(topicStats.max_write_time_lag().nanos() / 1000)) + , BytesWrittenPerMinute_(topicStats.bytes_written().per_minute()) + , BytesWrittenPerHour_(topicStats.bytes_written().per_hour()) + , BytesWrittenPerDay_(topicStats.bytes_written().per_day()) +{ +} + +uint64_t TTopicStats::GetStoreSizeBytes() const { + return StoreSizeBytes_; +} + +TInstant TTopicStats::GetMinLastWriteTime() const { + return MinLastWriteTime_; +} + +TDuration TTopicStats::GetMaxWriteTimeLag() const { + return MaxWriteTimeLag_; +} + +uint64_t TTopicStats::GetBytesWrittenPerMinute() const { + return BytesWrittenPerMinute_; +} + +uint64_t TTopicStats::GetBytesWrittenPerHour() const { + return BytesWrittenPerHour_; +} + +uint64_t TTopicStats::GetBytesWrittenPerDay() const { + return BytesWrittenPerDay_; +} + + +TPartitionStats::TPartitionStats(const Ydb::Topic::PartitionStats& partitionStats) + : StartOffset_(partitionStats.partition_offsets().start()) + , EndOffset_(partitionStats.partition_offsets().end()) + , StoreSizeBytes_(partitionStats.store_size_bytes()) + , LastWriteTime_(TInstant::Seconds(partitionStats.last_write_time().seconds())) + , MaxWriteTimeLag_(TDuration::Seconds(partitionStats.max_write_time_lag().seconds()) + TDuration::MicroSeconds(partitionStats.max_write_time_lag().nanos() / 1000)) + , BytesWrittenPerMinute_(partitionStats.bytes_written().per_minute()) + , BytesWrittenPerHour_(partitionStats.bytes_written().per_hour()) + , BytesWrittenPerDay_(partitionStats.bytes_written().per_day()) + +{} + +uint64_t TPartitionStats::GetStartOffset() const { + return StartOffset_; +} + +uint64_t TPartitionStats::GetEndOffset() const { + return EndOffset_; +} + +uint64_t TPartitionStats::GetStoreSizeBytes() const { + return StoreSizeBytes_; +} + +TInstant TPartitionStats::GetLastWriteTime() const { + return LastWriteTime_; +} + +TDuration TPartitionStats::GetMaxWriteTimeLag() const { + return MaxWriteTimeLag_; +} + +uint64_t TPartitionStats::GetBytesWrittenPerMinute() const { + return BytesWrittenPerMinute_; +} + +uint64_t TPartitionStats::GetBytesWrittenPerHour() const { + return BytesWrittenPerHour_; +} + +uint64_t TPartitionStats::GetBytesWrittenPerDay() const { + return BytesWrittenPerDay_; +} + + +TPartitionConsumerStats::TPartitionConsumerStats(const Ydb::Topic::DescribeConsumerResult::PartitionConsumerStats& partitionStats) + : CommittedOffset_(partitionStats.committed_offset()) + , LastReadOffset_(partitionStats.last_read_offset()) + , ReaderName_(partitionStats.reader_name()) + , ReadSessionId_(partitionStats.read_session_id()) + , LastReadTime_(TInstant::Seconds(partitionStats.last_read_time().seconds())) + , MaxReadTimeLag_(TDuration::Seconds(partitionStats.max_read_time_lag().seconds())) + , MaxWriteTimeLag_(TDuration::Seconds(partitionStats.max_write_time_lag().seconds())) +{} + +uint64_t TPartitionConsumerStats::GetCommittedOffset() const { + return CommittedOffset_; +} + +uint64_t TPartitionConsumerStats::GetLastReadOffset() const { + return LastReadOffset_; +} + +std::string TPartitionConsumerStats::GetReaderName() const { + return ReaderName_; +} + +std::string TPartitionConsumerStats::GetReadSessionId() const { + return ReadSessionId_; +} + +const TInstant& TPartitionConsumerStats::GetLastReadTime() const { + return LastReadTime_; +} + +const TDuration& TPartitionConsumerStats::GetMaxReadTimeLag() const { + return MaxReadTimeLag_; +} + +const TDuration& TPartitionConsumerStats::GetMaxWriteTimeLag() const { + return MaxWriteTimeLag_; +} + +TPartitionLocation::TPartitionLocation(const Ydb::Topic::PartitionLocation& partitionLocation) + : NodeId_(partitionLocation.node_id()) + , Generation_(partitionLocation.generation()) +{ +} + +int32_t TPartitionLocation::GetNodeId() const { + return NodeId_; +} + +int64_t TPartitionLocation::GetGeneration() const { + return Generation_; +} + +TPartitionInfo::TPartitionInfo(const Ydb::Topic::DescribeTopicResult::PartitionInfo& partitionInfo) + : PartitionId_(partitionInfo.partition_id()) + , Active_(partitionInfo.active()) + , PartitionStats_() +{ + for (const auto& partId : partitionInfo.child_partition_ids()) { + ChildPartitionIds_.push_back(partId); + } + + for (const auto& partId : partitionInfo.parent_partition_ids()) { + ParentPartitionIds_.push_back(partId); + } + + if (partitionInfo.has_partition_stats()) { + PartitionStats_ = TPartitionStats{partitionInfo.partition_stats()}; + } + + if (partitionInfo.has_partition_location()) { + PartitionLocation_ = TPartitionLocation{partitionInfo.partition_location()}; + } + + if (partitionInfo.has_key_range() && partitionInfo.key_range().has_from_bound()) { + FromBound_ = std::string{partitionInfo.key_range().from_bound()}; + } + + if (partitionInfo.has_key_range() && partitionInfo.key_range().has_to_bound()) { + ToBound_ = std::string{partitionInfo.key_range().to_bound()}; + } +} + +TPartitionInfo::TPartitionInfo(const Ydb::Topic::DescribeConsumerResult::PartitionInfo& partitionInfo) + : PartitionId_(partitionInfo.partition_id()) + , Active_(partitionInfo.active()) + , PartitionStats_() +{ + for (const auto& partId : partitionInfo.child_partition_ids()) { + ChildPartitionIds_.push_back(partId); + } + + for (const auto& partId : partitionInfo.parent_partition_ids()) { + ParentPartitionIds_.push_back(partId); + } + if (partitionInfo.has_partition_stats()) { + PartitionStats_ = TPartitionStats{partitionInfo.partition_stats()}; + PartitionConsumerStats_ = TPartitionConsumerStats{partitionInfo.partition_consumer_stats()}; + } + if (partitionInfo.has_partition_location()) { + PartitionLocation_ = TPartitionLocation{partitionInfo.partition_location()}; + } +} + +const std::optional& TPartitionInfo::GetPartitionStats() const { + return PartitionStats_; +} + +const std::optional& TPartitionInfo::GetPartitionConsumerStats() const { + return PartitionConsumerStats_; +} + +const std::optional& TPartitionInfo::GetPartitionLocation() const { + return PartitionLocation_; +} + +const std::vector TPartitionInfo::GetChildPartitionIds() const { + return ChildPartitionIds_; +} + +const std::vector TPartitionInfo::GetParentPartitionIds() const { + return ParentPartitionIds_; +} + +bool TPartitionInfo::GetActive() const { + return Active_; +} + +uint64_t TPartitionInfo::GetPartitionId() const { + return PartitionId_; +} + +const std::optional& TPartitionInfo::GetFromBound() const { + return FromBound_; +} + +const std::optional& TPartitionInfo::GetToBound() const { + return ToBound_; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TTopicClient + +TTopicClient::TTopicClient(const TDriver& driver, const TTopicClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncStatus TTopicClient::CreateTopic(const std::string& path, const TCreateTopicSettings& settings) { + return Impl_->CreateTopic(path, settings); +} + +TAsyncStatus TTopicClient::AlterTopic(const std::string& path, const TAlterTopicSettings& settings) { + return Impl_->AlterTopic(path, settings); +} + +TAsyncStatus TTopicClient::DropTopic(const std::string& path, const TDropTopicSettings& settings) { + return Impl_->DropTopic(path, settings); +} + +TAsyncDescribeTopicResult TTopicClient::DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings) { + return Impl_->DescribeTopic(path, settings); +} + +TAsyncDescribeConsumerResult TTopicClient::DescribeConsumer(const std::string& path, const std::string& consumer, const TDescribeConsumerSettings& settings) { + return Impl_->DescribeConsumer(path, consumer, settings); +} + +TAsyncDescribePartitionResult TTopicClient::DescribePartition(const std::string& path, int64_t partitionId, const TDescribePartitionSettings& settings) { + return Impl_->DescribePartition(path, partitionId, settings); +} + +std::shared_ptr TTopicClient::CreateReadSession(const TReadSessionSettings& settings) { + return Impl_->CreateReadSession(settings); +} + +std::shared_ptr TTopicClient::CreateSimpleBlockingWriteSession( + const TWriteSessionSettings& settings) { + return Impl_->CreateSimpleWriteSession(settings); +} + +std::shared_ptr TTopicClient::CreateWriteSession(const TWriteSessionSettings& settings) { + return Impl_->CreateWriteSession(settings); +} + +TAsyncStatus TTopicClient::CommitOffset(const std::string& path, uint64_t partitionId, const std::string& consumerName, uint64_t offset, + const TCommitOffsetSettings& settings) { + return Impl_->CommitOffset(path, partitionId, consumerName, offset, settings); +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.cpp new file mode 100644 index 000000000000..b0ace7334fe0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.cpp @@ -0,0 +1,78 @@ +#include "topic_impl.h" + +#include "read_session.h" +#include "write_session.h" + +namespace NYdb::inline V3::NTopic { + +std::shared_ptr TTopicClient::TImpl::CreateReadSession(const TReadSessionSettings& settings) { + std::optional maybeSettings; + if (!settings.DecompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_) { + maybeSettings = settings; + std::lock_guard guard(Lock); + if (!settings.DecompressionExecutor_) { + maybeSettings->DecompressionExecutor(Settings.DefaultCompressionExecutor_); + } + if (!settings.EventHandlers_.HandlersExecutor_) { + maybeSettings->EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + } + } + auto session = std::make_shared(maybeSettings.value_or(settings), shared_from_this(), Connections_, DbDriverState_); + session->Start(); + return std::move(session); +} + +std::shared_ptr TTopicClient::TImpl::CreateWriteSession( + const TWriteSessionSettings& settings +) { + std::optional maybeSettings; + if (!settings.CompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_) { + maybeSettings = settings; + std::lock_guard guard(Lock); + if (!settings.CompressionExecutor_) { + maybeSettings->CompressionExecutor(Settings.DefaultCompressionExecutor_); + } + if (!settings.EventHandlers_.HandlersExecutor_) { + maybeSettings->EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + } + } + auto session = std::make_shared( + maybeSettings.value_or(settings), shared_from_this(), Connections_, DbDriverState_ + ); + session->Start(TDuration::Zero()); + return std::move(session); +} + +std::shared_ptr TTopicClient::TImpl::CreateSimpleWriteSession( + const TWriteSessionSettings& settings +) { + auto alteredSettings = settings; + { + std::lock_guard guard(Lock); + alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + if (!settings.CompressionExecutor_) { + alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_); + } + } + + auto session = std::make_shared( + alteredSettings, shared_from_this(), Connections_, DbDriverState_ + ); + return std::move(session); +} + +std::shared_ptr TTopicClient::TImpl::CreateReadSessionConnectionProcessorFactory() { + using TService = Ydb::Topic::V1::TopicService; + using TRequest = Ydb::Topic::StreamReadMessage::FromClient; + using TResponse = Ydb::Topic::StreamReadMessage::FromServer; + return CreateConnectionProcessorFactory(&TService::Stub::AsyncStreamRead, Connections_, DbDriverState_); +} + +std::shared_ptr TTopicClient::TImpl::CreateWriteSessionConnectionProcessorFactory() { + using TService = Ydb::Topic::V1::TopicService; + using TRequest = Ydb::Topic::StreamWriteMessage::FromClient; + using TResponse = Ydb::Topic::StreamWriteMessage::FromServer; + return CreateConnectionProcessorFactory(&TService::Stub::AsyncStreamWrite, Connections_, DbDriverState_); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h new file mode 100644 index 000000000000..9c12e2b523fa --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h @@ -0,0 +1,395 @@ +#pragma once + +#include "transaction.h" + +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include + +namespace NYdb::inline V3::NTopic { +struct TOffsetsRange { + ui64 Start; + ui64 End; +}; + +struct TPartitionOffsets { + ui64 PartitionId; + std::vector Offsets; +}; + +struct TTopicOffsets { + std::string Path; + std::vector Partitions; +}; + +struct TUpdateOffsetsInTransactionSettings : public TOperationRequestSettings { + using TOperationRequestSettings::TOperationRequestSettings; +}; + +class TTopicClient::TImpl : public TClientImplCommon { +public: + // Constructor for main client. + TImpl(std::shared_ptr connections, const TTopicClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + , Settings(settings) + { + } + + template + static void ConvertConsumerToProto(const TConsumerSettings& settings, Ydb::Topic::Consumer& consumerProto) { + consumerProto.set_name(TStringType{settings.ConsumerName_}); + consumerProto.set_important(settings.Important_); + consumerProto.mutable_read_from()->set_seconds(settings.ReadFrom_.Seconds()); + + for (const auto& codec : settings.SupportedCodecs_) { + consumerProto.mutable_supported_codecs()->add_codecs((static_cast(codec))); + } + for (auto& pair : settings.Attributes_) { + (*consumerProto.mutable_attributes())[pair.first] = pair.second; + } + } + + static void ConvertAlterConsumerToProto(const TAlterConsumerSettings& settings, Ydb::Topic::AlterConsumer& consumerProto) { + consumerProto.set_name(TStringType{settings.ConsumerName_}); + if (settings.SetImportant_) + consumerProto.set_set_important(*settings.SetImportant_); + if (settings.SetReadFrom_) + consumerProto.mutable_set_read_from()->set_seconds(settings.SetReadFrom_->Seconds()); + + if (settings.SetSupportedCodecs_) { + for (const auto& codec : *settings.SetSupportedCodecs_) { + consumerProto.mutable_set_supported_codecs()->add_codecs((static_cast(codec))); + } + } + + for (auto& pair : settings.AlterAttributes_) { + (*consumerProto.mutable_alter_attributes())[pair.first] = pair.second; + } + } + + + static Ydb::Topic::CreateTopicRequest MakePropsCreateRequest(const std::string& path, const TCreateTopicSettings& settings) { + Ydb::Topic::CreateTopicRequest request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + request.mutable_partitioning_settings()->set_min_active_partitions(settings.PartitioningSettings_.GetMinActivePartitions()); + request.mutable_partitioning_settings()->set_partition_count_limit(settings.PartitioningSettings_.GetPartitionCountLimit()); + request.mutable_partitioning_settings()->set_max_active_partitions(settings.PartitioningSettings_.GetMaxActivePartitions()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->set_strategy(static_cast(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetStrategy())); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->mutable_stabilization_window()->set_seconds(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetStabilizationWindow().Seconds()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_up_utilization_percent(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetUpUtilizationPercent()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_down_utilization_percent(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetDownUtilizationPercent()); + + request.mutable_retention_period()->set_seconds(settings.RetentionPeriod_.Seconds()); + + for (const auto& codec : settings.SupportedCodecs_) { + request.mutable_supported_codecs()->add_codecs((static_cast(codec))); + } + request.set_partition_write_speed_bytes_per_second(settings.PartitionWriteSpeedBytesPerSecond_); + request.set_partition_write_burst_bytes(settings.PartitionWriteBurstBytes_); + request.set_retention_storage_mb(settings.RetentionStorageMb_); + request.set_metering_mode(TProtoAccessor::GetProto(settings.MeteringMode_)); + + for (auto& pair : settings.Attributes_) { + (*request.mutable_attributes())[pair.first] = pair.second; + } + + for (const auto& consumer : settings.Consumers_) { + Ydb::Topic::Consumer& consumerProto = *request.add_consumers(); + ConvertConsumerToProto(consumer, consumerProto); + } + + return request; + } + + TAsyncStatus CreateTopic(const std::string& path, const TCreateTopicSettings& settings) { + auto request = MakePropsCreateRequest(path, settings); + + return RunSimple( + std::move(request), + &Ydb::Topic::V1::TopicService::Stub::AsyncCreateTopic, + TRpcRequestSettings::Make(settings)); + } + + + static Ydb::Topic::AlterTopicRequest MakePropsAlterRequest(const std::string& path, const TAlterTopicSettings& settings) { + Ydb::Topic::AlterTopicRequest request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + if (settings.AlterPartitioningSettings_) { + if (settings.AlterPartitioningSettings_->MinActivePartitions_) { + request.mutable_alter_partitioning_settings()->set_set_min_active_partitions(*settings.AlterPartitioningSettings_->MinActivePartitions_); + } + if (settings.AlterPartitioningSettings_->MaxActivePartitions_) { + request.mutable_alter_partitioning_settings()->set_set_max_active_partitions(*settings.AlterPartitioningSettings_->MaxActivePartitions_); + } + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_) { + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->Strategy_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->set_set_strategy(static_cast(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->Strategy_)); + } + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->DownUtilizationPercent_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->set_set_down_utilization_percent(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->DownUtilizationPercent_); + } + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->UpUtilizationPercent_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->set_set_up_utilization_percent(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->UpUtilizationPercent_); + } + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->StabilizationWindow_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->mutable_set_stabilization_window()->set_seconds(settings.AlterPartitioningSettings_->AutoPartitioningSettings_->StabilizationWindow_->Seconds()); + } + } + } + if (settings.SetRetentionPeriod_) { + request.mutable_set_retention_period()->set_seconds(settings.SetRetentionPeriod_->Seconds()); + } + if (settings.SetSupportedCodecs_) { + for (const auto& codec : *settings.SetSupportedCodecs_) { + request.mutable_set_supported_codecs()->add_codecs((static_cast(codec))); + } + } + if (settings.SetPartitionWriteSpeedBytesPerSecond_) { + request.set_set_partition_write_speed_bytes_per_second(*settings.SetPartitionWriteSpeedBytesPerSecond_); + } + if (settings.SetPartitionWriteBurstBytes_) { + request.set_set_partition_write_burst_bytes(*settings.SetPartitionWriteBurstBytes_); + } + if (settings.SetRetentionStorageMb_) { + request.set_set_retention_storage_mb(*settings.SetRetentionStorageMb_); + } + if (settings.SetMeteringMode_) { + request.set_set_metering_mode(TProtoAccessor::GetProto(*settings.SetMeteringMode_)); + } + + for (auto& pair : settings.AlterAttributes_) { + (*request.mutable_alter_attributes())[pair.first] = pair.second; + } + + for (const auto& consumer : settings.AddConsumers_) { + Ydb::Topic::Consumer& consumerProto = *request.add_add_consumers(); + ConvertConsumerToProto(consumer, consumerProto); + } + + for (const auto& consumer : settings.DropConsumers_) { + request.add_drop_consumers(TStringType{consumer}); + } + + for (const auto& consumer : settings.AlterConsumers_) { + Ydb::Topic::AlterConsumer& consumerProto = *request.add_alter_consumers(); + ConvertAlterConsumerToProto(consumer, consumerProto); + } + + return request; + } + + + TAsyncStatus AlterTopic(const std::string& path, const TAlterTopicSettings& settings) { + auto request = MakePropsAlterRequest(path, settings); + + return RunSimple( + std::move(request), + &Ydb::Topic::V1::TopicService::Stub::AsyncAlterTopic, + TRpcRequestSettings::Make(settings)); + } + + + TAsyncStatus DropTopic(const std::string& path, const TDropTopicSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + return RunSimple( + std::move(request), + &Ydb::Topic::V1::TopicService::Stub::AsyncDropTopic, + TRpcRequestSettings::Make(settings)); + } + + TAsyncDescribeTopicResult DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + + if (settings.IncludeStats_) { + request.set_include_stats(true); + } + + if (settings.IncludeLocation_) { + request.set_include_location(true); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Topic::DescribeTopicResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeTopicResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Topic::V1::TopicService::Stub::AsyncDescribeTopic, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncDescribeConsumerResult DescribeConsumer(const std::string& path, const std::string& consumer, const TDescribeConsumerSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + request.set_consumer(TStringType{consumer}); + + if (settings.IncludeStats_) { + request.set_include_stats(true); + } + + if (settings.IncludeLocation_) { + request.set_include_location(true); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Topic::DescribeConsumerResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeConsumerResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Topic::V1::TopicService::Stub::AsyncDescribeConsumer, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncDescribePartitionResult DescribePartition(const std::string& path, i64 partitionId, const TDescribePartitionSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + request.set_partition_id(partitionId); + + if (settings.IncludeStats_) { + request.set_include_stats(true); + } + + if (settings.IncludeLocation_) { + request.set_include_location(true); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise](google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Topic::DescribePartitionResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribePartitionResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Topic::V1::TopicService::Stub::AsyncDescribePartition, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncStatus CommitOffset(const std::string& path, ui64 partitionId, const std::string& consumerName, ui64 offset, + const TCommitOffsetSettings& settings) { + Ydb::Topic::CommitOffsetRequest request = MakeOperationRequest(settings); + request.set_path(TStringType{path}); + request.set_partition_id(partitionId); + request.set_consumer(TStringType{consumerName}); + request.set_offset(offset); + + return RunSimple( + std::move(request), + &Ydb::Topic::V1::TopicService::Stub::AsyncCommitOffset, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus UpdateOffsetsInTransaction(const TTransactionId& tx, + const std::vector& topics, + const std::string& consumerName, + const TUpdateOffsetsInTransactionSettings& settings) + { + auto request = MakeOperationRequest(settings); + + request.mutable_tx()->set_id(TStringType{GetTxId(tx)}); + request.mutable_tx()->set_session(TStringType{GetSessionId(tx)}); + + for (auto& t : topics) { + auto* topic = request.mutable_topics()->Add(); + topic->set_path(TStringType{t.Path}); + + for (auto& p : t.Partitions) { + auto* partition = topic->mutable_partitions()->Add(); + partition->set_partition_id(p.PartitionId); + + for (auto& r : p.Offsets) { + auto *range = partition->mutable_partition_offsets()->Add(); + range->set_start(r.Start); + range->set_end(r.End); + } + } + } + + request.set_consumer(TStringType{consumerName}); + + return RunSimple( + std::move(request), + &Ydb::Topic::V1::TopicService::Stub::AsyncUpdateOffsetsInTransaction, + TRpcRequestSettings::Make(settings) + ); + } + + // Runtime API. + std::shared_ptr CreateReadSession(const TReadSessionSettings& settings); + std::shared_ptr CreateSimpleWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr CreateWriteSession(const TWriteSessionSettings& settings); + + using IReadSessionConnectionProcessorFactory = + ISessionConnectionProcessorFactory; + + std::shared_ptr CreateReadSessionConnectionProcessorFactory(); + + using IWriteSessionConnectionProcessorFactory = + ISessionConnectionProcessorFactory; + + std::shared_ptr CreateWriteSessionConnectionProcessorFactory(); + + NYdbGrpc::IQueueClientContextPtr CreateContext() { + return Connections_->CreateContext(); + } + +private: + const TTopicClientSettings Settings; + TAdaptiveLock Lock; +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/transaction.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/transaction.cpp new file mode 100644 index 000000000000..662a4e61218d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/transaction.cpp @@ -0,0 +1,26 @@ +#include "transaction.h" +#include + +namespace NYdb::inline V3::NTopic { + +TTransactionId MakeTransactionId(const NTable::TTransaction& tx) +{ + return {tx.GetSession().GetId(), tx.GetId()}; +} + +TStatus MakeStatus(EStatus code, NYdb::NIssue::TIssues&& issues) +{ + return {code, std::move(issues)}; +} + +TStatus MakeSessionExpiredError() +{ + return MakeStatus(EStatus::SESSION_EXPIRED, {}); +} + +TStatus MakeCommitTransactionSuccess() +{ + return MakeStatus(EStatus::SUCCESS, {}); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/transaction.h b/ydb/public/sdk/cpp/src/client/topic/impl/transaction.h new file mode 100644 index 000000000000..721ecf619380 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/transaction.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NTable { + +class TTransaction; + +} + +namespace NYdb::inline V3::NTopic { + +using TTransactionId = std::pair; + +inline +const std::string& GetSessionId(const TTransactionId& x) +{ + return x.first; +} + +inline +const std::string& GetTxId(const TTransactionId& x) +{ + return x.second; +} + +TTransactionId MakeTransactionId(const NTable::TTransaction& tx); + +TStatus MakeSessionExpiredError(); +TStatus MakeCommitTransactionSuccess(); + +} diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/write_session.cpp new file mode 100644 index 000000000000..b19d03eae48d --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session.cpp @@ -0,0 +1,181 @@ +#include "write_session.h" + +#include + +namespace NYdb::inline V3::NTopic { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSession + +TWriteSession::TWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : TContextOwner(settings, std::move(client), std::move(connections), std::move(dbDriverState)) { +} + +void TWriteSession::Start(const TDuration& delay) { + TryGetImpl()->Start(delay); +} + +NThreading::TFuture TWriteSession::GetInitSeqNo() { + return TryGetImpl()->GetInitSeqNo(); +} + +std::optional TWriteSession::GetEvent(bool block) { + return TryGetImpl()->EventsQueue->GetEvent(block); +} + +std::vector TWriteSession::GetEvents(bool block, std::optional maxEventsCount) { + return TryGetImpl()->EventsQueue->GetEvents(block, maxEventsCount); +} + +NThreading::TFuture TWriteSession::WaitEvent() { + return TryGetImpl()->EventsQueue->WaitEvent(); +} + +void TWriteSession::WriteEncoded(TContinuationToken&& token, std::string_view data, ECodec codec, ui32 originalSize, + std::optional seqNo, std::optional createTimestamp) { + auto message = TWriteMessage::CompressedMessage(std::move(data), codec, originalSize); + if (seqNo.has_value()) + message.SeqNo(*seqNo); + if (createTimestamp.has_value()) + message.CreateTimestamp(*createTimestamp); + TryGetImpl()->WriteInternal(std::move(token), std::move(message)); +} + +void TWriteSession::WriteEncoded(TContinuationToken&& token, TWriteMessage&& message, + NTable::TTransaction* tx) +{ + if (tx) { + message.Tx(*tx); + } + TryGetImpl()->WriteInternal(std::move(token), std::move(message)); +} + +void TWriteSession::Write(TContinuationToken&& token, std::string_view data, std::optional seqNo, + std::optional createTimestamp) { + TWriteMessage message{std::move(data)}; + if (seqNo.has_value()) + message.SeqNo(*seqNo); + if (createTimestamp.has_value()) + message.CreateTimestamp(*createTimestamp); + TryGetImpl()->WriteInternal(std::move(token), std::move(message)); +} + +void TWriteSession::Write(TContinuationToken&& token, TWriteMessage&& message, + NTable::TTransaction* tx) { + if (tx) { + message.Tx(*tx); + } + TryGetImpl()->WriteInternal(std::move(token), std::move(message)); +} + +bool TWriteSession::Close(TDuration closeTimeout) { + return TryGetImpl()->Close(closeTimeout); +} + +TWriteSession::~TWriteSession() { + TryGetImpl()->Close(TDuration::Zero()); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TSimpleBlockingWriteSession + +TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState +) { + auto subSettings = settings; + if (settings.EventHandlers_.AcksHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use AcksHandler, resetting."); + subSettings.EventHandlers_.AcksHandler({}); + } + if (settings.EventHandlers_.ReadyToAcceptHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use ReadyToAcceptHandler, resetting."); + subSettings.EventHandlers_.ReadyToAcceptHandler({}); + } + if (settings.EventHandlers_.SessionClosedHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use SessionClosedHandler, resetting."); + subSettings.EventHandlers_.SessionClosedHandler({}); + } + if (settings.EventHandlers_.CommonHandler_) { + LOG_LAZY(dbDriverState->Log, TLOG_WARNING, "TSimpleBlockingWriteSession: Cannot use CommonHandler, resetting."); + subSettings.EventHandlers_.CommonHandler({}); + } + Writer = std::make_shared(subSettings, client, connections, dbDriverState); + Writer->Start(TDuration::Max()); +} + +uint64_t TSimpleBlockingWriteSession::GetInitSeqNo() { + return Writer->GetInitSeqNo().GetValueSync(); +} + +bool TSimpleBlockingWriteSession::Write( + std::string_view data, std::optional seqNo, std::optional createTimestamp, const TDuration& blockTimeout +) { + auto message = TWriteMessage(std::move(data)) + .SeqNo(seqNo) + .CreateTimestamp(createTimestamp); + return Write(std::move(message), nullptr, blockTimeout); +} + +bool TSimpleBlockingWriteSession::Write( + TWriteMessage&& message, NTable::TTransaction* tx, const TDuration& blockTimeout +) { + auto continuationToken = WaitForToken(blockTimeout); + if (continuationToken.has_value()) { + Writer->Write(std::move(*continuationToken), std::move(message), tx); + return true; + } + return false; +} + +std::optional TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) { + TInstant startTime = TInstant::Now(); + TDuration remainingTime = timeout; + + std::optional token = std::nullopt; + + while (IsAlive() && remainingTime > TDuration::Zero()) { + Writer->WaitEvent().Wait(remainingTime); + + for (auto event : Writer->GetEvents()) { + if (auto* readyEvent = std::get_if(&event)) { + Y_ABORT_UNLESS(!token.has_value()); + token = std::move(readyEvent->ContinuationToken); + } else if (auto* ackEvent = std::get_if(&event)) { + // discard + } else if (auto* closeSessionEvent = std::get_if(&event)) { + Closed.store(true); + return std::nullopt; + } + } + + if (token.has_value()) { + return token; + } + + remainingTime = timeout - (TInstant::Now() - startTime); + } + + return std::nullopt; +} + +TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() { + return Writer->GetCounters(); +} + +bool TSimpleBlockingWriteSession::IsAlive() const { + return !Closed.load(); +} + +bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) { + Closed.store(true); + return Writer->Close(std::move(closeTimeout)); +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session.h b/ydb/public/sdk/cpp/src/client/topic/impl/write_session.h new file mode 100644 index 000000000000..77beb2898dfe --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3::NTopic { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSession + +class TWriteSession : public IWriteSession, + public TContextOwner { +private: + friend class TSimpleBlockingWriteSession; + friend class TTopicClient; + +public: + TWriteSession(const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + std::optional GetEvent(bool block = false) override; + std::vector GetEvents(bool block = false, + std::optional maxEventsCount = std::nullopt) override; + NThreading::TFuture GetInitSeqNo() override; + + void Write(TContinuationToken&& continuationToken, std::string_view data, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + + void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, ui32 originalSize, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + + void Write(TContinuationToken&& continuationToken, TWriteMessage&& message, + NTable::TTransaction* tx = nullptr) override; + + void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& message, + NTable::TTransaction* tx = nullptr) override; + + NThreading::TFuture WaitEvent() override; + + // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. + bool Close(TDuration closeTimeout = TDuration::Max()) override; + + TWriterCounters::TPtr GetCounters() override {Y_ABORT("Unimplemented"); } //ToDo - unimplemented; + + ~TWriteSession(); // will not call close - destroy everything without acks + +private: + void Start(const TDuration& delay); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TSimpleBlockingWriteSession + +class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { +public: + TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, + const TDuration& blockTimeout = TDuration::Max()) override; + + bool Write(TWriteMessage&& message, + NTable::TTransaction* tx = nullptr, + const TDuration& blockTimeout = TDuration::Max()) override; + + uint64_t GetInitSeqNo() override; + + bool Close(TDuration closeTimeout = TDuration::Max()) override; + bool IsAlive() const override; + + TWriterCounters::TPtr GetCounters() override; + +protected: + std::shared_ptr Writer; + +private: + std::optional WaitForToken(const TDuration& timeout); + + std::atomic_bool Closed = false; +}; + + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp new file mode 100644 index 000000000000..0401e17f43d3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp @@ -0,0 +1,1748 @@ +#include "write_session_impl.h" + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +namespace NYdb::inline V3::NTopic { + +const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); +// Error code from file ydb/public/api/protos/persqueue_error_codes_v1.proto +const uint64_t WRITE_ERROR_PARTITION_INACTIVE = 500029; + +namespace { + +using TTxId = std::pair; +using TTxIdOpt = std::optional; + +TTxIdOpt GetTransactionId(const Ydb::Topic::StreamWriteMessage_WriteRequest& request) +{ + Y_ABORT_UNLESS(request.messages_size()); + + if (!request.has_tx()) { + return std::nullopt; + } + + const Ydb::Topic::TransactionIdentity& tx = request.tx(); + return TTxId(tx.session(), tx.id()); +} + +TTxIdOpt GetTransactionId(const NTable::TTransaction* tx) +{ + if (!tx) { + return std::nullopt; + } + + return TTxId(tx->GetSession().GetId(), tx->GetId()); +} + +} + +namespace NCompressionDetails { + std::unique_ptr CreateCoder(ECodec codec, TBuffer& result, int quality); +} + +namespace NCompressionDetails { + std::unique_ptr CreateCoder(ECodec codec, TBuffer& result, int quality); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSessionImpl + +TWriteSessionImpl::TWriteSessionImpl( + const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState) + : Settings(settings) + , Client(std::move(client)) + , Connections(std::move(connections)) + , DbDriverState(std::move(dbDriverState)) + , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") + , InitSeqNoPromise(NThreading::NewPromise()) + , WakeupInterval( + Settings.BatchFlushInterval_.value_or(TDuration::Zero()) ? + std::min(Settings.BatchFlushInterval_.value_or(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100)) + : + TDuration::MilliSeconds(100) + ) +{ + if (!Settings.RetryPolicy_) { + Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); + } + if (Settings.Counters_.has_value()) { + Counters = *Settings.Counters_; + } else { + Counters = MakeIntrusive(new ::NMonitoring::TDynamicCounters()); + } +} + +void TWriteSessionImpl::Start(const TDuration& delay) { + Y_ABORT_UNLESS(SelfContext); + + if (!EventsQueue) { +#define WRAP_HANDLER(type, handler, ...) \ + if (auto h = Settings.EventHandlers_.handler##_) { \ + Settings.EventHandlers_.handler([ctx = SelfContext, h = std::move(h)](__VA_ARGS__ type& ev){ \ + if (auto self = ctx->LockShared()) { \ + h(ev); \ + } \ + }); \ + } + WRAP_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler); + WRAP_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHandler); + WRAP_HANDLER(TSessionClosedEvent, SessionClosedHandler, const); + WRAP_HANDLER(TWriteSessionEvent::TEvent, CommonHandler); +#undef WRAP_HANDLER + + EventsQueue = std::make_shared(Settings); + } + + if (!Started) { + with_lock(Lock) { + HandleWakeUpImpl(); + } + InitWriter(); + } + with_lock(Lock) { + ++ConnectionAttemptsDone; + Started = true; + if (Settings.DirectWriteToPartition_ && (Settings.PartitionId_.has_value() || DirectWriteToPartitionId.has_value())) { + PreferredPartitionLocation = {}; + ConnectToPreferredPartitionLocation(delay); + return; + } + } + Connect(delay); +} + +// Returns true if we need to switch to another DirectWriteToPartitionId. +bool NeedToSwitchPartition(const TPlainStatus& status) { + switch (status.Status) { + // Server statuses: + case EStatus::OVERLOADED: + // In general OVERLOADED is temporary, but it's also returned on partition split/merge, + // in which case we need to switch to another partition. + for (auto const& issue : status.Issues) { + if (issue.IssueCode == WRITE_ERROR_PARTITION_INACTIVE) { + return true; + } + } + case EStatus::UNAUTHORIZED: + case EStatus::SUCCESS: + case EStatus::UNAVAILABLE: + case EStatus::SESSION_EXPIRED: + case EStatus::CANCELLED: + case EStatus::UNDETERMINED: + case EStatus::SESSION_BUSY: + case EStatus::TIMEOUT: + + // Client statuses: + case EStatus::TRANSPORT_UNAVAILABLE: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::CLIENT_DEADLINE_EXCEEDED: + case EStatus::CLIENT_INTERNAL_ERROR: + case EStatus::CLIENT_OUT_OF_RANGE: + case EStatus::CLIENT_LIMITS_REACHED: + case EStatus::CLIENT_DISCOVERY_FAILED: + return false; + default: + return true; + } +} + +TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStatus& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + TRACE_LAZY(DbDriverState->Log, "Error", + TRACE_KV("status", status.Status)); + + THandleResult result; + if (Aborting.load()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session is aborting and will not restart"); + return result; + } + SessionEstablished = false; + + // Keep DirectWriteToPartitionId value on temporary errors. + if (DirectWriteToPartitionId.has_value() && NeedToSwitchPartition(status)) { + TRACE_LAZY(DbDriverState->Log, "ClearDirectWriteToPartitionId"); + DirectWriteToPartitionId.reset(); + // We need to clear PreferredPartitionLocation here, because in Start, + // with both Settings.PartitionId_ and DirectWriteToPartitionId undefined, + // Connect is called, and PreferredPartitionLocation is used there to fill in the reqSettings. + PreferredPartitionLocation = {}; + } + + if (!RetryState) { + RetryState = Settings.RetryPolicy_->CreateRetryState(); + } + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); + + if (nextDelay) { + result.StartDelay = *nextDelay; + result.DoRestart = true; + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will restart in " << result.StartDelay); + ResetForRetryImpl(); + } else { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session will not restart after a fatal error"); + result.DoStop = true; + CheckHandleResultImpl(result); + } + return result; +} + +std::string FullTopicPath(const std::string& dbPath, std::string_view topic) { + if (topic.starts_with(dbPath)) { + return std::string(topic); + } + std::string full; + full.reserve(dbPath.size() + 1 + topic.size()); + full.append(dbPath); + if (!full.ends_with('/')) { + full.push_back('/'); + } + if (topic.starts_with('/')) { + topic = topic.substr(1); + } + full.append(topic); + return full; +} + +void TWriteSessionImpl::ConnectToPreferredPartitionLocation(const TDuration& delay) +{ + Y_ABORT_UNLESS(Lock.IsLocked()); + Y_ABORT_UNLESS(Settings.DirectWriteToPartition_ && (Settings.PartitionId_.has_value() || DirectWriteToPartitionId.has_value())); + + if (Aborting.load()) { + return; + } + + auto partition_id = Settings.PartitionId_.has_value() ? *Settings.PartitionId_ : *DirectWriteToPartitionId; + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Get partition location async, partition " << partition_id << ", delay " << delay ); + + NYdbGrpc::IQueueClientContextPtr prevDescribePartitionContext; + NYdbGrpc::IQueueClientContextPtr describePartitionContext = Client->CreateContext(); + + if (!describePartitionContext) { + AbortImpl(); + return; + } + + ++ConnectionGeneration; + + prevDescribePartitionContext = std::exchange(DescribePartitionContext, describePartitionContext); + Y_ASSERT(DescribePartitionContext); + Cancel(prevDescribePartitionContext); + + Ydb::Topic::DescribePartitionRequest request; + // Currently, the whole topic path needs to be sent in the DescribePartitionRequest. + request.set_path(FullTopicPath(DbDriverState->Database, Settings.Path_)); + request.set_partition_id(partition_id); + request.set_include_location(true); + + TRACE_LAZY(DbDriverState->Log, "DescribePartitionRequest", + TRACE_KV("path", request.path()), + TRACE_KV("partition_id", request.partition_id())); + + auto extractor = [cbContext = SelfContext, context = describePartitionContext](Ydb::Topic::DescribePartitionResponse* response, TPlainStatus status) mutable { + Ydb::Topic::DescribePartitionResult result; + if (response) + response->operation().result().UnpackTo(&result); + + TStatus st = status.Status == EStatus::SUCCESS ? MakeErrorFromProto(response->operation()) : std::move(status); + if (auto self = cbContext->LockShared()) { + self->OnDescribePartition(st, result, context); + } + }; + + auto callback = [req = std::move(request), extr = std::move(extractor), + connections = std::shared_ptr(Connections), dbState = DbDriverState, + context = describePartitionContext, prefix = std::string(LogPrefix()), + partId = partition_id]() mutable { + LOG_LAZY(dbState->Log, TLOG_DEBUG, prefix + " Getting partition location, partition " + ToString(partId)); + connections->Run( + std::move(req), + std::move(extr), + &Ydb::Topic::V1::TopicService::Stub::AsyncDescribePartition, + dbState, + {}, + context); + }; + + Connections->ScheduleOneTimeTask(std::move(callback), delay); +} + +void TWriteSessionImpl::OnDescribePartition(const TStatus& status, const Ydb::Topic::DescribePartitionResult& proto, const NYdbGrpc::IQueueClientContextPtr& describePartitionContext) +{ + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got PartitionLocation response. Status " << status.GetStatus() << ", proto:\n" << proto.DebugString()); + std::string endpoint, name; + THandleResult handleResult; + + with_lock(Lock) { + if (DescribePartitionContext == describePartitionContext) { + DescribePartitionContext = nullptr; + } else { + return; + } + } + + if (!status.IsSuccess()) { + with_lock(Lock) { + if (status.GetStatus() == EStatus::CLIENT_CALL_UNIMPLEMENTED) { + Settings.DirectWriteToPartition_ = false; + handleResult = OnErrorImpl({ + EStatus::UNAVAILABLE, + MakeIssueWithSubIssues("The server does not support direct write, fallback to in-direct write", status.GetIssues()) + }); + } else { + handleResult = OnErrorImpl({status.GetStatus(), MakeIssueWithSubIssues("Failed to get partition location", status.GetIssues())}); + } + } + ProcessHandleResult(handleResult); + return; + } + + const Ydb::Topic::DescribeTopicResult_PartitionInfo& partition = proto.partition(); + + TRACE_LAZY(DbDriverState->Log, "DescribePartitionResponse", + TRACE_KV("partition_id", partition.partition_id()), + TRACE_KV("active", partition.active()), + TRACE_KV("pl_node_id", partition.partition_location().node_id()), + TRACE_KV("pl_generation", partition.partition_location().generation())); + + if (partition.partition_id() != Settings.PartitionId_ && Settings.PartitionId_.has_value() || + !partition.has_partition_location() || partition.partition_location().node_id() == 0 || partition.partition_location().generation() == 0) { + { + std::lock_guard guard(Lock); + handleResult = OnErrorImpl({EStatus::INTERNAL_ERROR, "Wrong partition location"}); + } + ProcessHandleResult(handleResult); + return; + } + + std::optional preferredEndpoint; + { + std::lock_guard guard(Lock); + preferredEndpoint = GetPreferredEndpointImpl(partition.partition_id(), partition.partition_location().node_id()); + } + + if (!preferredEndpoint.has_value()) { + { + std::lock_guard guard(Lock); + handleResult = OnErrorImpl({EStatus::UNAVAILABLE, "Partition preferred endpoint is not found"}); + } + ProcessHandleResult(handleResult); + return; + } + + { + std::lock_guard guard(Lock); + PreferredPartitionLocation = {*preferredEndpoint, partition.partition_location().generation()}; + } + + TRACE_LAZY(DbDriverState->Log, "PreferredPartitionLocation", + TRACE_KV("Endpoint", PreferredPartitionLocation.Endpoint.Endpoint), + TRACE_KV("NodeId", PreferredPartitionLocation.Endpoint.NodeId), + TRACE_KV("Generation", PreferredPartitionLocation.Generation)); + + Connect(TDuration::Zero()); +} + +std::optional TWriteSessionImpl::GetPreferredEndpointImpl(ui32 partitionId, uint64_t partitionNodeId) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + TEndpointKey preferredEndpoint{"", partitionNodeId}; + + bool nodeIsKnown = (bool)DbDriverState->EndpointPool.GetEndpoint(preferredEndpoint, true); + if (nodeIsKnown) + { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "GetPreferredEndpoint: partitionId " << partitionId << ", partitionNodeId " << partitionNodeId << " exists in the endpoint pool."); + return preferredEndpoint; + } + else + { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "GetPreferredEndpoint: partitionId " << partitionId << ", nodeId " << partitionNodeId << " does not exist in the endpoint pool."); + DbDriverState->EndpointPool.UpdateAsync(); + return {}; + } +} + +std::string GenerateProducerId() { + return CreateGuidAsString(); +} + +void TWriteSessionImpl::InitWriter() { // No Lock, very initial start - no race yet as well. + if (!Settings.DeduplicationEnabled_.has_value()) { + // Deduplication settings not provided - will enable deduplication if ProducerId or MessageGroupId is provided. + Settings.DeduplicationEnabled_ = !Settings.ProducerId_.empty() || !Settings.MessageGroupId_.empty(); + } else if (*Settings.DeduplicationEnabled_) { + // Deduplication explicitly enabled. + + // If both are provided, will validate they are equal in the check below. + if (Settings.ProducerId_.empty()) { + if (Settings.MessageGroupId_.empty()) { + // Both ProducerId and MessageGroupId are empty, will generate random string and use it + Settings.MessageGroupId(GenerateProducerId()); + } + // MessageGroupId is non-empty (either provided by user of generated above) and ProducerId is empty, copy value there. + Settings.ProducerId(Settings.MessageGroupId_); + } else if (Settings.MessageGroupId_.empty()) { + // MessageGroupId is empty, copy ProducerId value. + Settings.MessageGroupId(Settings.ProducerId_); + } + } else { + // Deduplication explicitly disabled, ProducerId & MessageGroupId must be empty. + if (!Settings.ProducerId_.empty() || !Settings.MessageGroupId_.empty()) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() + << "ProducerId or MessageGroupId is not empty when deduplication is switched off"); + ThrowFatalError("Explicitly disabled deduplication conflicts with non-empty ProducerId or MessageGroupId"); + } + } + if (!Settings.ProducerId_.empty() && !Settings.MessageGroupId_.empty() && Settings.ProducerId_ != Settings.MessageGroupId_) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() + << "ProducerId and MessageGroupId mismatch"); + ThrowFatalError("ProducerId != MessageGroupId scenario is currently not supported"); + } + CompressionExecutor = Settings.CompressionExecutor_; + + Settings.CompressionExecutor_->Start(); + Settings.EventHandlers_.HandlersExecutor_->Start(); + +} +// Client method +NThreading::TFuture TWriteSessionImpl::GetInitSeqNo() { + if (!Settings.DeduplicationEnabled_.value_or(true)) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "GetInitSeqNo called with deduplication disabled"); + ThrowFatalError("Cannot call GetInitSeqNo when deduplication is disabled"); + } + if (Settings.ValidateSeqNo_) { + if (AutoSeqNoMode.has_value() && *AutoSeqNoMode) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Cannot call GetInitSeqNo in Auto SeqNo mode"); + ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode"); + } + else + AutoSeqNoMode = false; + } + return InitSeqNoPromise.GetFuture(); +} + +std::string DebugString(const TWriteSessionEvent::TEvent& event) { + return std::visit([](const auto& ev) { return ev.DebugString(); }, event); +} + +// Client method +std::optional TWriteSessionImpl::GetEvent(bool block) { + return EventsQueue->GetEvent(block); +} + +// Client method +std::vector TWriteSessionImpl::GetEvents(bool block, std::optional maxEventsCount) { + return EventsQueue->GetEvents(block, maxEventsCount); +} + +uint64_t TWriteSessionImpl::GetIdImpl(uint64_t seqNo) { + Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); + Y_ABORT_UNLESS(!*AutoSeqNoMode || InitSeqNo.has_value() && seqNo > *InitSeqNo); + return *AutoSeqNoMode ? seqNo - *InitSeqNo : seqNo; +} + +uint64_t TWriteSessionImpl::GetSeqNoImpl(uint64_t id) { + Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); + Y_ABORT_UNLESS(InitSeqNo.has_value()); + return *AutoSeqNoMode ? id + *InitSeqNo : id; + +} + +uint64_t TWriteSessionImpl::GetNextIdImpl(const std::optional& seqNo) { + + Y_ABORT_UNLESS(Lock.IsLocked()); + + uint64_t id = ++NextId; + if (!AutoSeqNoMode.has_value()) { + AutoSeqNoMode = !seqNo.has_value(); + } + if (seqNo.has_value()) { + if (!Settings.DeduplicationEnabled_.value_or(true)) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "SeqNo is provided on write when deduplication is disabled"); + ThrowFatalError("Cannot provide SeqNo on Write() when deduplication is disabled"); + } + if (*AutoSeqNoMode) { + LOG_LAZY(DbDriverState->Log, + TLOG_ERR, + LogPrefix() << "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" + ); + ThrowFatalError( + "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" + ); + } else { + id = *seqNo; + } + } else if (!(*AutoSeqNoMode)) { + LOG_LAZY(DbDriverState->Log, + TLOG_ERR, + LogPrefix() << "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" + ); + ThrowFatalError( + "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" + ); + } + return id; +} + +inline void TWriteSessionImpl::CheckHandleResultImpl(THandleResult& result) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true); +} + +void TWriteSessionImpl::ProcessHandleResult(THandleResult& result) { + if (result.DoRestart) { + Start(result.StartDelay); + } else if (result.DoSetSeqNo) { + InitSeqNoPromise.SetException("session closed"); + } +} + +NThreading::TFuture TWriteSessionImpl::WaitEvent() { + return EventsQueue->WaitEvent(); +} + +void TWriteSessionImpl::TrySubscribeOnTransactionCommit(TTransaction* tx) +{ + if (!tx) { + return; + } + + const TTransactionId txId = MakeTransactionId(*tx); + TTransactionInfoPtr txInfo = GetOrCreateTxInfo(txId); + + with_lock(txInfo->Lock) { + if (txInfo->Subscribed) { + return; + } + + txInfo->IsActive = true; + txInfo->Subscribed = true; + txInfo->AllAcksReceived = NThreading::NewPromise(); + } + + auto callback = [cbContext = this->SelfContext, txId, txInfo]() { + with_lock(txInfo->Lock) { + Y_ABORT_UNLESS(!txInfo->CommitCalled); + + txInfo->CommitCalled = true; + + if (txInfo->WriteCount == txInfo->AckCount) { + txInfo->AllAcksReceived.SetValue(MakeCommitTransactionSuccess()); + if (auto self = cbContext->LockShared()) { + self->DeleteTx(txId); + } + return txInfo->AllAcksReceived.GetFuture(); + } + + if (txInfo->IsActive) { + return txInfo->AllAcksReceived.GetFuture(); + } + } + + return NThreading::MakeFuture(MakeSessionExpiredError()); + }; + + tx->AddPrecommitCallback(std::move(callback)); +} + +void TWriteSessionImpl::TrySignalAllAcksReceived(ui64 seqNo) +{ + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto p = WrittenInTx.find(seqNo); + if (p == WrittenInTx.end()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, + LogPrefix() << "OnAck: seqNo=" << seqNo << ", txId=?"); + return; + } + + const TTransactionId& txId = p->second; + TTransactionInfoPtr txInfo = GetOrCreateTxInfo(txId); + + with_lock(txInfo->Lock) { + ++txInfo->AckCount; + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, + LogPrefix() << "OnAck: seqNo=" << seqNo << ", txId=" << GetTxId(txId) << ", WriteCount=" << txInfo->WriteCount << ", AckCount=" << txInfo->AckCount); + + if (txInfo->CommitCalled && (txInfo->WriteCount == txInfo->AckCount)) { + txInfo->AllAcksReceived.SetValue(MakeCommitTransactionSuccess()); + + Txs.erase(txId); + } + } +} + +auto TWriteSessionImpl::GetOrCreateTxInfo(const TTransactionId& txId) -> TTransactionInfoPtr +{ + auto p = Txs.find(txId); + if (p == Txs.end()) { + TTransactionInfoPtr& txInfo = Txs[txId]; + txInfo = std::make_shared(); + txInfo->Subscribed = false; + txInfo->CommitCalled = false; + p = Txs.find(txId); + } + return p->second; +} + +void TWriteSessionImpl::DeleteTx(const TTransactionId& txId) +{ + with_lock (Lock) { + Txs.erase(txId); + } +} + +void TWriteSessionImpl::WriteInternal(TContinuationToken&&, TWriteMessage&& message) { + TInstant createdAtValue = message.CreateTimestamp_.value_or(TInstant::Now()); + bool readyToAccept = false; + size_t bufferSize = message.Data.size(); + { + std::lock_guard guard(Lock); + TrySubscribeOnTransactionCommit(message.GetTxPtr()); + + ui64 seqNo = GetNextIdImpl(message.SeqNo_); + + if (message.GetTxPtr()) { + const auto& txId = MakeTransactionId(*message.GetTxPtr()); + TTransactionInfoPtr txInfo = GetOrCreateTxInfo(txId); + with_lock(txInfo->Lock) { + ++txInfo->WriteCount; + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, + LogPrefix() << "OnWrite: seqNo=" << seqNo << ", txId=" << GetTxId(txId) << ", WriteCount=" << txInfo->WriteCount << ", AckCount=" << txInfo->AckCount); + } + WrittenInTx[seqNo] = txId; + } + + CurrentBatch.Add( + seqNo, createdAtValue, message.Data, message.Codec, message.OriginalSize, + message.MessageMeta_, + message.GetTxPtr() + ); + + FlushWriteIfRequiredImpl(); + readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk; + } + if (readyToAccept) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } +} + +// Client method. +void TWriteSessionImpl::Write(TContinuationToken&& token, TWriteMessage&& message) { + WriteInternal(std::move(token), std::move(message)); +} + +// Client method. +void TWriteSessionImpl::WriteEncoded(TContinuationToken&& token, TWriteMessage&& message) +{ + WriteInternal(std::move(token), std::move(message)); +} + +TWriteSessionImpl::THandleResult TWriteSessionImpl::OnErrorImpl(NYdb::TPlainStatus&& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + (*Counters->Errors)++; + auto result = RestartImpl(status); + if (result.DoStop) { + CloseImpl(status.Status, std::move(status.Issues)); + } + return result; +} + +// No lock +void TWriteSessionImpl::Connect(const TDuration& delay) { + NYdbGrpc::IQueueClientContextPtr prevConnectContext; + NYdbGrpc::IQueueClientContextPtr prevConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr prevConnectDelayContext; + NYdbGrpc::IQueueClientContextPtr connectContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectDelayContext = nullptr; + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr; + TRpcRequestSettings reqSettings; + std::shared_ptr connectionFactory; + + // Callbacks + std::function connectCallback; + std::function connectTimeoutCallback; + + with_lock(Lock) { + if (Aborting) { + return; + } + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Start write session. Will connect to nodeId: " << PreferredPartitionLocation.Endpoint.NodeId); + + ++ConnectionGeneration; + + if (!ClientContext) { + ClientContext = Client->CreateContext(); + if (!ClientContext) { + AbortImpl(); + // Grpc and WriteSession is closing right now. + return; + } + } + + ServerMessage = std::make_shared(); + + connectionFactory = Client->CreateWriteSessionConnectionProcessorFactory(); + ConnectionFactory = connectionFactory; + + connectContext = ClientContext->CreateContext(); + if (delay) + connectDelayContext = ClientContext->CreateContext(); + connectTimeoutContext = ClientContext->CreateContext(); + + // Previous operations contexts. + + // Set new context + prevConnectContext = std::exchange(ConnectContext, connectContext); + prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); + prevConnectDelayContext = std::exchange(ConnectDelayContext, connectDelayContext); + Y_ASSERT(ConnectContext); + Y_ASSERT(ConnectTimeoutContext); + + // Cancel previous operations. + Cancel(prevConnectContext); + if (prevConnectDelayContext) { + Cancel(prevConnectDelayContext); + } + Cancel(prevConnectTimeoutContext); + + if (Processor) { + Processor->Cancel(); + } + + reqSettings = TRpcRequestSettings::Make(Settings, PreferredPartitionLocation.Endpoint); + + connectCallback = [cbContext = SelfContext, + connectContext = connectContext](TPlainStatus&& st, typename IProcessor::TPtr&& processor) { + if (auto self = cbContext->LockShared()) { + self->OnConnect(std::move(st), std::move(processor), connectContext); + } + }; + + connectTimeoutCallback = [cbContext = SelfContext, connectTimeoutContext = connectTimeoutContext](bool ok) { + if (ok) { + if (auto self = cbContext->LockShared()) { + self->OnConnectTimeout(connectTimeoutContext); + } + } + }; + } + + connectionFactory->CreateProcessor( + std::move(connectCallback), + reqSettings, + std::move(connectContext), + TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting. + std::move(connectTimeoutContext), + std::move(connectTimeoutCallback), + delay, + std::move(connectDelayContext) + ); +} + +// RPC callback. +void TWriteSessionImpl::OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext) { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session: connect timeout"); + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (ConnectTimeoutContext == connectTimeoutContext) { + Cancel(ConnectContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + } else { + return; + } + TStringBuilder description; + description << "Failed to establish connection to server. Attempts done: " << ConnectionAttemptsDone; + handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description)); + if (handleResult.DoStop) { + CloseImpl( + EStatus::TIMEOUT, + description + ); + } + } + ProcessHandleResult(handleResult); +} + +// RPC callback. +void TWriteSessionImpl::OnConnect( + TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NYdbGrpc::IQueueClientContextPtr& connectContext +) { + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (ConnectContext == connectContext) { + Cancel(ConnectTimeoutContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + + if (st.Ok()) { + Processor = std::move(processor); + InitImpl(); + // Still should call ReadFromProcessor(); + } + } else { + return; + } + if (!st.Ok()) { + handleResult = RestartImpl(st); + if (handleResult.DoStop) { + CloseImpl( + st.Status, + MakeIssueWithSubIssues( + TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint + << "\". Attempts done: " << ConnectionAttemptsDone, + st.Issues + ) + ); + } + } + } + if (st.Ok()) + ReadFromProcessor(); // Out of Init + ProcessHandleResult(handleResult); +} + +// Produce init request for session. +void TWriteSessionImpl::InitImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + TClientMessage req; + auto* init = req.mutable_init_request(); + init->set_path(TStringType{Settings.Path_}); + init->set_producer_id(TStringType{Settings.ProducerId_}); + + if (Settings.DirectWriteToPartition_ && (Settings.PartitionId_.has_value() || DirectWriteToPartitionId.has_value())) { + auto partition_id = Settings.PartitionId_.has_value() ? *Settings.PartitionId_ : *DirectWriteToPartitionId; + auto* p = init->mutable_partition_with_generation(); + p->set_partition_id(partition_id); + p->set_generation(PreferredPartitionLocation.Generation); + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: direct write to partition: " << partition_id << ", generation " << PreferredPartitionLocation.Generation); + } else if (Settings.PartitionId_.has_value()) { + init->set_partition_id(*Settings.PartitionId_); + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: write to partition: " << *Settings.PartitionId_); + } else { + init->set_message_group_id(TStringType{Settings.MessageGroupId_}); + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: write to message_group: " << Settings.MessageGroupId_); + } + + for (const auto& attr : Settings.Meta_.Fields) { + (*init->mutable_write_session_meta())[attr.first] = attr.second; + } + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: send init request: "<< req.ShortDebugString()); + + TRACE_LAZY(DbDriverState->Log, "InitRequest", + TRACE_KV_IF(init->partitioning_case() == Ydb::Topic::StreamWriteMessage_InitRequest::kPartitionId, "partition_id", init->partition_id()), + TRACE_IF(init->partitioning_case() == Ydb::Topic::StreamWriteMessage_InitRequest::kPartitionWithGeneration, + TRACE_KV("pwg_partition_id", init->partition_with_generation().partition_id()), + TRACE_KV("pwg_generation", init->partition_with_generation().generation()) + )); + + WriteToProcessorImpl(std::move(req)); +} + +// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe +void TWriteSessionImpl::WriteToProcessorImpl(TWriteSessionImpl::TClientMessage&& req) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + Y_ASSERT(Processor); + if (Aborting) { + return; + } + auto callback = [cbContext = SelfContext, + connectionGeneration = ConnectionGeneration](NYdbGrpc::TGrpcStatus&& grpcStatus) { + if (auto self = cbContext->LockShared()) { + self->OnWriteDone(std::move(grpcStatus), connectionGeneration); + } + }; + + Processor->Write(std::move(req), callback); +} + +void TWriteSessionImpl::ReadFromProcessor() { + Y_ASSERT(Processor); + IProcessor::TPtr prc; + uint64_t generation; + std::function callback; + { + std::lock_guard guard(Lock); + if (Aborting) { + return; + } + prc = Processor; + generation = ConnectionGeneration; + callback = [cbContext = SelfContext, + connectionGeneration = generation, + processor = prc, + serverMessage = ServerMessage] + (NYdbGrpc::TGrpcStatus&& grpcStatus) { + if (auto self = cbContext->LockShared()) { + self->OnReadDone(std::move(grpcStatus), connectionGeneration); + } + }; + } + prc->Read(ServerMessage.get(), std::move(callback)); +} + +void TWriteSessionImpl::OnWriteDone(NYdbGrpc::TGrpcStatus&& status, size_t connectionGeneration) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: OnWriteDone " << status.ToDebugString()); + + THandleResult handleResult; + { + std::lock_guard guard(Lock); + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (Aborting) { + return; + } + if(!status.Ok()) { + handleResult = OnErrorImpl(status); + } + } + ProcessHandleResult(handleResult); +} + +void TWriteSessionImpl::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: OnReadDone " << grpcStatus.ToDebugString()); + + TPlainStatus errorStatus; + TProcessSrvMessageResult processResult; + bool needSetValue = false; + if (!grpcStatus.Ok()) { + errorStatus = TPlainStatus(std::move(grpcStatus)); + } + bool doRead = false; + { + std::lock_guard guard(Lock); + UpdateTimedCountersImpl(); + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (errorStatus.Ok()) { + if (IsErrorMessage(*ServerMessage)) { + errorStatus = MakeErrorFromProto(*ServerMessage); + } else { + processResult = ProcessServerMessageImpl(); + needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.has_value() && (InitSeqNoSetDone = true); + if (errorStatus.Ok() && processResult.Ok) { + doRead = true; + } + } + } + } + + for (auto& event : processResult.Events) { + EventsQueue->PushEvent(std::move(event)); + } + + if (doRead) + ReadFromProcessor(); + + { + std::lock_guard guard(Lock); + if (!errorStatus.Ok()) { + if (processResult.Ok) { // Otherwise, OnError was already called + processResult.HandleResult = RestartImpl(errorStatus); + } + } + if (processResult.HandleResult.DoStop) { + CloseImpl(std::move(errorStatus)); + } + } + if (needSetValue) { + InitSeqNoPromise.SetValue(*processResult.InitSeqNo); + processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case. + } + ProcessHandleResult(processResult.HandleResult); +} + +TStringBuilder TWriteSessionImpl::LogPrefix() const { + TStringBuilder ret; + ret << " SessionId [" << SessionId << "] "; + + if (Settings.PartitionId_.has_value() || DirectWriteToPartitionId.has_value()) { + auto partition_id = Settings.PartitionId_.has_value() ? *Settings.PartitionId_ : *DirectWriteToPartitionId; + ret << " PartitionId [" << partition_id << "] "; + if (Settings.DirectWriteToPartition_) { + ret << " Generation [" << PreferredPartitionLocation.Generation << "] "; + } + } else { + ret << " MessageGroupId [" << Settings.MessageGroupId_ << "] "; + } + + return ret; +} + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const { + const auto* self = static_cast(this); + res << "AcksEvent:"; + for (auto& ack : self->Acks) { + res << " { seqNo : " << ack.SeqNo << ", State : " << ack.State; + if (ack.Details) { + res << ", offset : " << ack.Details->Offset << ", partitionId : " << ack.Details->PartitionId; + } + res << " }"; + } + if (!self->Acks.empty() && self->Acks.back().Stat) { + auto& stat = self->Acks.back().Stat; + res << " write stat: Write time " << stat->WriteTime + << " minimal time in partition queue " << stat->MinTimeInPartitionQueue + << " maximal time in partition queue " << stat->MaxTimeInPartitionQueue + << " partition quoted time " << stat->PartitionQuotedTime + << " topic quoted time " << stat->TopicQuotedTime; + } +} + +template<> +void TPrintable::DebugString(TStringBuilder& res, bool) const { + res << "ReadyToAcceptEvent"; +} + +TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMessageImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + TProcessSrvMessageResult result; + switch (ServerMessage->server_message_case()) { + case TServerMessage::SERVER_MESSAGE_NOT_SET: { + SessionEstablished = false; + result.HandleResult = OnErrorImpl({ + static_cast(ServerMessage->status()), + {NYdb::NIssue::TIssue{ServerMessage->DebugString()}} + }); + result.Ok = false; + break; + } + case TServerMessage::kInitResponse: { + const auto& initResponse = ServerMessage->init_response(); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session established. Init response: " << initResponse.ShortDebugString()); + TRACE_LAZY(DbDriverState->Log, "InitResponse", + TRACE_KV("partition_id", initResponse.partition_id()), + TRACE_KV("session_id", initResponse.session_id())); + SessionId = initResponse.session_id(); + + auto prevDirectWriteToPartitionId = DirectWriteToPartitionId; + if (Settings.DirectWriteToPartition_ && !Settings.PartitionId_.has_value()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: set DirectWriteToPartitionId " << initResponse.partition_id()); + DirectWriteToPartitionId = initResponse.partition_id(); + } + PartitionId = initResponse.partition_id(); + + uint64_t newLastSeqNo = initResponse.last_seq_no(); + if (!Settings.DeduplicationEnabled_.value_or(true)) { + newLastSeqNo = 0; + } + result.InitSeqNo = newLastSeqNo; + if (!InitSeqNo.has_value()) { + InitSeqNo = newLastSeqNo; + } + + OnErrorResolved(); + + if (Settings.DirectWriteToPartition_ && DirectWriteToPartitionId.has_value() && !prevDirectWriteToPartitionId.has_value()) { + result.HandleResult.DoRestart = true; + result.HandleResult.StartDelay = TDuration::Zero(); + break; + } + + SessionEstablished = true; + LastCountersUpdateTs = TInstant::Now(); + SessionStartedTs = TInstant::Now(); + + if (!FirstTokenSent) { + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + FirstTokenSent = true; + } + // Kickstart send after session reestablishment + SendImpl(); + break; + } + case TServerMessage::kWriteResponse: { + TWriteSessionEvent::TAcksEvent acksEvent; + const auto& batchWriteResponse = ServerMessage->write_response(); + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Write session got write response: " << batchWriteResponse.ShortDebugString() + ); + TWriteStat::TPtr writeStat = new TWriteStat{}; + const auto& stat = batchWriteResponse.write_statistics(); + + auto durationConv = [](const ::google::protobuf::Duration& dur) { + return TDuration::MilliSeconds(::google::protobuf::util::TimeUtil::DurationToMilliseconds(dur)); + }; + + writeStat->WriteTime = durationConv(stat.persisting_time()); + writeStat->MinTimeInPartitionQueue = durationConv(stat.min_queue_wait_time()); + writeStat->MaxTimeInPartitionQueue = durationConv(stat.max_queue_wait_time()); + writeStat->PartitionQuotedTime = durationConv(stat.partition_quota_wait_time()); + writeStat->TopicQuotedTime = durationConv(stat.topic_quota_wait_time()); + + for (const auto& ack : batchWriteResponse.acks()) { + // TODO: Fill writer statistics + uint64_t msgId = GetIdImpl(ack.seq_no()); + + Y_ABORT_UNLESS(ack.has_written() || ack.has_skipped() || ack.has_written_in_tx()); + + TrySignalAllAcksReceived(msgId); + + TWriteSessionEvent::TWriteAck::EEventState msgWriteStatus; + if (ack.has_written_in_tx()) { + msgWriteStatus = TWriteSessionEvent::TWriteAck::EES_WRITTEN_IN_TX; + } else if (ack.has_written()) { + msgWriteStatus = TWriteSessionEvent::TWriteAck::EES_WRITTEN; + } else { + msgWriteStatus = + (ack.skipped().reason() == Ydb::Topic::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason_REASON_ALREADY_WRITTEN) + ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN + : TWriteSessionEvent::TWriteAck::EES_DISCARDED; + } + + uint64_t offset = ack.has_written() ? ack.written().offset() : 0; + + acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{ + msgId, + msgWriteStatus, + TWriteSessionEvent::TWriteAck::TWrittenMessageDetails { + offset, + PartitionId, + }, + writeStat, + }); + + if (CleanupOnAcknowledged(msgId)) { + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } + } + //EventsQueue->PushEvent(std::move(acksEvent)); + result.Events.emplace_back(std::move(acksEvent)); + break; + } + case TServerMessage::kUpdateTokenResponse: { + UpdateTokenInProgress = false; + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: token updated successfully"); + UpdateTokenIfNeededImpl(); + break; + } + } + return result; +} + +bool TWriteSessionImpl::CleanupOnAcknowledged(uint64_t id) { + bool result = false; + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: acknoledged message " << id); + UpdateTimedCountersImpl(); + const auto& sentFront = SentOriginalMessages.front(); + uint64_t size = 0; + uint64_t compressedSize = 0; + if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == id) { + auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); + result = memoryUsage.NowOk && !memoryUsage.WasOk; + const auto& front = SentPackedMessage.front(); + if (front.Compressed) { + compressedSize = front.Data.size(); + } else { + size = front.Data.size(); + } + + (*Counters->MessagesWritten) += front.MessageCount; + (*Counters->MessagesInflight) -= front.MessageCount; + (*Counters->BytesWritten) += front.OriginalSize; + + SentPackedMessage.pop(); + } else { + size = sentFront.Size; + (*Counters->BytesWritten) += sentFront.Size; + (*Counters->MessagesWritten)++; + (*Counters->MessagesInflight)--; + } + + (*Counters->BytesInflightCompressed) -= compressedSize; + (*Counters->BytesWrittenCompressed) += compressedSize; + (*Counters->BytesInflightUncompressed) -= size; + + Y_ABORT_UNLESS(Counters->BytesInflightCompressed->Val() >= 0); + Y_ABORT_UNLESS(Counters->BytesInflightUncompressed->Val() >= 0); + + Y_ABORT_UNLESS(sentFront.Id == id); + + (*Counters->BytesInflightTotal) = MemoryUsage; + SentOriginalMessages.pop(); + + WrittenInTx.erase(id); + + return result; +} + +TMemoryUsageChange TWriteSessionImpl::OnMemoryUsageChangedImpl(i64 diff) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_; + //if (diff < 0) { + // Y_ABORT_UNLESS(MemoryUsage >= static_cast(std::abs(diff))); + //} + MemoryUsage += diff; + bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_; + if (wasOk != nowOk) { + if (wasOk) { + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Estimated memory usage " << MemoryUsage + << "[B] reached maximum (" << Settings.MaxMemoryUsage_ << "[B])" + ); + } + else { + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Estimated memory usage got back to normal " << MemoryUsage << "[B]" + ); + } + } + return {wasOk, nowOk}; +} + +TBuffer CompressBuffer(std::shared_ptr client, std::vector& data, ECodec codec, i32 level) { + TBuffer result; + Y_UNUSED(client); + std::unique_ptr coder = TCodecMap::GetTheCodecMap().GetOrThrow((ui32)codec)->CreateCoder(result, level); + for (auto& buffer : data) { + coder->Write(buffer.data(), buffer.size()); + } + coder->Finish(); + return result; +} + +// May call OnCompressed with sync executor. No external lock. +void TWriteSessionImpl::CompressImpl(TBlock&& block_) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (Aborting) { + return; + } + Y_ABORT_UNLESS(block_.Valid); + + std::shared_ptr blockPtr(std::make_shared()); + blockPtr->Move(block_); + auto lambda = [cbContext = SelfContext, + codec = Settings.Codec_, + level = Settings.CompressionLevel_, + isSyncCompression = !CompressionExecutor->IsAsync(), + blockPtr, + client = Client]() mutable { + Y_ABORT_UNLESS(!blockPtr->Compressed); + + auto compressedData = CompressBuffer( + std::move(client), blockPtr->OriginalDataRefs, codec, level + ); + Y_ABORT_UNLESS(!compressedData.Empty()); + blockPtr->Data = std::move(compressedData); + blockPtr->Compressed = true; + blockPtr->CodecID = static_cast(codec); + if (auto self = cbContext->LockShared()) { + self->OnCompressed(std::move(*blockPtr), isSyncCompression); + } + }; + + CompressionExecutor->Post(lambda); +} + +void TWriteSessionImpl::OnCompressed(TBlock&& block, bool isSyncCompression) { + TMemoryUsageChange memoryUsage; + if (!isSyncCompression) { + std::lock_guard guard(Lock); + memoryUsage = OnCompressedImpl(std::move(block)); + } else { + memoryUsage = OnCompressedImpl(std::move(block)); + } + if (memoryUsage.NowOk && !memoryUsage.WasOk) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); + } +} + +TMemoryUsageChange TWriteSessionImpl::OnCompressedImpl(TBlock&& block) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTimedCountersImpl(); + Y_ABORT_UNLESS(block.Valid); + auto memoryUsage = OnMemoryUsageChangedImpl(static_cast(block.Data.size()) - block.OriginalMemoryUsage); + (*Counters->BytesInflightUncompressed) -= block.OriginalSize; + (*Counters->BytesInflightCompressed) += block.Data.size(); + + PackedMessagesToSend.emplace(std::move(block)); + + if (!SendImplScheduled.exchange(true)) { + CompressionExecutor->Post([cbContext = SelfContext]() { + if (auto self = cbContext->LockShared()) { + self->SendImplScheduled = false; + with_lock (self->Lock) { + self->SendImpl(); + } + } + }); + } + return memoryUsage; +} + +// Set SessionEstablished = false and bring back "sent" messages to proper queues +void TWriteSessionImpl::ResetForRetryImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + SessionEstablished = false; + const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size(); + const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size(); + while (!SentPackedMessage.empty()) { + PackedMessagesToSend.emplace(std::move(SentPackedMessage.front())); + SentPackedMessage.pop(); + } + uint64_t minId = PackedMessagesToSend.empty() ? NextId + 1 : PackedMessagesToSend.top().Offset; + std::queue freshOriginalMessagesToSend; + OriginalMessagesToSend.swap(freshOriginalMessagesToSend); + while (!SentOriginalMessages.empty()) { + OriginalMessagesToSend.emplace(std::move(SentOriginalMessages.front())); + SentOriginalMessages.pop(); + } + while (!freshOriginalMessagesToSend.empty()) { + OriginalMessagesToSend.emplace(std::move(freshOriginalMessagesToSend.front())); + freshOriginalMessagesToSend.pop(); + } + if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().Id < minId) + minId = OriginalMessagesToSend.front().Id; + Y_ABORT_UNLESS(PackedMessagesToSend.size() == totalPackedMessages); + Y_ABORT_UNLESS(OriginalMessagesToSend.size() == totalOriginalMessages); +} + +// Called from client Write() methods +void TWriteSessionImpl::FlushWriteIfRequiredImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) { + MessagesAcquired += static_cast(CurrentBatch.Acquire()); + if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.value_or(TDuration::Zero()) + || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.value_or(0) + || CurrentBatch.CurrentSize >= MaxBlockSize + || CurrentBatch.Messages.size() >= MaxBlockMessageCount + || CurrentBatch.HasCodec() + ) { + WriteBatchImpl(); + return; + } + } +} + +size_t TWriteSessionImpl::WriteBatchImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Write " << CurrentBatch.Messages.size() << " messages with Id from " + << CurrentBatch.Messages.begin()->Id << " to " << CurrentBatch.Messages.back().Id + ); + + Y_ABORT_UNLESS(CurrentBatch.Messages.size() <= MaxBlockMessageCount); + + const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); + if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { + MessagesAcquired += static_cast(CurrentBatch.Acquire()); + } + + size_t size = 0; + for (size_t i = 0; i != CurrentBatch.Messages.size();) { + TBlock block{}; + for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) { + auto& currMessage = CurrentBatch.Messages[i]; + + // If MaxBlockSize or MaxBlockMessageCount values are ever changed from infinity and 1 correspondingly, + // create a new block, if the existing one is non-empty AND (adding another message will overflow it OR + // its codec is different from the codec of the next message). + + auto id = currMessage.Id; + auto createTs = currMessage.CreatedAt; + + if (!block.MessageCount) { + block.Offset = id; + } + + block.MessageCount += 1; + const auto& datum = currMessage.DataRef; + block.OriginalSize += datum.size(); + block.OriginalMemoryUsage = CurrentBatch.Data.size(); + block.OriginalDataRefs.emplace_back(datum); + if (CurrentBatch.Messages[i].Codec.has_value()) { + Y_ABORT_UNLESS(CurrentBatch.Messages.size() == 1); + block.CodecID = static_cast(*currMessage.Codec); + block.OriginalSize = currMessage.OriginalSize; + block.Compressed = false; + } + size += datum.size(); + UpdateTimedCountersImpl(); + (*Counters->BytesInflightUncompressed) += datum.size(); + (*Counters->MessagesInflight)++; + if (!currMessage.MessageMeta.empty()) { + OriginalMessagesToSend.emplace(id, createTs, datum.size(), + std::move(currMessage.MessageMeta), + currMessage.Tx); + } else { + OriginalMessagesToSend.emplace(id, createTs, datum.size(), + currMessage.Tx); + } + } + block.Data = std::move(CurrentBatch.Data); + if (skipCompression) { + PackedMessagesToSend.emplace(std::move(block)); + } else { + CompressImpl(std::move(block)); + } + } + CurrentBatch.Reset(); + if (skipCompression) { + SendImpl(); + } + return size; +} + +size_t GetMaxGrpcMessageSize() { + return 120_MB; +} + +bool TWriteSessionImpl::IsReadyToSendNextImpl() const { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!SessionEstablished) { + return false; + } + if (Aborting) + return false; + if (PackedMessagesToSend.empty()) { + return false; + } + Y_ABORT_UNLESS(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages"); + Y_ABORT_UNLESS(OriginalMessagesToSend.front().Id <= PackedMessagesToSend.top().Offset, "Lost original message(s)"); + + return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().Id; +} + + +void TWriteSessionImpl::UpdateTokenIfNeededImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: try to update token"); + + if (!DbDriverState->CredentialsProvider || UpdateTokenInProgress || !SessionEstablished) { + return; + } + + auto token = DbDriverState->CredentialsProvider->GetAuthInfo(); + if (token == PrevToken) { + return; + } + + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: updating token"); + + UpdateTokenInProgress = true; + PrevToken = token; + + TClientMessage clientMessage; + clientMessage.mutable_update_token_request()->set_token(TStringType{token}); + Processor->Write(std::move(clientMessage)); +} + +bool TWriteSessionImpl::TxIsChanged(const Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest) const +{ + Y_ABORT_UNLESS(writeRequest); + + if (!writeRequest->messages_size()) { + return false; + } + + Y_ABORT_UNLESS(!OriginalMessagesToSend.empty()); + + return GetTransactionId(*writeRequest) != GetTransactionId(OriginalMessagesToSend.front().Tx); +} + +void TWriteSessionImpl::SendImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB. + while (IsReadyToSendNextImpl()) { + TClientMessage clientMessage; + auto* writeRequest = clientMessage.mutable_write_request(); + + ui32 prevCodec = 0; + + ui64 currentSize = 0; + + // Send blocks while we can without messages reordering. + while (IsReadyToSendNextImpl() && currentSize < GetMaxGrpcMessageSize()) { + const auto& block = PackedMessagesToSend.top(); + Y_ABORT_UNLESS(block.Valid); + if (writeRequest->messages_size() > 0 && prevCodec != block.CodecID) { + break; + } + if (TxIsChanged(writeRequest)) { + break; + } + prevCodec = block.CodecID; + writeRequest->set_codec(static_cast(block.CodecID)); + Y_ABORT_UNLESS(block.MessageCount == 1); + for (size_t i = 0; i != block.MessageCount; ++i) { + Y_ABORT_UNLESS(!OriginalMessagesToSend.empty()); + + auto& message = OriginalMessagesToSend.front(); + auto* msgData = writeRequest->add_messages(); + + if (message.Tx) { + writeRequest->mutable_tx()->set_id(TStringType{message.Tx->GetId()}); + writeRequest->mutable_tx()->set_session(TStringType{message.Tx->GetSession().GetId()}); + } + + msgData->set_seq_no(GetSeqNoImpl(message.Id)); + *msgData->mutable_created_at() = ::google::protobuf::util::TimeUtil::MillisecondsToTimestamp(message.CreatedAt.MilliSeconds()); + + for (auto& [k, v] : message.MessageMeta) { + auto* pair = msgData->add_metadata_items(); + pair->set_key(TStringType{k}); + pair->set_value(TStringType{v}); + } + SentOriginalMessages.emplace(std::move(message)); + OriginalMessagesToSend.pop(); + + msgData->set_uncompressed_size(block.OriginalSize); + if (block.Compressed) { + msgData->set_data(block.Data.data(), block.Data.size()); + } else { + for (auto& buffer: block.OriginalDataRefs) { + msgData->set_data(buffer.data(), buffer.size()); + } + } + } + + TBlock moveBlock; + moveBlock.Move(block); + SentPackedMessage.emplace(std::move(moveBlock)); + PackedMessagesToSend.pop(); + + currentSize += writeRequest->ByteSizeLong(); + } + UpdateTokenIfNeededImpl(); + LOG_LAZY(DbDriverState->Log, + TLOG_DEBUG, + LogPrefix() << "Send " << writeRequest->messages_size() << " message(s) (" + << OriginalMessagesToSend.size() << " left), first sequence number is " + << writeRequest->messages(0).seq_no() + ); + Processor->Write(std::move(clientMessage)); + } +} + +// Client method, no Lock +bool TWriteSessionImpl::Close(TDuration closeTimeout) { + if (Aborting.load()) { + return false; + } + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session: close. Timeout " << closeTimeout); + auto startTime = TInstant::Now(); + auto remaining = closeTimeout; + bool ready = false; + bool needSetSeqNoValue = false; + while (remaining > TDuration::Zero()) { + { + std::lock_guard guard(Lock); + if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) { + ready = true; + } + if (Aborting.load()) + break; + } + if (ready) { + break; + } + remaining = closeTimeout - (TInstant::Now() - startTime); + Sleep(Min(TDuration::MilliSeconds(100), remaining)); + } + { + std::lock_guard guard(Lock); + ready = (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) && !Aborting.load(); + } + { + std::lock_guard guard(Lock); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); + needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + if (needSetSeqNoValue) { + InitSeqNoPromise.SetException("session closed"); + } + if (ready) { + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session: gracefully shut down, all writes complete"); + } else { + LOG_LAZY(DbDriverState->Log, TLOG_WARNING, LogPrefix() << "Write session: could not confirm all writes in time or session aborted, perform hard shutdown"); + } + return ready; +} + +void TWriteSessionImpl::HandleWakeUpImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + FlushWriteIfRequiredImpl(); + if (Aborting.load()) { + return; + } + auto callback = [cbContext = SelfContext] (bool ok) + { + if (!ok) { + return; + } + if (auto self = cbContext->LockShared()) { + std::lock_guard guard(self->Lock); + self->HandleWakeUpImpl(); + } + }; + if (TInstant::Now() - LastTokenUpdate > UPDATE_TOKEN_PERIOD) { + LastTokenUpdate = TInstant::Now(); + UpdateTokenIfNeededImpl(); + } + + const auto flushAfter = CurrentBatch.StartedAt == TInstant::Zero() + ? WakeupInterval + : WakeupInterval - Min(Now() - CurrentBatch.StartedAt, WakeupInterval); + Connections->ScheduleCallback(flushAfter, std::move(callback)); +} + +void TWriteSessionImpl::UpdateTimedCountersImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + auto now = TInstant::Now(); + auto delta = (now - LastCountersUpdateTs).MilliSeconds(); + double percent = 100.0 / Settings.MaxMemoryUsage_; + + Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta); + Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta); + Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta); + + *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds(); + LastCountersUpdateTs = now; + if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) { + LastCountersLogTs = TInstant::Now(); + +#define LOG_COUNTER(counter) \ + << " " Y_STRINGIZE(counter) ": " \ + << Counters->counter->Val() \ + /**/ + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() + << "Counters: {" + LOG_COUNTER(Errors) + LOG_COUNTER(CurrentSessionLifetimeMs) + LOG_COUNTER(BytesWritten) + LOG_COUNTER(MessagesWritten) + LOG_COUNTER(BytesWrittenCompressed) + LOG_COUNTER(BytesInflightUncompressed) + LOG_COUNTER(BytesInflightCompressed) + LOG_COUNTER(BytesInflightTotal) + LOG_COUNTER(MessagesInflight) + << " }" + ); + +#undef LOG_COUNTER + } +} + +void TWriteSessionImpl::AbortImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (!Aborting.load()) { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: aborting"); + Aborting.store(1); + Cancel(DescribePartitionContext); + Cancel(ConnectContext); + Cancel(ConnectTimeoutContext); + Cancel(ConnectDelayContext); + if (Processor) + Processor->Cancel(); + Cancel(ClientContext); + ClientContext.reset(); // removes context from contexts set from underlying gRPC-client. + + CancelTransactions(); + } +} + +void TWriteSessionImpl::CancelTransactions() +{ + for (auto& [_, txInfo] : Txs) { + with_lock(txInfo->Lock) { + txInfo->IsActive = false; + if (txInfo->WriteCount != txInfo->AckCount) { + txInfo->AllAcksReceived.SetValue(MakeSessionExpiredError()); + } + } + } + + Txs.clear(); +} + +void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); + EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues))); + AbortImpl(); +} + +void TWriteSessionImpl::CloseImpl(EStatus statusCode, const std::string& message) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(message); + CloseImpl(statusCode, std::move(issues)); +} + +void TWriteSessionImpl::CloseImpl(TPlainStatus&& status) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); + EventsQueue->Close(TSessionClosedEvent(std::move(status))); + AbortImpl(); +} + +TWriteSessionImpl::~TWriteSessionImpl() { + LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: destroy"); + bool needClose = false; + { + std::lock_guard guard(Lock); + if (!Aborting.load()) { + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); + + needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + } + if (needClose) { + InitSeqNoPromise.SetException("session closed"); + } +} + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h new file mode 100644 index 000000000000..448134b60f42 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h @@ -0,0 +1,500 @@ +#pragma once + +#include "transaction.h" + +#include +#include +#include + +#include + + +namespace NYdb::inline V3::NTopic { + +class TWriteSessionEventsQueue: public TBaseSessionEventsQueue { + using TParent = TBaseSessionEventsQueue; + +public: + TWriteSessionEventsQueue(const TWriteSessionSettings& settings) + : TParent(settings) + {} + + void PushEvent(TEventInfo eventInfo) { + if (Closed || ApplyHandler(eventInfo)) { + return; + } + + TWaiter waiter; + { + std::lock_guard guard(Mutex); + Events.emplace(std::move(eventInfo)); + waiter = PopWaiterImpl(); + } + waiter.Signal(); // Does nothing if waiter is empty. + } + + std::optional GetEvent(bool block = false) { + std::optional eventInfo; + { + std::lock_guard guard(Mutex); + if (block) { + WaitEventsImpl(); + } + if (HasEventsImpl()) { + eventInfo = GetEventImpl(); + } else { + return std::nullopt; + } + } + eventInfo->OnUserRetrievedEvent(); + return std::move(eventInfo->Event); + } + + std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt) { + std::vector eventInfos; + { + std::lock_guard guard(Mutex); + if (block) { + WaitEventsImpl(); + } + eventInfos.reserve(Min(Events.size() + CloseEvent.has_value(), maxEventsCount ? *maxEventsCount : std::numeric_limits::max())); + while (!Events.empty()) { + eventInfos.emplace_back(GetEventImpl()); + if (maxEventsCount && eventInfos.size() >= *maxEventsCount) { + break; + } + } + if (CloseEvent && Events.empty() && (!maxEventsCount || eventInfos.size() < *maxEventsCount)) { + eventInfos.push_back({*CloseEvent}); + } + } + + std::vector result; + result.reserve(eventInfos.size()); + for (TEventInfo& eventInfo : eventInfos) { + eventInfo.OnUserRetrievedEvent(); + result.emplace_back(std::move(eventInfo.Event)); + } + return result; + } + + void Close(const TSessionClosedEvent& event) { + TWaiter waiter; + { + std::lock_guard guard(Mutex); + CloseEvent = event; + Closed = true; + waiter = TWaiter(Waiter.ExtractPromise(), this); + } + + TEventInfo info(event); + ApplyHandler(info); + + waiter.Signal(); + } + +private: + struct THandlersVisitor : public TParent::TBaseHandlersVisitor { + using TParent::TBaseHandlersVisitor::TBaseHandlersVisitor; + +#define DECLARE_HANDLER(type, handler, answer) \ + bool operator()(type&) { \ + if (this->PushHandler( \ + std::move(TParent::TBaseHandlersVisitor::Event), \ + this->Settings.EventHandlers_.handler, \ + this->Settings.EventHandlers_.CommonHandler_)) { \ + return answer; \ + } \ + return false; \ + } \ + /**/ + DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true); + DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHandler_, true); + DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied + +#undef DECLARE_HANDLER + bool Visit() { + return std::visit(*this, Event); + } + }; + + bool ApplyHandler(TEventInfo& eventInfo) { + THandlersVisitor visitor(Settings, eventInfo.Event); + return visitor.Visit(); + } + + TEventInfo GetEventImpl() { // Assumes that we're under lock and that the event queue has events. + Y_ASSERT(HasEventsImpl()); + if (!Events.empty()) { + TEventInfo event = std::move(Events.front()); + Events.pop(); + RenewWaiterImpl(); + return event; + } + Y_ASSERT(CloseEvent); + return {*CloseEvent}; + } +}; + +struct TMemoryUsageChange { + bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update + bool NowOk; //!< Same, only after update +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TWriteSessionImpl + +class TWriteSessionImpl : public TContinuationTokenIssuer, + public TEnableSelfContext { +private: + friend class TWriteSession; + friend class TSimpleBlockingWriteSession; + +private: + using TClientMessage = Ydb::Topic::StreamWriteMessage::FromClient; + using TServerMessage = Ydb::Topic::StreamWriteMessage::FromServer; + using IWriteSessionConnectionProcessorFactory = + TTopicClient::TImpl::IWriteSessionConnectionProcessorFactory; + using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor; + + struct TMessage { + uint64_t Id; + TInstant CreatedAt; + std::string_view DataRef; + std::optional Codec; + ui32 OriginalSize; // only for coded messages + std::vector> MessageMeta; + const NTable::TTransaction* Tx; + + TMessage(uint64_t id, const TInstant& createdAt, std::string_view data, std::optional codec = {}, + ui32 originalSize = 0, const std::vector>& messageMeta = {}, + const NTable::TTransaction* tx = nullptr) + : Id(id) + , CreatedAt(createdAt) + , DataRef(data) + , Codec(codec) + , OriginalSize(originalSize) + , MessageMeta(messageMeta) + , Tx(tx) + {} + }; + + struct TMessageBatch { + TBuffer Data; + std::vector Messages; + uint64_t CurrentSize = 0; + TInstant StartedAt = TInstant::Zero(); + bool Acquired = false; + bool FlushRequested = false; + + void Add(uint64_t id, const TInstant& createdAt, std::string_view data, std::optional codec, ui32 originalSize, + const std::vector>& messageMeta, + const NTable::TTransaction* tx) { + if (StartedAt == TInstant::Zero()) + StartedAt = TInstant::Now(); + CurrentSize += codec ? originalSize : data.size(); + Messages.emplace_back(id, createdAt, data, codec, originalSize, messageMeta, tx); + Acquired = false; + } + + bool HasCodec() const { + return Messages.empty() ? false : Messages.front().Codec.has_value(); + } + + bool Acquire() { + if (Acquired || Messages.empty()) + return false; + auto currSize = Data.size(); + Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size()); + Messages.back().DataRef = std::string_view(Data.data() + currSize, Data.size() - currSize); + Acquired = true; + return true; + } + + bool Empty() const noexcept { + return CurrentSize == 0 && Messages.empty(); + } + + void Reset() { + StartedAt = TInstant::Zero(); + Messages.clear(); + Data.Clear(); + Acquired = false; + CurrentSize = 0; + FlushRequested = false; + } + }; + + struct TBlock { + size_t Offset = 0; //!< First message sequence number in the block + size_t MessageCount = 0; + size_t PartNumber = 0; + size_t OriginalSize = 0; + size_t OriginalMemoryUsage = 0; + ui32 CodecID = static_cast(ECodec::RAW); + mutable std::vector OriginalDataRefs; + mutable TBuffer Data; + bool Compressed = false; + mutable bool Valid = true; + + TBlock& operator=(TBlock&&) = default; + TBlock(TBlock&&) = default; + TBlock() = default; + + //For taking ownership by copying from const object, f.e. lambda -> std::function, priority_queue + void Move(const TBlock& rhs) { + Offset = rhs.Offset; + MessageCount = rhs.MessageCount; + PartNumber = rhs.PartNumber; + OriginalSize = rhs.OriginalSize; + OriginalMemoryUsage = rhs.OriginalMemoryUsage; + CodecID = rhs.CodecID; + OriginalDataRefs.swap(rhs.OriginalDataRefs); + Data.Swap(rhs.Data); + Compressed = rhs.Compressed; + + rhs.Data.Clear(); + rhs.OriginalDataRefs.clear(); + } + }; + + struct TOriginalMessage { + uint64_t Id; + TInstant CreatedAt; + size_t Size; + std::vector> MessageMeta; + const NTable::TTransaction* Tx; + + TOriginalMessage(const uint64_t id, const TInstant createdAt, const size_t size, + const NTable::TTransaction* tx) + : Id(id) + , CreatedAt(createdAt) + , Size(size) + , Tx(tx) + {} + + TOriginalMessage(const uint64_t id, const TInstant createdAt, const size_t size, + std::vector>&& messageMeta, + const NTable::TTransaction* tx) + : Id(id) + , CreatedAt(createdAt) + , Size(size) + , MessageMeta(std::move(messageMeta)) + , Tx(tx) + {} + }; + + //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue + struct Greater { + bool operator() (const TBlock& lhs, const TBlock& rhs) { + return lhs.Offset > rhs.Offset; + } + }; + + struct THandleResult { + bool DoRestart = false; + TDuration StartDelay = TDuration::Zero(); + bool DoStop = false; + bool DoSetSeqNo = false; + }; + struct TProcessSrvMessageResult { + THandleResult HandleResult; + std::optional InitSeqNo; + std::vector Events; + bool Ok = true; + }; + + struct TPartitionLocation { + TEndpointKey Endpoint; + i64 Generation; + }; + + struct TTransactionInfo { + TSpinLock Lock; + bool IsActive = false; + bool Subscribed = false; + NThreading::TPromise AllAcksReceived; + bool CommitCalled = false; + ui64 WriteCount = 0; + ui64 AckCount = 0; + }; + + using TTransactionInfoPtr = std::shared_ptr; + + THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action + +public: + TWriteSessionImpl(const TWriteSessionSettings& settings, + std::shared_ptr client, + std::shared_ptr connections, + TDbDriverStatePtr dbDriverState); + + std::optional GetEvent(bool block = false); + std::vector GetEvents(bool block = false, + std::optional maxEventsCount = std::nullopt); + NThreading::TFuture GetInitSeqNo(); + + void Write(TContinuationToken&& continuationToken, TWriteMessage&& message); + + void Write(TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, + std::optional createTimestamp = std::nullopt) { + Y_UNUSED(seqNo); + Y_UNUSED(createTimestamp); + Y_ABORT("Do not use this method"); + }; + + void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& message); + + void WriteEncoded(TContinuationToken&&, std::string_view, ECodec, ui32, + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) { + Y_UNUSED(seqNo); + Y_UNUSED(createTimestamp); + Y_ABORT("Do not use this method"); + } + + + NThreading::TFuture WaitEvent(); + + // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. + bool Close(TDuration closeTimeout = TDuration::Max()); + + TWriterCounters::TPtr GetCounters() {Y_ABORT("Unimplemented"); } //ToDo - unimplemented; + + const TWriteSessionSettings& GetSettings() const { + return Settings; + } + + ~TWriteSessionImpl(); // will not call close - destroy everything without acks + +private: + + TStringBuilder LogPrefix() const; + + void UpdateTokenIfNeededImpl(); + + void WriteInternal(TContinuationToken&& continuationToken, TWriteMessage&& message); + + void FlushWriteIfRequiredImpl(); + size_t WriteBatchImpl(); + void Start(const TDuration& delay); + void InitWriter(); + + void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor, + const NYdbGrpc::IQueueClientContextPtr& connectContext); + void OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext); + void ResetForRetryImpl(); + THandleResult RestartImpl(const TPlainStatus& status); + void Connect(const TDuration& delay); + void InitImpl(); + void ReadFromProcessor(); // Assumes that we're under lock. + void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock. + void OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration); + void OnWriteDone(NYdbGrpc::TGrpcStatus&& status, size_t connectionGeneration); + TProcessSrvMessageResult ProcessServerMessageImpl(); + TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff); + TBuffer CompressBufferImpl(std::vector& data, ECodec codec, i32 level); + void CompressImpl(TBlock&& block); + void OnCompressed(TBlock&& block, bool isSyncCompression=false); + TMemoryUsageChange OnCompressedImpl(TBlock&& block); + + //std::string GetDebugIdentity() const; + TClientMessage GetInitClientMessage(); + bool CleanupOnAcknowledged(uint64_t id); + bool IsReadyToSendNextImpl() const; + uint64_t GetNextIdImpl(const std::optional& seqNo); + uint64_t GetSeqNoImpl(uint64_t id); + uint64_t GetIdImpl(uint64_t seqNo); + void SendImpl(); + void AbortImpl(); + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); + void CloseImpl(EStatus statusCode, const std::string& message); + void CloseImpl(TPlainStatus&& status); + + void OnErrorResolved() { + RetryState = nullptr; + } + void CheckHandleResultImpl(THandleResult& result); + void ProcessHandleResult(THandleResult& result); + void HandleWakeUpImpl(); + void UpdateTimedCountersImpl(); + + void ConnectToPreferredPartitionLocation(const TDuration& delay); + void OnDescribePartition(const TStatus& status, const Ydb::Topic::DescribePartitionResult& proto, const NYdbGrpc::IQueueClientContextPtr& describePartitionContext); + + std::optional GetPreferredEndpointImpl(ui32 partitionId, uint64_t partitionNodeId); + + bool TxIsChanged(const Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest) const; + + void TrySubscribeOnTransactionCommit(TTransaction* tx); + void CancelTransactions(); + TTransactionInfoPtr GetOrCreateTxInfo(const TTransactionId& txId); + void TrySignalAllAcksReceived(ui64 seqNo); + void DeleteTx(const TTransactionId& txId); + +private: + TWriteSessionSettings Settings; + std::shared_ptr Client; + std::shared_ptr Connections; + + std::shared_ptr ConnectionFactory; + TDbDriverStatePtr DbDriverState; + std::string PrevToken; + bool UpdateTokenInProgress = false; + TInstant LastTokenUpdate = TInstant::Zero(); + std::shared_ptr EventsQueue; + NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. + NYdbGrpc::IQueueClientContextPtr ConnectContext; + NYdbGrpc::IQueueClientContextPtr ConnectTimeoutContext; + NYdbGrpc::IQueueClientContextPtr ConnectDelayContext; + NYdbGrpc::IQueueClientContextPtr DescribePartitionContext; + size_t ConnectionGeneration = 0; + size_t ConnectionAttemptsDone = 0; + TAdaptiveLock Lock; + IProcessor::TPtr Processor; + IRetryPolicy::IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). + std::shared_ptr ServerMessage; // Server message to write server response to. + + std::string SessionId; + IExecutor::TPtr CompressionExecutor; + size_t MemoryUsage = 0; //!< Estimated amount of memory used + bool FirstTokenSent = false; + + TMessageBatch CurrentBatch; + + std::queue OriginalMessagesToSend; + std::priority_queue, Greater> PackedMessagesToSend; + //! Messages that are sent but yet not acknowledged + std::queue SentOriginalMessages; + std::queue SentPackedMessage; + + const size_t MaxBlockSize = std::numeric_limits::max(); + const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility + bool Connected = false; + bool Started = false; + std::atomic SendImplScheduled = false; + std::atomic Aborting = 0; + bool SessionEstablished = false; + ui32 PartitionId = 0; + TPartitionLocation PreferredPartitionLocation = {}; + uint64_t NextId = 0; + std::optional InitSeqNo; + std::optional AutoSeqNoMode; + + NThreading::TPromise InitSeqNoPromise; + bool InitSeqNoSetDone = false; + TInstant SessionStartedTs; + TInstant LastCountersUpdateTs = TInstant::Zero(); + TInstant LastCountersLogTs; + TWriterCounters::TPtr Counters; + TDuration WakeupInterval; + + // Set by the write session, if Settings.DirectWriteToPartition is true and Settings.PartitionId is unset. Otherwise ignored. + std::optional DirectWriteToPartitionId; +protected: + uint64_t MessagesAcquired = 0; + + std::unordered_map> Txs; + std::unordered_map WrittenInTx; // SeqNo -> TxId +}; + +} // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/ya.make b/ydb/public/sdk/cpp/src/client/topic/impl/ya.make new file mode 100644 index 000000000000..29215afd9d28 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/impl/ya.make @@ -0,0 +1,42 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + common.h + common.cpp + counters_logger.h + deferred_commit.cpp + event_handlers.cpp + offsets_collector.cpp + read_session_event.cpp + read_session_impl.ipp + read_session.h + read_session.cpp + topic_impl.h + topic_impl.cpp + topic.cpp + transaction.cpp + write_session_impl.h + write_session_impl.cpp + write_session.h + write_session.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client + library/cpp/monlib/dynamic_counters + library/cpp/monlib/metrics + library/cpp/string_utils/url + ydb/public/sdk/cpp/src/library/persqueue/obfuscate + ydb/public/api/grpc/draft + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/impl/ydb_internal/make_request + ydb/public/sdk/cpp/src/client/common_client/impl + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic/common + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic + ydb/public/sdk/cpp/src/client/proto +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/topic/out.cpp b/ydb/public/sdk/cpp/src/client/topic/out.cpp new file mode 100644 index 000000000000..ef1085824a8b --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/out.cpp @@ -0,0 +1,9 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::NTopic::TDescribeTopicResult, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTopic::TDescribeConsumerResult, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/topic/proto_accessor.cpp b/ydb/public/sdk/cpp/src/client/topic/proto_accessor.cpp new file mode 100644 index 000000000000..4b333ca7bdd3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/proto_accessor.cpp @@ -0,0 +1,38 @@ +#include + +namespace NYdb::inline V3 { + const Ydb::Topic::DescribeTopicResult& TProtoAccessor::GetProto(const NTopic::TTopicDescription& topicDescription) { + return topicDescription.GetProto(); + } + + const Ydb::Topic::DescribeConsumerResult& TProtoAccessor::GetProto(const NTopic::TConsumerDescription& consumerDescription) { + return consumerDescription.GetProto(); + } + + Ydb::Topic::MeteringMode TProtoAccessor::GetProto(NTopic::EMeteringMode mode) { + switch (mode) { + case NTopic::EMeteringMode::Unspecified: + return Ydb::Topic::METERING_MODE_UNSPECIFIED; + case NTopic::EMeteringMode::RequestUnits: + return Ydb::Topic::METERING_MODE_REQUEST_UNITS; + case NTopic::EMeteringMode::ReservedCapacity: + return Ydb::Topic::METERING_MODE_RESERVED_CAPACITY; + case NTopic::EMeteringMode::Unknown: + return Ydb::Topic::METERING_MODE_UNSPECIFIED; + } + } + + NTopic::EMeteringMode TProtoAccessor::FromProto(Ydb::Topic::MeteringMode mode) { + switch (mode) { + case Ydb::Topic::MeteringMode::METERING_MODE_UNSPECIFIED: + return NTopic::EMeteringMode::Unspecified; + case Ydb::Topic::MeteringMode::METERING_MODE_REQUEST_UNITS: + return NTopic::EMeteringMode::RequestUnits; + case Ydb::Topic::MeteringMode::METERING_MODE_RESERVED_CAPACITY: + return NTopic::EMeteringMode::ReservedCapacity; + default: + return NTopic::EMeteringMode::Unknown; + } + } +}// namespace NYdb + diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp similarity index 95% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/basic_usage_ut.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp index ece3d26da8e5..919438c0c939 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp @@ -1,21 +1,23 @@ #include "ut_utils/managed_executor.h" #include "ut_utils/topic_sdk_test_setup.h" -#include +#include -#include +#include + +#include -#include - -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include #include +#include + #include namespace NYdb::NTopic::NTests { @@ -99,7 +101,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NYdb::TDriverConfig cfg; cfg.SetEndpoint(TStringBuilder() << "invalid:" << setup.GetServer().GrpcPort); cfg.SetDatabase("/Invalid"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); auto driver = NYdb::TDriver(cfg); { @@ -113,13 +115,13 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto writeSession = client.CreateWriteSession(writeSettings); auto event = writeSession->GetEvent(true); - UNIT_ASSERT(event.Defined() && std::holds_alternative(event.GetRef())); + UNIT_ASSERT(event && std::holds_alternative(event.value())); } { auto settings = TTopicClientSettings() .Database({"/Root"}) - .DiscoveryEndpoint({TStringBuilder() << "localhost:" << setup.GetServer().GrpcPort}); + .DiscoveryEndpoint("localhost:" + std::to_string(setup.GetServer().GrpcPort)); TTopicClient client(driver, settings); @@ -130,7 +132,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto writeSession = client.CreateWriteSession(writeSettings); auto event = writeSession->GetEvent(true); - UNIT_ASSERT(event.Defined() && !std::holds_alternative(event.GetRef())); + UNIT_ASSERT(event && !std::holds_alternative(event.value())); } } @@ -171,13 +173,13 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto readSession = client.CreateReadSession(readSettings); auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& startPartitionSession = std::get(*event); startPartitionSession.Confirm(); event = readSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& dataReceived = std::get(*event); dataReceived.Commit(); @@ -234,16 +236,16 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto readSession = client.CreateReadSession(readSettings); auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& startPartitionSession = std::get(*event); startPartitionSession.Confirm(); UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvent(true, 0), TContractViolation); - UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, Nothing(), 0), TContractViolation); + UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, std::nullopt, 0), TContractViolation); event = readSession->GetEvent(true, 1); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& dataReceived = std::get(*event); dataReceived.Commit(); @@ -335,7 +337,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto description = result.GetConsumerDescription(); UNIT_ASSERT(description.GetPartitions().size() == 1); auto stats = description.GetPartitions().front().GetPartitionConsumerStats(); - UNIT_ASSERT(stats.Defined()); + UNIT_ASSERT(stats.has_value()); UNIT_ASSERT(stats->GetCommittedOffset() == 50); } @@ -463,7 +465,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { UNIT_ASSERT(!futureRead.HasValue()); Cerr << ">>>TEST: future read has no value " << Endl; - RunTasks(stepByStepExecutor, {2}); // Run decompression task. + RunTasks(stepByStepExecutor, {2}); // Run decompression task. futureRead.GetValueSync(); UNIT_ASSERT(futureRead.HasValue()); Cerr << ">>>TEST: future read has value " << Endl; @@ -693,7 +695,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto description = result.GetTopicDescription(); UNIT_ASSERT(description.GetPartitions().size() == 1); auto stats = description.GetPartitions().front().GetPartitionStats(); - UNIT_ASSERT(stats.Defined()); + UNIT_ASSERT(stats.has_value()); UNIT_ASSERT_VALUES_EQUAL(stats->GetEndOffset(), count); } @@ -777,7 +779,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { std::visit(TOverloaded { [&](TReadSessionEvent::TDataReceivedEvent& event) { for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); + std::string sourceId = message.GetMessageGroupId(); ui32 seqNo = message.GetSeqNo(); UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); ++readMessageCount; @@ -827,11 +829,11 @@ Y_UNIT_TEST_SUITE(TSettingsValidation) { auto client = setup.MakeClient(); ui64 producerIndex = 0u; - auto runTest = [&](TString producer, TString msgGroup, const TMaybe& useDedup, bool useSeqNo, EExpectedTestResult result) ->bool + auto runTest = [&](TString producer, TString msgGroup, const std::optional& useDedup, bool useSeqNo, EExpectedTestResult result) ->bool { TWriteSessionSettings writeSettings; writeSettings.Path(setup.GetTopicPath()).Codec(NTopic::ECodec::RAW); - TString useDedupStr = useDedup.Defined() ? ToString(*useDedup) : ""; + TString useDedupStr = useDedup.has_value() ? ToString(*useDedup) : ""; if (producer) { producer += ToString(producerIndex); } @@ -846,7 +848,7 @@ Y_UNIT_TEST_SUITE(TSettingsValidation) { << useDedupStr << ", manual SeqNo: " << useSeqNo << Endl; try { - if (useDedup.Defined()) { + if (useDedup.has_value()) { writeSettings.DeduplicationEnabled(useDedup); } auto session = client.CreateWriteSession(writeSettings); @@ -855,12 +857,12 @@ Y_UNIT_TEST_SUITE(TSettingsValidation) { ui64 written = 0; while (written < 10) { auto event = session->GetEvent(true); - if (std::holds_alternative(event.GetRef())) { + if (std::holds_alternative(event.value())) { auto closed = std::get(*event); Cerr << "Session failed with error: " << closed.DebugString() << Endl; UNIT_ASSERT(result == EExpectedTestResult::FAIL_ON_RPC); return false; - } else if (std::holds_alternative(event.GetRef())) { + } else if (std::holds_alternative(event.value())) { token = std::move(std::get(*event).ContinuationToken); if (useSeqNo) { session->Write(std::move(*token), "data", seqNo++); @@ -946,10 +948,10 @@ Y_UNIT_TEST_SUITE(TSettingsValidation) { auto readSession = client.CreateReadSession(readSettings); auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); + UNIT_ASSERT(event.has_value()); auto& closeEvent = std::get(*event); - UNIT_ASSERT(closeEvent.DebugString().Contains("Too small max memory usage")); + UNIT_ASSERT(closeEvent.DebugString().contains("Too small max memory usage")); } } // Y_UNIT_TEST_SUITE(TSettingsValidation) diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/describe_topic_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/describe_topic_ut.cpp similarity index 95% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/describe_topic_ut.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/describe_topic_ut.cpp index ec490b67b974..ea3b5472aa77 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/describe_topic_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/describe_topic_ut.cpp @@ -1,12 +1,12 @@ #include "ut_utils/topic_sdk_test_setup.h" #include -#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -144,7 +144,7 @@ namespace NYdb::NTopic::NTests { UNIT_ASSERT_GT(consumerStats->GetLastReadOffset(), 0); UNIT_ASSERT_GT(consumerStats->GetCommittedOffset(), 0); - UNIT_ASSERT_GE(consumerStats->GetReadSessionId(), 0); + UNIT_ASSERT_GE(TString{consumerStats->GetReadSessionId()}, 0); UNIT_ASSERT_VALUES_EQUAL(consumerStats->GetReaderName(), ""); UNIT_ASSERT_GE(consumerStats->GetMaxWriteTimeLag(), TDuration::Seconds(100)); } else { @@ -297,9 +297,9 @@ namespace NYdb::NTopic::NTests { // Event 1: start partition session { - TMaybe event = readSession->GetEvent(true); + std::optional event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto startPartitionSession = std::get_if(event.Get()); + auto startPartitionSession = std::get_if(&event.value()); UNIT_ASSERT_C(startPartitionSession, DebugString(*event)); startPartitionSession->Confirm(); @@ -307,9 +307,9 @@ namespace NYdb::NTopic::NTests { // Event 2: data received { - TMaybe event = readSession->GetEvent(true); + std::optional event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto dataReceived = std::get_if(event.Get()); + auto dataReceived = std::get_if(&event.value()); UNIT_ASSERT_C(dataReceived, DebugString(*event)); dataReceived->Commit(); @@ -317,9 +317,9 @@ namespace NYdb::NTopic::NTests { // Event 3: commit acknowledgement { - TMaybe event = readSession->GetEvent(true); + std::optional event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto commitOffsetAck = std::get_if(event.Get()); + auto commitOffsetAck = std::get_if(&event.value()); UNIT_ASSERT_C(commitOffsetAck, DebugString(*event)); @@ -380,7 +380,7 @@ namespace NYdb::NTopic::NTests { if (allowUpdateRow) { acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, authToken); } - setup.GetServer().AnnoyingClient->ModifyACL("/Root", TEST_TOPIC, acl.SerializeAsString()); + setup.GetServer().AnnoyingClient->ModifyACL("/Root", TString{TEST_TOPIC}, acl.SerializeAsString()); while (true) { TDescribePartitionResult result = client.DescribePartition(existingTopic ? TEST_TOPIC : "bad-topic", testPartitionId, settings).GetValueSync(); @@ -403,7 +403,7 @@ namespace NYdb::NTopic::NTests { bool allowUpdateRow; bool allowDescribeSchema; EStatus status; - NYql::TIssueCode issueCode; + NYdb::NIssue::TIssueCode issueCode; }; std::vector expectations{ @@ -421,7 +421,7 @@ namespace NYdb::NTopic::NTests { auto result = RunPermissionTest(setup, userId++, existing, update, describe); auto resultStatus = result.GetStatus(); auto line = TStringBuilder() << "=== status=" << resultStatus; - NYql::TIssueCode resultIssue = 0; + NYdb::NIssue::TIssueCode resultIssue = 0; if (!result.GetIssues().Empty()) { resultIssue = result.GetIssues().begin()->GetCode(); line << " issueCode=" << resultIssue; @@ -433,7 +433,7 @@ namespace NYdb::NTopic::NTests { if (resultStatus == EStatus::SUCCESS) { auto& p = result.GetPartitionDescription().GetPartition(); UNIT_ASSERT(p.GetActive()); - UNIT_ASSERT(p.GetPartitionLocation().Defined()); + UNIT_ASSERT(p.GetPartitionLocation().has_value()); } } } diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/local_partition_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/local_partition_ut.cpp similarity index 94% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/local_partition_ut.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/local_partition_ut.cpp index 75e070ed1a86..6c61ef2eff20 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/local_partition_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/local_partition_ut.cpp @@ -2,16 +2,16 @@ #include -#include +#include -#include -#include -#include +#include +#include +#include -#include +#include -#include -#include +#include +#include #include #include @@ -83,16 +83,16 @@ namespace NYdb::NTopic::NTests { auto readSession = client.CreateReadSession(CreateReadSessionSettings()); - TMaybe event = readSession->GetEvent(true); + std::optional event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto startPartitionSession = std::get_if(event.Get()); + auto startPartitionSession = std::get_if(&event.value()); UNIT_ASSERT_C(startPartitionSession, DebugString(*event)); startPartitionSession->Confirm(); event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto dataReceived = std::get_if(event.Get()); + auto dataReceived = std::get_if(&event.value()); UNIT_ASSERT_C(dataReceived, DebugString(*event)); dataReceived->Commit(); @@ -103,7 +103,7 @@ namespace NYdb::NTopic::NTests { event = readSession->GetEvent(true); UNIT_ASSERT(event); - auto commitOffsetAck = std::get_if(event.Get()); + auto commitOffsetAck = std::get_if(&event.value()); UNIT_ASSERT_C(commitOffsetAck, DebugString(*event)); UNIT_ASSERT_VALUES_EQUAL(commitOffsetAck->GetCommittedOffset(), expectedCommitedOffset); } @@ -391,7 +391,7 @@ namespace NYdb::NTopic::NTests { discovery.SetGoodEndpoints(*setup); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto sessionSettings = TWriteSessionSettings() @@ -424,7 +424,7 @@ namespace NYdb::NTopic::NTests { discovery.SetGoodEndpoints(*setup); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto retryPolicy = std::make_shared(); @@ -471,13 +471,13 @@ namespace NYdb::NTopic::NTests { // But at first we only add node 1 to the discovery service. auto setup = CreateSetup(TEST_CASE_NAME, 2, /* createTopic = */ false); setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, false); - setup->CreateTopic(TEST_TOPIC, TEST_CONSUMER, 1); + setup->CreateTopic(TString{TEST_TOPIC}, TEST_CONSUMER, 1); setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, true); TMockDiscoveryService discovery; discovery.SetEndpoints(setup->GetRuntime().GetNodeId(0), 1, setup->GetServer().GrpcPort); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto retryPolicy = std::make_shared(); @@ -522,7 +522,7 @@ namespace NYdb::NTopic::NTests { discovery.SetEndpoints(setup->GetRuntime().GetNodeId(0), 1, 0); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto retryPolicy = std::make_shared(); @@ -560,13 +560,13 @@ namespace NYdb::NTopic::NTests { auto setup = CreateSetup(TEST_CASE_NAME, 2, /* createTopic = */ false); // Make the node 1 unavailable. setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, false); - setup->CreateTopic(TEST_TOPIC, TEST_CONSUMER, 2); + setup->CreateTopic(TString{TEST_TOPIC}, TEST_CONSUMER, 2); TMockDiscoveryService discovery; discovery.SetGoodEndpoints(*setup); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto retryPolicy = std::make_shared(); @@ -627,14 +627,14 @@ namespace NYdb::NTopic::NTests { // Allow UpdateRow only, no DescribeSchema permission. NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, authToken); - setup->GetServer().AnnoyingClient->ModifyACL("/Root", TEST_TOPIC, acl.SerializeAsString()); + setup->GetServer().AnnoyingClient->ModifyACL("/Root", TString{TEST_TOPIC}, acl.SerializeAsString()); } TMockDiscoveryService discovery; discovery.SetGoodEndpoints(*setup); auto* tracingBackend = new TTracingBackend(); auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()) - .SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})) + .SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())) .SetAuthToken(authToken); TDriver driver(driverConfig); TTopicClient client(driver); @@ -664,13 +664,13 @@ namespace NYdb::NTopic::NTests { Y_UNIT_TEST(WithoutPartitionWithSplit) { auto setup = CreateSetupForSplitMerge(TEST_CASE_NAME); - setup.CreateTopic(TEST_TOPIC, TEST_CONSUMER, 1, 100); + setup.CreateTopic(TString{TEST_TOPIC}, TEST_CONSUMER, 1, 100); TMockDiscoveryService discovery; discovery.SetGoodEndpoints(setup); auto driverConfig = CreateConfig(setup, discovery.GetDiscoveryAddr()); auto* tracingBackend = new TTracingBackend(); - driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + driverConfig.SetLog(std::unique_ptr(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend}).Release())); TDriver driver(driverConfig); TTopicClient client(driver); auto writeSettings = TWriteSessionSettings() diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp similarity index 97% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp index f33f2b5a0e89..2f6b621a9a13 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp @@ -1,8 +1,8 @@ #include "ut_utils/topic_sdk_test_setup.h" -#include -#include -#include +#include +#include +#include #include #include @@ -68,7 +68,7 @@ class TFixture : public NUnitTest::TBaseFixture { struct TReadMessageSettings { NTable::TTransaction& Tx; bool CommitOffsets = false; - TMaybe Offset; + std::optional Offset; }; void ReadMessage(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset); @@ -79,7 +79,7 @@ class TFixture : public NUnitTest::TBaseFixture { const TString& topic, const TString& groupId, NTable::TTransaction& tx); - void CreateTopic(const TString& path = TEST_TOPIC, + void CreateTopic(const TString& path = TString{TEST_TOPIC}, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, std::optional maxPartitionCount = std::nullopt); @@ -98,10 +98,10 @@ class TFixture : public NUnitTest::TBaseFixture { TTopicWriteSessionPtr CreateTopicWriteSession(const TString& topicPath, const TString& messageGroupId, - TMaybe partitionId); + std::optional partitionId); TTopicWriteSessionContext& GetTopicWriteSession(const TString& topicPath, const TString& messageGroupId, - TMaybe partitionId); + std::optional partitionId); TTopicReadSessionPtr CreateTopicReadSession(const TString& topicPath, const TString& consumerName, @@ -114,7 +114,7 @@ class TFixture : public NUnitTest::TBaseFixture { const TString& messageGroupId, const TString& message, NTable::TTransaction* tx = nullptr, - TMaybe partitionId = Nothing()); + std::optional partitionId = std::nullopt); TVector ReadFromTopic(const TString& topicPath, const TString& consumerName, const TDuration& duration, @@ -386,7 +386,7 @@ void TFixture::ReadMessage(TTopicReadSessionPtr reader, NTable::TTransaction& tx void TFixture::ReadMessage(TTopicReadSessionPtr reader, const TReadMessageSettings& settings) { auto event = ReadEvent(reader, settings.Tx); - if (settings.Offset.Defined()) { + if (settings.Offset.has_value()) { UNIT_ASSERT_VALUES_EQUAL(event.GetMessages()[0].GetOffset(), *settings.Offset); } if (settings.CommitOffsets) { @@ -529,8 +529,8 @@ void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) auto writeSession = client.CreateWriteSession(options); auto event = writeSession->GetEvent(true); - UNIT_ASSERT(event.Defined() && std::holds_alternative(event.GetRef())); - auto token = std::move(std::get(event.GetRef()).ContinuationToken); + UNIT_ASSERT(event && std::holds_alternative(event.value())); + auto token = std::move(std::get(event.value()).ContinuationToken); NTopic::TWriteMessage params("message"); params.Tx(tx); @@ -546,8 +546,8 @@ void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) while (true) { event = writeSession->GetEvent(true); - UNIT_ASSERT(event.Defined()); - auto& v = event.GetRef(); + UNIT_ASSERT(event.has_value()); + auto& v = event.value(); if (auto e = std::get_if(&v); e) { UNIT_ASSERT(false); } else if (auto e = std::get_if(&v); e) { @@ -645,8 +645,8 @@ Y_UNIT_TEST_F(WriteToTopic_Invalid_Session, TFixture) Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) { TString topicPath[2] = { - TEST_TOPIC, - TEST_TOPIC + "_2" + TString{TEST_TOPIC}, + TString{TEST_TOPIC} + "_2" }; CreateTopic(topicPath[1]); @@ -664,8 +664,8 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) params.Tx(tx); auto event = ws->GetEvent(true); - UNIT_ASSERT(event.Defined() && std::holds_alternative(event.GetRef())); - auto token = std::move(std::get(event.GetRef()).ContinuationToken); + UNIT_ASSERT(event && std::holds_alternative(event.value())); + auto token = std::move(std::get(event.value()).ContinuationToken); ws->Write(std::move(token), std::move(params)); }; @@ -693,7 +693,7 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) } } - auto& v = event.GetRef(); + auto& v = event.value(); if (auto e = std::get_if(&v); e) { ++acks; } else if (auto e = std::get_if(&v); e) { @@ -708,7 +708,7 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) auto TFixture::CreateTopicWriteSession(const TString& topicPath, const TString& messageGroupId, - TMaybe partitionId) -> TTopicWriteSessionPtr + std::optional partitionId) -> TTopicWriteSessionPtr { NTopic::TTopicClient client(GetDriver()); NTopic::TWriteSessionSettings options; @@ -722,7 +722,7 @@ auto TFixture::CreateTopicWriteSession(const TString& topicPath, auto TFixture::GetTopicWriteSession(const TString& topicPath, const TString& messageGroupId, - TMaybe partitionId) -> TTopicWriteSessionContext& + std::optional partitionId) -> TTopicWriteSessionContext& { std::pair key(topicPath, messageGroupId); auto i = TopicWriteSessions.find(key); @@ -861,7 +861,7 @@ void TFixture::WriteToTopic(const TString& topicPath, const TString& messageGroupId, const TString& message, NTable::TTransaction* tx, - TMaybe partitionId) + std::optional partitionId) { TTopicWriteSessionContext& context = GetTopicWriteSession(topicPath, messageGroupId, partitionId); context.WaitForContinuationToken(); @@ -896,7 +896,7 @@ TVector TFixture::ReadFromTopic(const TString& topicPath, if (auto* e = std::get_if(&event)) { Cerr << e->HasCompressedMessages() << " " << e->GetMessagesCount() << Endl; for (auto& m : e->GetMessages()) { - messages.push_back(m.GetData()); + messages.emplace_back(m.GetData()); } if (!tx) { @@ -2312,31 +2312,31 @@ Y_UNIT_TEST_F(ReadRuleGeneration, TFixture) NotifySchemeShard({.EnablePQConfigTransactionsAtSchemeShard = false}); // Users have created their own topic on it - CreateTopic(TEST_TOPIC); + CreateTopic(TString{TEST_TOPIC}); // And they wrote their messages into it - WriteToTopic(TEST_TOPIC, TEST_MESSAGE_GROUP_ID, "message-1"); - WriteToTopic(TEST_TOPIC, TEST_MESSAGE_GROUP_ID, "message-2"); - WriteToTopic(TEST_TOPIC, TEST_MESSAGE_GROUP_ID, "message-3"); + WriteToTopic(TString{TEST_TOPIC}, TEST_MESSAGE_GROUP_ID, "message-1"); + WriteToTopic(TString{TEST_TOPIC}, TEST_MESSAGE_GROUP_ID, "message-2"); + WriteToTopic(TString{TEST_TOPIC}, TEST_MESSAGE_GROUP_ID, "message-3"); // And he had a consumer - AddConsumer(TEST_TOPIC, {"consumer-1"}); + AddConsumer(TString{TEST_TOPIC}, {"consumer-1"}); // We read messages from the topic and committed offsets - Read_Exactly_N_Messages_From_Topic(TEST_TOPIC, "consumer-1", 3); - CloseTopicReadSession(TEST_TOPIC, "consumer-1"); + Read_Exactly_N_Messages_From_Topic(TString{TEST_TOPIC}, "consumer-1", 3); + CloseTopicReadSession(TString{TEST_TOPIC}, "consumer-1"); // And then the Logbroker team turned on the feature flag NotifySchemeShard({.EnablePQConfigTransactionsAtSchemeShard = true}); // Users continued to write to the topic - WriteToTopic(TEST_TOPIC, TEST_MESSAGE_GROUP_ID, "message-4"); + WriteToTopic(TString{TEST_TOPIC}, TEST_MESSAGE_GROUP_ID, "message-4"); // Users have added new consumers - AddConsumer(TEST_TOPIC, {"consumer-2"}); + AddConsumer(TString{TEST_TOPIC}, {"consumer-2"}); // And they wanted to continue reading their messages - Read_Exactly_N_Messages_From_Topic(TEST_TOPIC, "consumer-1", 1); + Read_Exactly_N_Messages_From_Topic(TString{TEST_TOPIC}, "consumer-1", 1); } Y_UNIT_TEST_F(WriteToTopic_Demo_40, TFixture) diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/trace_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/trace_ut.cpp similarity index 99% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/trace_ut.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/trace_ut.cpp index b89ac38e14a5..4272e57e77af 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/trace_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/trace_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NYdb::NTopic::NTests { diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/managed_executor.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/managed_executor.cpp similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/managed_executor.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/managed_executor.cpp diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/managed_executor.h b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/managed_executor.h similarity index 89% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/managed_executor.h rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/managed_executor.h index 300d67a71b58..02f6d521d3d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/managed_executor.h +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/managed_executor.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp index dabad5b62f2d..c3253d1ea766 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp @@ -100,7 +100,7 @@ TDriverConfig TTopicSdkTestSetup::MakeDriverConfig() const config.SetEndpoint(GetEndpoint()); config.SetDatabase(GetDatabase()); config.SetAuthToken("root@builtin"); - config.SetLog(MakeHolder(&Cerr)); + config.SetLog(std::make_unique(&Cerr)); return config; } diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h similarity index 66% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h index cc459d6bda60..7a28780148a4 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h @@ -1,14 +1,14 @@ #pragma once -#include +#include -#include +#include namespace NYdb::NTopic::NTests { #define TEST_CASE_NAME (this->Name_) -inline static const TString TEST_TOPIC = "test-topic"; +inline static const std::string TEST_TOPIC = "test-topic"; inline static const TString TEST_CONSUMER = "test-consumer"; inline static const TString TEST_MESSAGE_GROUP_ID = "test-message_group_id"; @@ -16,15 +16,15 @@ class TTopicSdkTestSetup { public: TTopicSdkTestSetup(const TString& testCaseName, const NKikimr::Tests::TServerSettings& settings = MakeServerSettings(), bool createTopic = true); - void CreateTopic(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, + void CreateTopic(const TString& path = TString{TEST_TOPIC}, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, std::optional maxPartitionCount = std::nullopt); - void CreateTopicWithAutoscale(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, + void CreateTopicWithAutoscale(const TString& path = TString{TEST_TOPIC}, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, size_t maxPartitionCount = 100); - TTopicDescription DescribeTopic(const TString& path = TEST_TOPIC); + TTopicDescription DescribeTopic(const TString& path = TString{TEST_TOPIC}); TString GetEndpoint() const; - TString GetTopicPath(const TString& name = TEST_TOPIC) const; + TString GetTopicPath(const TString& name = TString{TEST_TOPIC}) const; TString GetTopicParent() const; TString GetDatabase() const; diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.cpp similarity index 98% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.cpp rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.cpp index 4eddb8eeb6d8..8b76f2dcd35c 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include "library/cpp/testing/unittest/registar.h" diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.h b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.h similarity index 94% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.h rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.h index 642ee7399dc4..8035e4ff800a 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/trace.h +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/trace.h @@ -1,6 +1,6 @@ #pragma once -#include "ydb/public/sdk/cpp/client/ydb_topic/common/trace_lazy.h" +#include "src/client/topic/common/trace_lazy.h" #include "library/cpp/logger/backend.h" #include "library/cpp/logger/record.h" @@ -61,7 +61,7 @@ bool CleanTraceEventBuf(TStringBuf& b, TStringBuf traceEventMarker); // To get the trace, call GetEvents method. class TTracingBackend : public TLogBackend { public: - TTracingBackend(const TString& traceEventMarker = TRACE_EVENT_MARKER, std::function parser = TTraceEvent::FromString) + TTracingBackend(const TString& traceEventMarker = TString{TRACE_EVENT_MARKER}, std::function parser = TTraceEvent::FromString) : TraceEventMarker(traceEventMarker) , EventParser(parser) {} diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/ya.make b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/ya.make similarity index 56% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/ya.make rename to ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/ya.make index e66da5786647..41c12850017d 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/ya.make +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ut_utils/ya.make @@ -1,5 +1,7 @@ LIBRARY() +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + SRCS( managed_executor.cpp managed_executor.h @@ -14,10 +16,10 @@ PEERDIR( library/cpp/testing/unittest library/cpp/threading/chunk_queue ydb/core/testlib/default - ydb/library/persqueue/topic_parser_public - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ya.make b/ydb/public/sdk/cpp/src/client/topic/ut/ya.make similarity index 50% rename from ydb/public/sdk/cpp/client/ydb_topic/ut/ya.make rename to ydb/public/sdk/cpp/src/client/topic/ut/ya.make index 54f513b22d31..61c371b6de60 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ya.make +++ b/ydb/public/sdk/cpp/src/client/topic/ut/ya.make @@ -1,4 +1,6 @@ -UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_topic) +UNITTEST_FOR(ydb/public/sdk/cpp/src/client/topic) + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) SIZE(LARGE) @@ -14,16 +16,15 @@ PEERDIR( ydb/core/testlib/default ydb/public/lib/json_value ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_driver - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_persqueue_public/impl - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_topic/include - ydb/public/sdk/cpp/client/ydb_topic/common - ydb/public/sdk/cpp/client/ydb_topic/impl - ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public/impl + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/topic/common + ydb/public/sdk/cpp/src/client/topic/impl + ydb/public/sdk/cpp/src/client/topic/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers ydb/core/persqueue/ut/common diff --git a/ydb/public/sdk/cpp/src/client/topic/ya.make b/ydb/public/sdk/cpp/src/client/topic/ya.make new file mode 100644 index 000000000000..4aa545a57bda --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/topic/ya.make @@ -0,0 +1,29 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + out.cpp + proto_accessor.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/topic/codecs + ydb/public/sdk/cpp/src/client/topic/common + ydb/public/sdk/cpp/src/client/topic/impl + ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic + + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table + + ydb/public/api/grpc + ydb/public/api/grpc/draft + ydb/public/api/protos +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/sdk/cpp/src/client/types/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/CMakeLists.txt new file mode 100644 index 000000000000..4826037a4c0e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/CMakeLists.txt @@ -0,0 +1,29 @@ +add_subdirectory(credentials) +add_subdirectory(exceptions) +add_subdirectory(fatal_error_handlers) +add_subdirectory(operation) +add_subdirectory(status) + +_ydb_sdk_add_library(client-ydb_types) + +target_link_libraries(client-ydb_types PUBLIC + yutil + protobuf::libprotobuf + grpc-client + yql-public-issue + enum_serialization_runtime +) + +generate_enum_serilization(client-ydb_types + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/types/s3_settings.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/types/s3_settings.h +) + +generate_enum_serilization(client-ydb_types + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/types/status_codes.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/types/status_codes.h +) + +_ydb_sdk_make_client_component(Types client-ydb_types) diff --git a/ydb/public/sdk/cpp/src/client/types/core_facility/core_facility.h b/ydb/public/sdk/cpp/src/client/types/core_facility/core_facility.h new file mode 100644 index 000000000000..772512bdb6b9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/core_facility/core_facility.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3 { +using TPeriodicCb = std::function; + +// !!!Experimental!!! +// Allows to communicate with sdk core +class ICoreFacility { +public: + virtual ~ICoreFacility() = default; + // Add task to execute periodicaly + // Task should return false to stop execution + virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/credentials/CMakeLists.txt new file mode 100644 index 000000000000..1d2fc50cdbcc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/CMakeLists.txt @@ -0,0 +1,20 @@ +add_subdirectory(login) +add_subdirectory(oauth2_token_exchange) + +_ydb_sdk_add_library(client-ydb_types-credentials) + +target_link_libraries(client-ydb_types-credentials PUBLIC + yutil + api-grpc + client-ydb_types-status + yql-public-issue + client-ydb_types-credentials-login + client-ydb_types-credentials-oauth2 +) + +target_sources(client-ydb_types-credentials PRIVATE + credentials.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-credentials) +_ydb_sdk_make_client_component(Credentials client-ydb_types-credentials) diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/credentials.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/credentials.cpp new file mode 100644 index 000000000000..2a3e0a83a948 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/credentials.cpp @@ -0,0 +1,82 @@ +#include +#include + +namespace NYdb::inline V3 { + +class TInsecureCredentialsProvider : public ICredentialsProvider { +public: + TInsecureCredentialsProvider() + {} + + std::string GetAuthInfo() const override { + return std::string(); + } + + bool IsValid() const override { + return false; + } +}; + +std::string ICredentialsProviderFactory::GetClientIdentity() const { + return ToString((ui64)this); +} + +class TInsecureCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TInsecureCredentialsProviderFactory() + {} + + std::shared_ptr CreateProvider() const override { + return std::make_shared(); + } + + std::string GetClientIdentity() const override { + return std::string(); + } +}; + +class TOAuthCredentialsProvider : public ICredentialsProvider { +public: + TOAuthCredentialsProvider(const std::string& token) + : Token(token) + {} + + std::string GetAuthInfo() const override { + return Token; + } + + bool IsValid() const override { + return !Token.empty(); + } + +private: + std::string Token; +}; + +class TOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TOAuthCredentialsProviderFactory(const std::string& token) + : Token(token) + {} + + std::shared_ptr CreateProvider() const override { + return std::make_shared(Token); + } + + std::string GetClientIdentity() const override { + return Token; + } + +private: + std::string Token; +}; + +std::shared_ptr CreateInsecureCredentialsProviderFactory() { + return std::make_shared(); +} +std::shared_ptr CreateOAuthCredentialsProviderFactory(const std::string& token) { + return std::make_shared(token); +} + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/login/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/credentials/login/CMakeLists.txt new file mode 100644 index 000000000000..218685ff0da7 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/login/CMakeLists.txt @@ -0,0 +1,18 @@ +_ydb_sdk_add_library(client-ydb_types-credentials-login) + +target_link_libraries(client-ydb_types-credentials-login + PUBLIC + yutil + PRIVATE + api-grpc + client-ydb_types-status + impl-ydb_internal-grpc_connections + yql-public-issue + jwt-cpp::jwt-cpp +) + +target_sources(client-ydb_types-credentials-login PRIVATE + login.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-credentials-login) diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp new file mode 100644 index 000000000000..8597aa4d7606 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp @@ -0,0 +1,249 @@ +#include +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + +namespace { + +// copy-pasted from +template +constexpr ui64 ToMicroseconds(std::chrono::duration value) { + return std::chrono::duration_cast(value).count(); +} + +template +constexpr TInstant ToInstant(std::chrono::time_point value) { + return TInstant::MicroSeconds(ToMicroseconds(value.time_since_epoch())); +} + +std::chrono::system_clock::time_point GetTokenExpiresAt(const std::string& token) { + try { + jwt::decoded_jwt decoded_token = jwt::decode(token); + return decoded_token.get_expires_at(); + } + catch (...) { + } + return {}; +} +} + +class TLoginCredentialsProvider : public ICredentialsProvider { +public: + TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params); + virtual std::string GetAuthInfo() const override; + virtual bool IsValid() const override; + +private: + void PrepareToken(); + void RequestToken(); + bool IsOk() const; + void ParseToken(); + std::string GetToken() const; + std::string GetError() const; + std::string GetTokenOrError() const; + + enum class EState { + Empty, + Requesting, + Done, + }; + + std::weak_ptr Facility_; + TLoginCredentialsParams Params_; + EState State_ = EState::Empty; + std::mutex Mutex_; + std::condition_variable Notify_; + std::atomic TokenReceived_ = 1; + std::atomic TokenParsed_ = 0; + std::optional Token_; + std::optional Error_; + TInstant TokenExpireAt_; + TInstant TokenRequestAt_; + TPlainStatus Status_; + Ydb::Auth::LoginResponse Response_; +}; + +TLoginCredentialsProvider::TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params) + : Facility_(facility) + , Params_(std::move(params)) +{ + auto strongFacility = facility.lock(); + if (strongFacility) { + auto periodicTask = [facility, this](NYdb::NIssue::TIssues&&, EStatus status) -> bool { + if (status != EStatus::SUCCESS) { + return false; + } + + auto strongFacility = facility.lock(); + if (!strongFacility) { + return false; + } + + if (!TokenRequestAt_) { + return true; + } + + if (TInstant::Now() >= TokenRequestAt_) { + RequestToken(); + } + + return true; + }; + strongFacility->AddPeriodicTask(std::move(periodicTask), TDuration::Minutes(1)); + } +} + +bool TLoginCredentialsProvider::IsValid() const { + return true; +} + +std::string TLoginCredentialsProvider::GetAuthInfo() const { + if (TokenParsed_ == TokenReceived_) { + return GetTokenOrError(); + } else { + const_cast(this)->PrepareToken(); // will block here + return GetTokenOrError(); + } +} + +void TLoginCredentialsProvider::RequestToken() { + auto strongFacility = Facility_.lock(); + if (strongFacility) { + TokenRequestAt_ = {}; + + auto responseCb = [facility = Facility_, this](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { + auto strongFacility = facility.lock(); + if (strongFacility) { + std::lock_guard lock(Mutex_); + Status_ = std::move(status); + if (resp != nullptr) { + Response_ = std::move(*resp); + } + State_ = EState::Done; + TokenReceived_++; + } + Notify_.notify_all(); + }; + + Ydb::Auth::LoginRequest request; + request.set_user(TStringType{Params_.User}); + request.set_password(TStringType{Params_.Password}); + TRpcRequestSettings rpcSettings; + rpcSettings.ClientTimeout = TDuration::Seconds(60); + + TGRpcConnectionsImpl::RunOnDiscoveryEndpoint( + strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, + rpcSettings); + } +} + +void TLoginCredentialsProvider::PrepareToken() { + std::unique_lock lock(Mutex_); + switch (State_) { + case EState::Empty: + State_ = EState::Requesting; + RequestToken(); + [[fallthrough]]; + case EState::Requesting: + Notify_.wait(lock, [&]{ + return State_ == EState::Done; + }); + [[fallthrough]]; + case EState::Done: + ParseToken(); + break; + } +} + +bool TLoginCredentialsProvider::IsOk() const { + return State_ == EState::Done + && Status_.Ok() + && Response_.operation().status() == Ydb::StatusIds::SUCCESS; +} + +void TLoginCredentialsProvider::ParseToken() { // works under mutex + if (TokenParsed_ != TokenReceived_) { + if (IsOk()) { + Token_ = GetToken(); + Error_.reset(); + TInstant now = TInstant::Now(); + TokenExpireAt_ = ToInstant(GetTokenExpiresAt(Token_.value())); + TokenRequestAt_ = now + TDuration::Minutes((TokenExpireAt_ - now).Minutes() / 2); + } else { + Token_.reset(); + Error_ = GetError(); + } + TokenParsed_ = TokenReceived_.load(); + } +} + +std::string TLoginCredentialsProvider::GetToken() const { + Ydb::Auth::LoginResult result; + Response_.operation().result().UnpackTo(&result); + return result.token(); +} + +std::string TLoginCredentialsProvider::GetError() const { + if (Status_.Ok()) { + if (Response_.operation().issues_size() > 0) { + return Response_.operation().issues(0).message(); + } else { + return Ydb::StatusIds_StatusCode_Name(Response_.operation().status()); + } + } else { + TStringBuilder str; + str << "Couldn't get token for provided credentials from " << Status_.Endpoint + << " with status " << Status_.Status << "."; + for (const auto& issue : Status_.Issues) { + str << Endl << "Issue: " << issue; + } + return str; + } +} + +std::string TLoginCredentialsProvider::GetTokenOrError() const { + if (Token_) { + return Token_.value(); + } + if (Error_) { + ythrow yexception() << Error_.value(); + } + ythrow yexception() << "Wrong state of credentials provider"; +} + +class TLoginCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TLoginCredentialsProviderFactory(TLoginCredentialsParams params); + virtual std::shared_ptr CreateProvider() const override; + virtual std::shared_ptr CreateProvider(std::weak_ptr facility) const override; + +private: + TLoginCredentialsParams Params_; +}; + +TLoginCredentialsProviderFactory::TLoginCredentialsProviderFactory(TLoginCredentialsParams params) + : Params_(std::move(params)) +{ +} + +std::shared_ptr TLoginCredentialsProviderFactory::CreateProvider() const { + ythrow yexception() << "Not supported"; +} + +std::shared_ptr TLoginCredentialsProviderFactory::CreateProvider(std::weak_ptr facility) const { + return std::make_shared(std::move(facility), Params_); +} + +std::shared_ptr CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params) { + return std::make_shared(std::move(params)); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/login/ya.make b/ydb/public/sdk/cpp/src/client/types/credentials/login/ya.make new file mode 100644 index 000000000000..0aa2dde56183 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/login/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + login.cpp +) + +PEERDIR( + contrib/libs/jwt-cpp + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/types/status + ydb/public/sdk/cpp/src/client/impl/ydb_internal/grpc_connections + ydb/public/sdk/cpp/src/library/issue +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt new file mode 100644 index 000000000000..5865f03a1be9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt @@ -0,0 +1,30 @@ +_ydb_sdk_add_library(client-ydb_types-credentials-oauth2) + +target_link_libraries(client-ydb_types-credentials-oauth2 + PUBLIC + yutil + jwt-cpp::jwt-cpp + cgiparam + http-misc + http-simple + json + retry + string_utils-base64 + uri + client-ydb_types-credentials + client-ydb_types +) + +target_compile_definitions(client-ydb_types-credentials-oauth2 + PUBLIC + YDB_SDK_USE_NEW_JWT +) + +target_sources(client-ydb_types-credentials-oauth2 + PRIVATE + credentials.cpp + from_file.cpp + jwt_token_source.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-credentials-oauth2) diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp new file mode 100644 index 000000000000..cae23a9b228f --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -0,0 +1,531 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#define PROV_ERR "Oauth 2 token exchange credentials provider: " +#define INV_ARG "Invalid argument for " PROV_ERR +#define EXCH_ERR "Exchange token error in " PROV_ERR + +namespace NYdb::inline V3 { + +namespace { + +// compares str to already lowered lowerStr +bool LowerAsciiEqual(std::string_view str, std::string_view lowerStr) { + if (str.size() != lowerStr.size()) { + return false; + } + for (size_t i = 0; i < lowerStr.size(); ++i) { + char c = str[i]; + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + if (c != lowerStr[i]) { + return false; + } + } + return true; +} + +using TChronoDuration = std::chrono::duration; + +class TFixedTokenSource: public ITokenSource { +public: + TFixedTokenSource(const std::string& token, const std::string& tokenType) + : Token{token, tokenType} + { + } + + TToken GetToken() const override { + return Token; + } + + TToken Token; +}; + +class TTokenExchangeError: public std::runtime_error { +public: + explicit TTokenExchangeError(const std::string& s, TKeepAliveHttpClient::THttpCode code = 0) + : std::runtime_error(s) + , HttpCode(code) + { + } + + explicit TTokenExchangeError(const char* s, TKeepAliveHttpClient::THttpCode code = 0) + : std::runtime_error(s) + , HttpCode(code) + { + } + +public: + TKeepAliveHttpClient::THttpCode HttpCode = 0; +}; + +bool IsRetryableError(TKeepAliveHttpClient::THttpCode code) { + return code == HTTP_REQUEST_TIME_OUT + || code == HTTP_AUTHENTICATION_TIMEOUT + || code == HTTP_TOO_MANY_REQUESTS + || code == HTTP_INTERNAL_SERVER_ERROR + || code == HTTP_BAD_GATEWAY + || code == HTTP_SERVICE_UNAVAILABLE + || code == HTTP_GATEWAY_TIME_OUT; +} + +ERetryErrorClass RetryPolicyClass(TKeepAliveHttpClient::THttpCode code) { + return IsRetryableError(code) ? ERetryErrorClass::LongRetry : ERetryErrorClass::ShortRetry; // In case when we already have token we know that all params are correct and all errors are temporary +} + +ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErrors) { + if (const TTokenExchangeError* err = dynamic_cast(ex)) { + if (IsRetryableError(err->HttpCode)) { + return ERetryErrorClass::LongRetry; + } + if (retryAllErrors) { + return ERetryErrorClass::ShortRetry; + } + } + if (const TSystemError* err = dynamic_cast(ex)) { + return ERetryErrorClass::ShortRetry; + } + + // Else this is a usual error like bad response + return ERetryErrorClass::NoRetry; +} + +using TRetryPolicy = IRetryPolicy; +using TSyncRetryPolicy = IRetryPolicy; + +struct TPrivateOauth2TokenExchangeParams: public TOauth2TokenExchangeParams { + TPrivateOauth2TokenExchangeParams(const TOauth2TokenExchangeParams& params) + : TOauth2TokenExchangeParams(params) + { + if (TokenEndpoint_.empty()) { + throw std::invalid_argument(INV_ARG "no token endpoint"); + } + + for (const std::string& aud : Audience_) { + if (aud.empty()) { + throw std::invalid_argument(INV_ARG "empty audience"); + } + } + + for (const std::string& scope : Scope_) { + if (scope.empty()) { + throw std::invalid_argument(INV_ARG "empty scope"); + } + } + + ParseTokenEndpoint(); + } + + TPrivateOauth2TokenExchangeParams(const TPrivateOauth2TokenExchangeParams&) = default; + +public: + std::string TokenHost_; // https://host + ui16 TokenPort_; + std::string TokenRequest_; // Path without host:port + +private: + void ParseTokenEndpoint() { + if (TokenEndpoint_.empty()) { + throw std::invalid_argument(INV_ARG "token endpoint not set"); + } + NUri::TUri url; + NUri::TUri::TState::EParsed parseStatus = url.Parse(TokenEndpoint_, NUri::TFeature::FeaturesAll); + if (parseStatus != NUri::TUri::TState::EParsed::ParsedOK) { + throw std::invalid_argument(INV_ARG "failed to parse url"); + } + if (url.IsNull(NUri::TUri::FieldScheme)) { + throw std::invalid_argument(TStringBuilder() << INV_ARG "token url without scheme: " << TokenEndpoint_); + } + TokenHost_ = TStringBuilder() << url.GetField(NUri::TUri::FieldScheme) << "://" << url.GetHost(); + + if (url.IsNull(NUri::TUri::FieldPort)) { + TokenPort_ = 80; + if (url.GetScheme() == NUri::TScheme::SchemeHTTPS) { + TokenPort_ = 443; + } + } else { + TokenPort_ = FromString(url.GetField(NUri::TUri::FieldPort)); + } + + TStringBuilder req; + req << url.GetField(NUri::TUri::FieldPath); + if (!url.IsNull(NUri::TUri::FieldQuery)) { + req << '?' << url.GetField(NUri::TUri::FieldQuery); + } + TokenRequest_ = std::move(req); + } +}; + +class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this +{ + struct TTokenExchangeResult { + std::string Token; + TInstant TokenDeadline; + TInstant TokenRefreshTime; + }; + +public: + explicit TOauth2TokenExchangeProviderImpl(const TPrivateOauth2TokenExchangeParams& params) + : Params(params) + { + ExchangeTokenSync(TInstant::Now()); + } + + void Stop() { + { + std::unique_lock lock(StopMutex); + Stopping = true; + StopVar.notify_all(); + } + + if (RefreshTokenThread.joinable()) { + RefreshTokenThread.join(); + } + } + + std::string GetAuthInfo() const { + const TInstant now = TInstant::Now(); + std::string token; + with_lock (Lock) { + if (Token.empty() || now >= TokenDeadline) { // Update sync. This can be if we have repeating error during token refresh process. In this case we will try for the last time and throw an error + ExchangeTokenSync(now, true); + token = Token; + } else { + if (now >= TokenRefreshTime) { + TryRefreshToken(); + } + token = Token; // Still valid + } + } + return token; + } + +private: + std::string GetScopeParam() const { + TStringBuilder scope; + for (const std::string& s : Params.Scope_) { + if (!scope.empty()) { + scope << ' '; + } + scope << s; + } + return std::move(scope); + } + + std::string GetRequestParams() const { + TCgiParameters params; + params.emplace("grant_type", Params.GrantType_); + params.emplace("requested_token_type", Params.RequestedTokenType_); + + auto addIfNotEmpty = [&](std::string_view name, const std::string& value) { + if (!value.empty()) { + params.emplace(name, value); + } + }; + + for (const std::string& res : Params.Resource_) { + params.emplace("resource", res); + } + for (const std::string& aud : Params.Audience_) { + params.emplace("audience", aud); + } + addIfNotEmpty("scope", GetScopeParam()); + + auto addTokenSource = [&](std::string_view tokenParamName, std::string_view tokenTypeParamName, const std::shared_ptr& tokenSource) { + if (tokenSource) { + const ITokenSource::TToken token = tokenSource->GetToken(); + params.emplace(tokenParamName, token.Token); + params.emplace(tokenTypeParamName, token.TokenType); + } + }; + + addTokenSource("subject_token", "subject_token_type", Params.SubjectTokenSource_); + addTokenSource("actor_token", "actor_token_type", Params.ActorTokenSource_); + + return params.Print(); + } + + void RaiseError(TKeepAliveHttpClient::THttpCode statusCode, TStringStream& responseStream) const { + TStringBuilder err; + err << EXCH_ERR << HttpCodeStrEx(statusCode); + + try { + NJson::TJsonValue responseJson; + NJson::ReadJsonTree(&responseStream, &responseJson, true); + const auto& resp = responseJson.GetMap(); + + auto optionalStringParam = [&](std::string_view name) -> std::string_view { + auto iter = resp.find(std::string{name}); + if (iter == resp.end()) { + return {}; + } + return iter->second.GetString(); + }; + + std::string_view error = optionalStringParam("error"); + if (!error.empty()) { + err << ", error: " << error; + } + + std::string_view description = optionalStringParam("error_description"); + if (!description.empty()) { + err << ", description: " << description; + } + + std::string_view uri = optionalStringParam("error_uri"); + if (!uri.empty()) { + err << ", error_uri: " << uri; + } + } catch (const std::exception& ex) { + err << ", could not parse response: " << ex.what(); + } + + throw TTokenExchangeError(err, statusCode); + } + + TTokenExchangeResult ProcessExchangeTokenResponse(TInstant now, TKeepAliveHttpClient::THttpCode statusCode, TStringStream& responseStream) const { + if (statusCode != HTTP_OK) { + RaiseError(statusCode, responseStream); + } + + NJson::TJsonValue responseJson; + try { + NJson::ReadJsonTree(&responseStream, &responseJson, true); + } catch (const std::exception& ex) { + throw std::runtime_error(TStringBuilder() << EXCH_ERR "json parsing error: " << ex.what()); + } + + const auto& resp = responseJson.GetMap(); + + auto requiredParam = [&](std::string_view name) -> const NJson::TJsonValue& { + auto iter = resp.find(std::string{name}); + if (iter == resp.end()) { + throw std::runtime_error(TStringBuilder() << EXCH_ERR "no field \"" << name << "\" in response"); + } + return iter->second; + }; + + const std::string& tokenType = requiredParam("token_type").GetString(); + if (!LowerAsciiEqual(tokenType, "bearer")) { + throw std::runtime_error(TStringBuilder() << EXCH_ERR "unsupported token type: \"" << tokenType << "\""); + } + + const long long expireTime = requiredParam("expires_in").GetIntegerRobust(); // also converts from string/double + if (expireTime <= 0) { + throw std::runtime_error(TStringBuilder() << EXCH_ERR "incorrect expiration time: " << expireTime); + } + + auto scopeIt = resp.find("scope"); + if (scopeIt != resp.end() && scopeIt->second.GetString() != GetScopeParam()) { + throw std::runtime_error(TStringBuilder() << EXCH_ERR "different scope. Expected \"" + << GetScopeParam() << "\", but got \"" << scopeIt->second.GetString() << "\""); + } + + const std::string& token = requiredParam("access_token").GetString(); + if (token.empty()) { + throw std::runtime_error(EXCH_ERR "got empty token"); + } + + const TDuration expireDelta = TDuration::Seconds(expireTime); + TTokenExchangeResult result; + result.TokenDeadline = now + expireDelta; + result.TokenRefreshTime = now + expireDelta / 2; + result.Token = TStringBuilder() << "Bearer " << token; + return result; + } + + // May be run without lock + // Can throw exceptions + TTokenExchangeResult ExchangeToken(TInstant now) const { + TKeepAliveHttpClient client(TString(Params.TokenHost_), Params.TokenPort_, Params.SocketTimeout_, Params.ConnectTimeout_); + TStringStream responseStream; + TKeepAliveHttpClient::THeaders headers; + headers["Content-Type"] = "application/x-www-form-urlencoded"; + const TKeepAliveHttpClient::THttpCode statusCode = client.DoPost(Params.TokenRequest_, GetRequestParams(), &responseStream, headers); + return ProcessExchangeTokenResponse(now, statusCode, responseStream); + } + + void ExchangeTokenSync(TInstant now, bool retryAllErrors = false) const { // Is run under lock + TSyncRetryPolicy::IRetryState::TPtr retryState; + while (true) { + try { + TTokenExchangeResult result = ExchangeToken(now); + Token = result.Token; + TokenDeadline = result.TokenDeadline; + TokenRefreshTime = result.TokenRefreshTime; + break; + } catch (const std::exception& ex) { + if (!retryState) { + retryState = TSyncRetryPolicy::GetFixedIntervalPolicy( // retry more aggresively when we are in sync mode + SyncRetryPolicyClass, + TDuration::MilliSeconds(100), // delay // default + TDuration::MilliSeconds(300), // long delay // default + std::numeric_limits::max(), // max retries // default + Params.SyncUpdateTimeout_ // max time + )->CreateRetryState(); + } + if (auto interval = retryState->GetNextRetryDelay(&ex, retryAllErrors)) { + Sleep(*interval); + } else { + throw; + } + } + } + } + + bool IsStopping() const { + std::unique_lock lock(StopMutex); + return Stopping; + } + + void RefreshToken() const { + TRetryPolicy::IRetryState::TPtr retryState; + TInstant deadline; + with_lock (Lock) { + deadline = TokenDeadline; + } + while (!IsStopping()) { + const TInstant now = TInstant::Now(); + try { + TTokenExchangeResult result = ExchangeToken(now); + with_lock (Lock) { + Token = result.Token; + TokenDeadline = result.TokenDeadline; + TokenRefreshTime = result.TokenRefreshTime; + break; + } + } catch (const std::exception& ex) { // If this error will repeat, we finally will get it syncronously in GetAuthInfo() and pass to client + if (!retryState) { + retryState = TRetryPolicy::GetExponentialBackoffPolicy( + RetryPolicyClass, + TDuration::MilliSeconds(10), // min delay // default + TDuration::MilliSeconds(200), // min long delay // default + TDuration::Seconds(30), // max delay // default + std::numeric_limits::max(), // max retries // default + deadline - TInstant::Now() // max time + )->CreateRetryState(); + } else { + TKeepAliveHttpClient::THttpCode code = HTTP_CODE_MAX; + if (const auto* err = dynamic_cast(&ex)) { + code = err->HttpCode; + } + if (auto delay = retryState->GetNextRetryDelay(code)) { + std::unique_lock lock(StopMutex); + const bool stopping = StopVar.wait_for( + lock, + TChronoDuration(delay->GetValue()), + [this]() { + return Stopping; + } + ); + if (stopping) { + break; + } + } else { + break; + } + } + } + } + TokenIsRefreshing = false; + } + + void TryRefreshToken() const { // Is run under lock + if (TokenIsRefreshing) { + return; + } + if (RefreshTokenThread.joinable()) { + RefreshTokenThread.join(); + } + + TokenIsRefreshing = true; + RefreshTokenThread = std::thread( + [w = weak_from_this()]() { + if (auto p = w.lock()) { + p->RefreshToken(); + } + } + ); + } + +private: + TPrivateOauth2TokenExchangeParams Params; + + TAdaptiveLock Lock; + mutable bool TokenIsRefreshing = false; + mutable std::thread RefreshTokenThread; + mutable std::string Token; + mutable TInstant TokenDeadline; + mutable TInstant TokenRefreshTime; + + // Stop + bool Stopping = false; + mutable std::mutex StopMutex; + mutable std::condition_variable StopVar; +}; + +class TOauth2TokenExchangeProvider: public ICredentialsProvider { +public: + explicit TOauth2TokenExchangeProvider(const TPrivateOauth2TokenExchangeParams& params) + : Impl(std::make_shared(params)) + { + } + + std::string GetAuthInfo() const override { + return Impl->GetAuthInfo(); + } + + bool IsValid() const override { + return true; + } + + ~TOauth2TokenExchangeProvider() { // The last link tp provider is gone + Impl->Stop(); + } + +private: + std::shared_ptr Impl; +}; + +class TOauth2TokenExchangeFactory: public ICredentialsProviderFactory { +public: + explicit TOauth2TokenExchangeFactory(const TOauth2TokenExchangeParams& params) + : Provider(std::make_shared(params)) + { + } + + TCredentialsProviderPtr CreateProvider() const override { + return Provider; + } + +private: + std::shared_ptr Provider; +}; + +} // namespace + +std::shared_ptr CreateOauth2TokenExchangeCredentialsProviderFactory(const TOauth2TokenExchangeParams& params) { + return std::make_shared(params); +} + +std::shared_ptr CreateFixedTokenSource(const std::string& token, const std::string& tokenType) { + return std::make_shared(token, tokenType); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/from_file.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/from_file.cpp new file mode 100644 index 000000000000..6c99107170a6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/from_file.cpp @@ -0,0 +1,271 @@ +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +#include + +namespace NYdb::inline V3 { + +namespace { + +struct TLessNoCase { + bool operator()(std::string_view l, std::string_view r) const { + auto ll = l.length(); + auto rl = r.length(); + if (ll != rl) { + return ll < rl; + } + return strnicmp(l.data(), r.data(), ll) < 0; + } +}; + +template +void ApplyAsymmetricAlg(TJwtTokenSourceParams* params, const std::string& privateKey) { + // Alg with first param as public key, second param as private key + params->SigningAlgorithm(std::string{}, privateKey); +} + +size_t Base64OutputLen(std::string_view input) { + while (!input.empty() && (input.back() == '=' || input.back() == ',')) { // padding + input.remove_suffix(1); + } + const size_t inputLen = input.size(); + const size_t tailEncoded = inputLen % 4; + if (tailEncoded == 1) { + throw std::runtime_error(TStringBuilder() << "invalid Base64 encoded data size: " << input.size()); + } + const size_t mainSize = (inputLen / 4) * 3; + size_t tailSize = 0; + switch (tailEncoded) { + case 2: // 12 bit => 1 byte + tailSize = 1; + break; + case 3: // 18 bits -> 2 bytes + tailSize = 2; + break; + } + return mainSize + tailSize; +} + +template +void ApplyHmacAlg(TJwtTokenSourceParams* params, const std::string& key) { + // HMAC keys are encoded in base64 encoding + const size_t base64OutputSize = Base64OutputLen(key); // throws + std::string binaryKey; + binaryKey.resize(Base64DecodeBufSize(key.size())); + // allows strings without padding + const size_t decodedBytes = Base64DecodeUneven(const_cast(binaryKey.data()), key); + if (decodedBytes != base64OutputSize) { + throw std::runtime_error("failed to decode HMAC secret from Base64"); + } + binaryKey.resize(decodedBytes); + // Alg with first param as key + params->SigningAlgorithm(binaryKey); +} + +const std::map JwtAlgorithmsFactory = { + {"RS256", &ApplyAsymmetricAlg}, + {"RS384", &ApplyAsymmetricAlg}, + {"RS512", &ApplyAsymmetricAlg}, + {"ES256", &ApplyAsymmetricAlg}, + {"ES384", &ApplyAsymmetricAlg}, + {"ES512", &ApplyAsymmetricAlg}, + {"PS256", &ApplyAsymmetricAlg}, + {"PS384", &ApplyAsymmetricAlg}, + {"PS512", &ApplyAsymmetricAlg}, + {"HS256", &ApplyHmacAlg}, + {"HS384", &ApplyHmacAlg}, + {"HS512", &ApplyHmacAlg}, +}; + +bool IsAsciiEqualUpper(const std::string& jsonParam, const std::string_view& constantInUpperCase) { + if (jsonParam.size() != constantInUpperCase.size()) { + return false; + } + for (size_t i = 0; i < constantInUpperCase.size(); ++i) { + char c = jsonParam[i]; + if (c >= 'a' && c <= 'z') { + c += 'A' - 'a'; + } + if (c != constantInUpperCase[i]) { + return false; + } + } + return true; +} + +// throws if not string +#define PROCESS_JSON_STRING_PARAM(jsonParamName, paramName, required) \ + try { \ + if (const NJson::TJsonValue* value = map.FindPtr(jsonParamName)) { \ + result.paramName(value->GetStringSafe()); \ + } else if (required) { \ + throw std::runtime_error( \ + "No \"" jsonParamName "\" parameter"); \ + } \ + } catch (const std::exception& ex) { \ + throw std::runtime_error(TStringBuilder() \ + << "Failed to parse \"" jsonParamName "\": " << ex.what()); \ + } + +// throws if not string or array of strings +#define PROCESS_JSON_ARRAY_PARAM(jsonParamName, paramName) \ + try { \ + if (const NJson::TJsonValue* value = map.FindPtr(jsonParamName)) { \ + if (value->IsArray()) { \ + for (const NJson::TJsonValue& v : value->GetArraySafe()) { \ + result.Append ## paramName(v.GetStringSafe()); \ + } \ + } else { \ + result.paramName(value->GetStringSafe()); \ + } \ + } \ + } catch (const std::exception& ex) { \ + throw std::runtime_error(TStringBuilder() \ + << "Failed to parse \"" jsonParamName "\": " << ex.what()); \ + } + +#define PROCESS_CREDS_PARAM(jsonParamName, paramName) \ + try { \ + if (const NJson::TJsonValue* value = map.FindPtr(jsonParamName)) { \ + result.paramName(ParseTokenSource(*value)); \ + } \ + } catch (const std::exception& ex) { \ + throw std::runtime_error(TStringBuilder() \ + << "Failed to parse \"" jsonParamName "\": " << ex.what()); \ + } + +// special struct to apply macros +struct TFixedTokenSourceParamsForParsing { + using TSelf = TFixedTokenSourceParamsForParsing; + + FLUENT_SETTING(std::string, Token); + FLUENT_SETTING(std::string, TokenType); +}; + +// special struct to apply macros +struct TJwtTokenSourceParamsForParsing : public TJwtTokenSourceParams { + FLUENT_SETTING(std::string, AlgStr); + FLUENT_SETTING(std::string, PrivateKeyStr); + FLUENT_SETTING(std::string, TtlStr); +}; + +std::shared_ptr ParseFixedTokenSource(const NJson::TJsonValue& cfg) { + const auto& map = cfg.GetMapSafe(); + TFixedTokenSourceParamsForParsing result; + PROCESS_JSON_STRING_PARAM("token", Token, true); // required + PROCESS_JSON_STRING_PARAM("token-type", TokenType, true); // required + return CreateFixedTokenSource(result.Token_, result.TokenType_); +} + +std::shared_ptr ParseJwtTokenSource(const NJson::TJsonValue& cfg) { + const auto& map = cfg.GetMapSafe(); + TJwtTokenSourceParamsForParsing result; + PROCESS_JSON_STRING_PARAM("kid", KeyId, false); + PROCESS_JSON_STRING_PARAM("iss", Issuer, false); + PROCESS_JSON_STRING_PARAM("sub", Subject, false); + PROCESS_JSON_ARRAY_PARAM("aud", Audience); + PROCESS_JSON_STRING_PARAM("jti", Id, false); + + // special fields + PROCESS_JSON_STRING_PARAM("ttl", TtlStr, false); + PROCESS_JSON_STRING_PARAM("alg", AlgStr, true); // required + PROCESS_JSON_STRING_PARAM("private-key", PrivateKeyStr, true); // required + + try { + if (!result.TtlStr_.empty()) { + result.TokenTtl(FromString(result.TtlStr_)); + } + } catch (const std::exception& ex) { + throw std::runtime_error(TStringBuilder() + << "Failed to parse \"ttl\": " << ex.what()); + } + + const auto jwtAlgIt = JwtAlgorithmsFactory.find(result.AlgStr_); + if (jwtAlgIt == JwtAlgorithmsFactory.end()) { + TStringBuilder err; + err << "Algorithm \"" << result.AlgStr_ << "\" is not supported. Supported algorithms are: "; + bool first = true; + for (const std::string& alg : GetSupportedOauth2TokenExchangeJwtAlgorithms()) { + if (!first) { + err << ", "; + } + first = false; + err << alg; + } + throw std::runtime_error(err); + } + jwtAlgIt->second(&result, result.PrivateKeyStr_); + return CreateJwtTokenSource(result); +} + +std::shared_ptr ParseTokenSource(const NJson::TJsonValue& cfg) { + const auto& map = cfg.GetMapSafe(); + const NJson::TJsonValue* type = map.FindPtr("type"); + if (!type) { + throw std::runtime_error("No \"type\" parameter"); + } + if (IsAsciiEqualUpper(type->GetStringSafe(), "JWT")) { + return ParseJwtTokenSource(cfg); + } else if (IsAsciiEqualUpper(type->GetStringSafe(), "FIXED")) { + return ParseFixedTokenSource(cfg); + } else { + throw std::runtime_error("Incorrect \"type\" parameter: only \"JWT\" and \"FIXED\" are supported"); + } +} + +TOauth2TokenExchangeParams ReadOauth2ConfigJson(const std::string& configJson, const std::string& tokenEndpoint) { + try { + NJson::TJsonValue cfg; + NJson::ReadJsonTree(configJson, &cfg, true); + const auto& map = cfg.GetMapSafe(); + + TOauth2TokenExchangeParams result; + if (!tokenEndpoint.empty()) { + result.TokenEndpoint(tokenEndpoint); + } else { + PROCESS_JSON_STRING_PARAM("token-endpoint", TokenEndpoint, false); // there is an explicit check in provider params after parsing + } + + PROCESS_JSON_STRING_PARAM("grant-type", GrantType, false); + PROCESS_JSON_ARRAY_PARAM("res", Resource); + PROCESS_JSON_STRING_PARAM("requested-token-type", RequestedTokenType, false); + PROCESS_JSON_ARRAY_PARAM("aud", Audience); + PROCESS_JSON_ARRAY_PARAM("scope", Scope); + PROCESS_CREDS_PARAM("subject-credentials", SubjectTokenSource); + PROCESS_CREDS_PARAM("actor-credentials", ActorTokenSource); + return result; + } catch (const std::exception& ex) { + throw std::runtime_error(TStringBuilder() << "Failed to parse config file for OAuth 2.0 token exchange: " << ex.what()); + } +} + +TOauth2TokenExchangeParams ReadOauth2ConfigFile(const std::string& configFilePath, const std::string& tokenEndpoint) { + return ReadOauth2ConfigJson(TFileInput(TString{configFilePath}).ReadAll(), tokenEndpoint); +} + +} // namespace + +std::vector GetSupportedOauth2TokenExchangeJwtAlgorithms() { + std::vector result; + result.reserve(JwtAlgorithmsFactory.size()); + for (const auto& [alg, _] : JwtAlgorithmsFactory) { + result.push_back(alg); + } + return result; +} + +std::shared_ptr CreateOauth2TokenExchangeFileCredentialsProviderFactory(const std::string& configFilePath, const std::string& tokenEndpoint) { + return CreateOauth2TokenExchangeCredentialsProviderFactory(ReadOauth2ConfigFile(configFilePath, tokenEndpoint)); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp new file mode 100644 index 000000000000..09bca9a42d97 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp @@ -0,0 +1,86 @@ +#include + +#include + +#include + +#define INV_ARG "Invalid argument for JWT token source: " + +namespace NYdb::inline V3 { + +#ifdef YDB_SDK_USE_NEW_JWT + using TJwtCppStorage = std::vector; +#else + using TJwtCppStorage = std::set; +#endif + +static const std::string TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"; + +class TJwtTokenSource: public ITokenSource { +public: + TJwtTokenSource(const TJwtTokenSourceParams& params) + : Params(params) + { + if (!Params.SigningAlgorithm_) { + throw std::invalid_argument(INV_ARG "no signing algorithm"); + } + + if (!Params.TokenTtl_) { + throw std::invalid_argument(INV_ARG "token TTL must be positive"); + } + + for (const auto& aud : Params.Audience_) { + if (aud.empty()) { + throw std::invalid_argument(INV_ARG "empty audience"); + } + } + } + + TToken GetToken() const override { + TToken t; + t.TokenType = TOKEN_TYPE; + + auto tokenBuilder = jwt::create(); + tokenBuilder.set_type("JWT"); + + const auto now = std::chrono::system_clock::now(); + const auto expire = now + std::chrono::microseconds(Params.TokenTtl_.MicroSeconds()); + tokenBuilder.set_issued_at(now); + tokenBuilder.set_expires_at(expire); + + if (!Params.KeyId_.empty()) { + tokenBuilder.set_key_id(Params.KeyId_); + } + + if (!Params.Issuer_.empty()) { + tokenBuilder.set_issuer(Params.Issuer_); + } + + if (!Params.Subject_.empty()) { + tokenBuilder.set_subject(Params.Subject_); + } + + if (!Params.Id_.empty()) { + tokenBuilder.set_id(Params.Id_); + } + + if (Params.Audience_.size() == 1) { + tokenBuilder.set_audience(Params.Audience_[0]); + } else if (Params.Audience_.size() > 1) { + TJwtCppStorage aud(Params.Audience_.begin(), Params.Audience_.end()); + tokenBuilder.set_audience(aud); + } + + t.Token = tokenBuilder.sign(*Params.SigningAlgorithm_); + return t; + } + +private: + TJwtTokenSourceParams Params; +}; + +std::shared_ptr CreateJwtTokenSource(const TJwtTokenSourceParams& params) { + return std::make_shared(params); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make new file mode 100644 index 000000000000..41b6427cb6be --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + credentials.cpp + from_file.cpp + jwt_token_source.cpp +) + +PEERDIR( + contrib/libs/jwt-cpp + library/cpp/cgiparam + library/cpp/http/misc + library/cpp/http/simple + library/cpp/json + library/cpp/retry + library/cpp/string_utils/base64 + library/cpp/uri + ydb/public/sdk/cpp/src/client/types + ydb/public/sdk/cpp/src/client/types/credentials +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/ya.make b/ydb/public/sdk/cpp/src/client/types/credentials/ya.make new file mode 100644 index 000000000000..4503014b4021 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/credentials/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + credentials.cpp +) + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/types/status + ydb/public/sdk/cpp/src/library/issue +) + +END() + +RECURSE( + oauth2_token_exchange +) diff --git a/ydb/public/sdk/cpp/src/client/types/exceptions/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/exceptions/CMakeLists.txt new file mode 100644 index 000000000000..365cc2195920 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/exceptions/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(client-ydb_types-exceptions) + +target_link_libraries(client-ydb_types-exceptions PUBLIC + yutil +) + +target_sources(client-ydb_types-exceptions PRIVATE + exceptions.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-exceptions) diff --git a/ydb/public/sdk/cpp/src/client/types/exceptions/exceptions.cpp b/ydb/public/sdk/cpp/src/client/types/exceptions/exceptions.cpp new file mode 100644 index 000000000000..f425e342cf02 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/exceptions/exceptions.cpp @@ -0,0 +1,12 @@ +#include + +namespace NYdb::inline V3 { + +TYdbException::TYdbException(const std::string& reason) { + Append(reason); +} + +TContractViolation::TContractViolation(const std::string& reason) + : TYdbException(reason) {} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/exceptions/ya.make b/ydb/public/sdk/cpp/src/client/types/exceptions/ya.make new file mode 100644 index 000000000000..06816457a5e3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/exceptions/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + exceptions.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/CMakeLists.txt new file mode 100644 index 000000000000..bf19c1526d14 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(client-ydb_types-fatal_error_handlers) + +target_link_libraries(client-ydb_types-fatal_error_handlers PUBLIC + yutil + client-ydb_types-exceptions +) + +target_sources(client-ydb_types-fatal_error_handlers PRIVATE + handlers.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-fatal_error_handlers) diff --git a/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/handlers.cpp b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/handlers.cpp new file mode 100644 index 000000000000..66fdb71f0d44 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/handlers.cpp @@ -0,0 +1,10 @@ +#include +#include + +namespace NYdb::inline V3 { + +void ThrowFatalError(const std::string& str) { + throw TContractViolation(str); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/ya.make b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/ya.make new file mode 100644 index 000000000000..0ad854b47c25 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/fatal_error_handlers/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + handlers.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/types/exceptions +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/operation/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/operation/CMakeLists.txt new file mode 100644 index 000000000000..cb20314eb8bc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/operation/CMakeLists.txt @@ -0,0 +1,16 @@ +_ydb_sdk_add_library(client-ydb_types-operation) + +target_link_libraries(client-ydb_types-operation PUBLIC + yutil + protobuf::libprotobuf + threading-future + library-operation_id + client-ydb_types +) + +target_sources(client-ydb_types-operation PRIVATE + operation.cpp + out.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-operation) diff --git a/ydb/public/sdk/cpp/src/client/types/operation/operation.cpp b/ydb/public/sdk/cpp/src/client/types/operation/operation.cpp new file mode 100644 index 000000000000..4d6acd766242 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/operation/operation.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include + +#include + +#include + +namespace NYdb::inline V3 { + + +class TOperation::TImpl { +public: + TImpl(TStatus&& status) + : Status_(std::move(status)) + , Ready_(true) + { } + + TImpl(TStatus&& status, Ydb::Operations::Operation&& operation) + : Id_(operation.id(), true /* allowEmpty */) + , Status_(std::move(status)) + , Ready_(operation.ready()) + , CreateTime_(ProtoTimestampToInstant(operation.create_time())) + , EndTime_(ProtoTimestampToInstant(operation.end_time())) + , Operation_(std::move(operation)) + { + } + + const TOperationId& Id() const { + return Id_; + } + + bool Ready() const { + return Ready_; + } + + const TStatus& Status() const { + return Status_; + } + + TInstant CreateTime() const { + return CreateTime_; + } + + TInstant EndTime() const { + return EndTime_; + } + + const std::string& CreatedBy() const { + return CreatedBy_; + } + + const Ydb::Operations::Operation& GetProto() const { + return Operation_; + } + +private: + const TOperationId Id_; + const TStatus Status_; + const bool Ready_; + const TInstant CreateTime_; + const TInstant EndTime_; + const std::string CreatedBy_; + const Ydb::Operations::Operation Operation_; +}; + +TOperation::TOperation(TStatus&& status) + : Impl_(std::make_shared(std::move(status))) +{ } + +TOperation::TOperation(TStatus&& status, Ydb::Operations::Operation&& operation) + : Impl_(std::make_shared(std::move(status), std::move(operation))) +{ } + +const TOperation::TOperationId& TOperation::Id() const { + return Impl_->Id(); +} + +bool TOperation::Ready() const { + return Impl_->Ready(); +} + +const TStatus& TOperation::Status() const { + return Impl_->Status(); +} + +TInstant TOperation::CreateTime() const { + return Impl_->CreateTime(); +} + +TInstant TOperation::EndTime() const { + return Impl_->EndTime(); +} + +const std::string& TOperation::CreatedBy() const { + return Impl_->CreatedBy(); +} + +std::string TOperation::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TOperation::Out(IOutputStream& o) const { + o << GetProto().DebugString(); +} + +std::string TOperation::ToJsonString() const { + using namespace google::protobuf::util; + + TStringType json; + auto status = MessageToJsonString(GetProto(), &json, JsonPrintOptions()); + Y_ABORT_UNLESS(status.ok()); + return json; +} + +const Ydb::Operations::Operation& TOperation::GetProto() const { + return Impl_->GetProto(); +} + +TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp) { + ui64 us = timestamp.seconds() * 1000000; + us += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(us); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/operation/out.cpp b/ydb/public/sdk/cpp/src/client/types/operation/out.cpp new file mode 100644 index 000000000000..e3ff8cbce2b5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/operation/out.cpp @@ -0,0 +1,5 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::TOperation, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/types/operation/ya.make b/ydb/public/sdk/cpp/src/client/types/operation/ya.make new file mode 100644 index 000000000000..a330e40296e1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/operation/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + operation.cpp + out.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/threading/future + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/types +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/status/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/types/status/CMakeLists.txt new file mode 100644 index 000000000000..280206f8ce22 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/status/CMakeLists.txt @@ -0,0 +1,16 @@ +_ydb_sdk_add_library(client-ydb_types-status) + +target_link_libraries(client-ydb_types-status PUBLIC + yutil + threading-future + impl-ydb_internal-plain_status + client-ydb_types + client-ydb_types-fatal_error_handlers + yql-public-issue +) + +target_sources(client-ydb_types-status PRIVATE + status.cpp +) + +_ydb_sdk_install_targets(TARGETS client-ydb_types-status) diff --git a/ydb/public/sdk/cpp/src/client/types/status/status.cpp b/ydb/public/sdk/cpp/src/client/types/status/status.cpp new file mode 100644 index 000000000000..1764499e40dc --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/status/status.cpp @@ -0,0 +1,124 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include + +#include + +namespace NYdb::inline V3 { + +class TStatus::TImpl { +public: + const TPlainStatus Status; + + TImpl(TPlainStatus&& status) + : Status(std::move(status)) + { } + + void CheckStatusOk(const std::string& str) const { + if (!Status.Ok()) { + ThrowFatalError(std::string("Attempt to use result with not successfull status. ") + str + "\n"); + } + } + + void RaiseError(const std::string& str) const { + ythrow TContractViolation(str); + } +}; + +TStatus::TStatus(EStatus statusCode, NYdb::NIssue::TIssues&& issues) + : Impl_(std::make_shared(TPlainStatus{statusCode, std::move(issues)})) +{ } + +TStatus::TStatus(TPlainStatus&& plain) + : Impl_(std::make_shared(std::move(plain))) +{ } + +const NYdb::NIssue::TIssues& TStatus::GetIssues() const { + return Impl_->Status.Issues; +} + +EStatus TStatus::GetStatus() const { + return Impl_->Status.Status; +} + +bool TStatus::IsSuccess() const { + return Impl_->Status.Status == EStatus::SUCCESS; +} + +bool TStatus::IsTransportError() const { + return static_cast(Impl_->Status.Status) >= TRANSPORT_STATUSES_FIRST + && static_cast(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST; +} + +void TStatus::CheckStatusOk(const std::string& str) const { + Impl_->CheckStatusOk(str); +} + +void TStatus::RaiseError(const std::string& str) const { + Impl_->RaiseError(str); +} + +const std::string& TStatus::GetEndpoint() const { + return Impl_->Status.Endpoint; +} + +const std::multimap& TStatus::GetResponseMetadata() const { + return Impl_->Status.Metadata; +} + +float TStatus::GetConsumedRu() const { + return Impl_->Status.ConstInfo.consumed_units(); +} + +void TStatus::Out(IOutputStream& out) const { + out << "{ status: " << GetStatus() + << ", issues: " << GetIssues().ToOneLineString() + << " }"; +} + +IOutputStream& operator<<(IOutputStream& out, const TStatus& st) { + out << "Status: " << ToString(st.GetStatus()) << Endl; + if (st.GetIssues()) { + out << "Issues: " << Endl; + st.GetIssues().PrintTo(out); + } + return out; +} + +//////////////////////////////////////////////////////////////////////////////// + +TStreamPartStatus::TStreamPartStatus(TStatus&& status) + : TStatus(std::move(status)) +{} + +bool TStreamPartStatus::EOS() const { + return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE; +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace NStatusHelpers { + +void ThrowOnError(TStatus status, std::function onSuccess) { + if (!status.IsSuccess()) { + throw TYdbErrorException(status) << status; + } else { + onSuccess(status); + } +} + +void ThrowOnErrorOrPrintIssues(TStatus status) { + ThrowOnError(status, [](TStatus status) { + if (status.GetIssues()) { + std::cerr << ToString(status); + } + }); +} + +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/status/ya.make b/ydb/public/sdk/cpp/src/client/types/status/ya.make new file mode 100644 index 000000000000..59f1ff53be6e --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/status/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + status.cpp +) + +PEERDIR( + library/cpp/threading/future + ydb/public/sdk/cpp/src/client/impl/ydb_internal/plain_status + ydb/public/sdk/cpp/src/client/types + ydb/public/sdk/cpp/src/client/types/fatal_error_handlers + ydb/public/sdk/cpp/src/library/issue +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/types/ya.make b/ydb/public/sdk/cpp/src/client/types/ya.make new file mode 100644 index 000000000000..536b97fd80fb --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +PEERDIR( + contrib/libs/protobuf + ydb/public/sdk/cpp/src/library/grpc/client + ydb/public/sdk/cpp/src/library/issue +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/s3_settings.h) +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h) + +END() diff --git a/ydb/public/sdk/cpp/src/client/value/CMakeLists.txt b/ydb/public/sdk/cpp/src/client/value/CMakeLists.txt new file mode 100644 index 000000000000..38b4b6032587 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/value/CMakeLists.txt @@ -0,0 +1,26 @@ +_ydb_sdk_add_library(client-ydb_value) + +target_link_libraries(client-ydb_value PUBLIC + yutil + enum_serialization_runtime + containers-stack_vector + api-protos + impl-ydb_internal-value_helpers + client-ydb_types-fatal_error_handlers + yql-public-decimal + library-uuid + proto_output +) + +target_sources(client-ydb_value PRIVATE + value.cpp + out.cpp +) + +generate_enum_serilization(client-ydb_value + ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/value/value.h + INCLUDE_HEADERS + include/ydb-cpp-sdk/client/value/value.h +) + +_ydb_sdk_make_client_component(Value client-ydb_value) diff --git a/ydb/public/sdk/cpp/src/client/value/out.cpp b/ydb/public/sdk/cpp/src/client/value/out.cpp new file mode 100644 index 000000000000..7c7f235d0741 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/value/out.cpp @@ -0,0 +1,5 @@ +#include + +Y_DECLARE_OUT_SPEC(, NYdb::TType, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/src/client/value/value.cpp b/ydb/public/sdk/cpp/src/client/value/value.cpp new file mode 100644 index 000000000000..1627f46311c2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/value/value.cpp @@ -0,0 +1,3372 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include +#include + +namespace NYdb::inline V3 { + +static void CheckKind(TTypeParser::ETypeKind actual, TTypeParser::ETypeKind expected, const std::string& method) +{ + if (expected != actual) { + ThrowFatalError(TStringBuilder() << method << "(): invalid state, expected type: " + << expected << ", actual: " << actual); + } +} + +static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) { + using ETypeKind = TTypeParser::ETypeKind; + + switch (type.type_case()) { + case Ydb::Type::kTypeId: + return ETypeKind::Primitive; + case Ydb::Type::kDecimalType: + return ETypeKind::Decimal; + case Ydb::Type::kPgType: + return ETypeKind::Pg; + case Ydb::Type::kOptionalType: + return ETypeKind::Optional; + case Ydb::Type::kListType: + return ETypeKind::List; + case Ydb::Type::kTupleType: + return ETypeKind::Tuple; + case Ydb::Type::kStructType: + return ETypeKind::Struct; + case Ydb::Type::kDictType: + return ETypeKind::Dict; + case Ydb::Type::kVariantType: + return ETypeKind::Variant; + case Ydb::Type::kVoidType: + return ETypeKind::Void; + case Ydb::Type::kNullType: + return ETypeKind::Null; + case Ydb::Type::kEmptyListType: + return ETypeKind::EmptyList; + case Ydb::Type::kEmptyDictType: + return ETypeKind::EmptyDict; + case Ydb::Type::kTaggedType: + return ETypeKind::Tagged; + default: + break; + } + + ThrowFatalError(TStringBuilder() << "Unexpected proto type kind: " << (ui32) type.type_case()); + return ETypeKind::Void; +} + +bool TypesEqual(const TType& t1, const TType& t2) { + return TypesEqual(t1.GetProto(), t2.GetProto()); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TType::TImpl { +public: + TImpl(const Ydb::Type& typeProto) + : ProtoType_(typeProto) {} + + TImpl(Ydb::Type&& typeProto) + : ProtoType_(std::move(typeProto)) {} + + Ydb::Type ProtoType_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TType::TType(const Ydb::Type& typeProto) + : Impl_(new TImpl(typeProto)) {} + +TType::TType(Ydb::Type&& typeProto) + : Impl_(new TImpl(std::move(typeProto))) {} + +std::string TType::ToString() const { + return FormatType(*this); +} + +void TType::Out(IOutputStream& o) const { + o << FormatType(*this); +} + +const Ydb::Type& TType::GetProto() const { + return Impl_->ProtoType_; +} + +Ydb::Type& TType::GetProto() +{ + return Impl_->ProtoType_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TTypeParser::TImpl { +public: + TImpl(const TType& type) + : Type_(type) + { + Reset(); + } + + void Reset() { + Path_.clear(); + Path_.emplace_back(TProtoPosition{&Type_.GetProto(), -1}); + } + + ETypeKind GetKind(ui32 offset = 0) const { + return NYdb::GetKind(GetProto(offset)); + } + + EPrimitiveType GetPrimitive() const { + CheckKind(ETypeKind::Primitive, "GetPrimitive"); + return EPrimitiveType(GetProto().type_id()); + } + + TDecimalType GetDecimal() const { + CheckKind(ETypeKind::Decimal, "GetDecimal"); + return TDecimalType( + GetProto().decimal_type().precision(), + GetProto().decimal_type().scale()); + } + + TPgType GetPg() const { + CheckKind(ETypeKind::Pg, "GetPg"); + const auto& pg = GetProto().pg_type(); + TPgType type(pg.type_name(), pg.type_modifier()); + type.Oid = pg.oid(); + type.Typlen = pg.typlen(); + type.Typmod = pg.typmod(); + return type; + } + + template + void Open() { + CheckKind(kind, "Open"); + ForwardStep(); + } + + void OpenVariant(int index) { + CheckKind(ETypeKind::Variant, "Open"); + const Ydb::VariantType& variantType = GetProto().variant_type(); + const google::protobuf::Message* nextPtr = nullptr; + switch (variantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + auto& tupleType = variantType.tuple_items(); + if (index >= tupleType.elements_size()) { + return FatalError("variant index is out of range"); + } + nextPtr = &tupleType.elements(index); + break; + } + case Ydb::VariantType::kStructItems: { + auto& structType = variantType.struct_items(); + if (index >= structType.members_size()) { + return FatalError("variant index is out of range"); + } + nextPtr = &structType.members(index).type(); + break; + } + default: { + return FatalError("unknown variant type case"); + } + } + Path_.emplace_back(TProtoPosition{nextPtr, -1}); + } + + template + void Close() { + CheckPreviousKind(kind, "Close"); + BackwardStep(); + Path_.back().Idx = -1; + } + + const std::string& GetMemberName() { + CheckPreviousKind(ETypeKind::Struct, "GetMemberName"); + return GetProto(1).struct_type().members(Path_[Path_.size() - 2].Idx).name(); + } + + template + bool TryNext() { + CheckPreviousKind(kind, "TryNext"); + + BackwardStep(); + Path_.back().Idx++; + return ForwardStep(); + } + + void DictKey() { + CheckPreviousKind(ETypeKind::Dict, "DictKey"); + BackwardStep(); + Path_.back().Idx = 0; + ForwardStep(); + } + + void DictPayload() { + CheckPreviousKind(ETypeKind::Dict, "DictPayload"); + BackwardStep(); + Path_.back().Idx = 1; + ForwardStep(); + } + + const std::string& GetTag() { + CheckPreviousKind(ETypeKind::Tagged, "GetTag"); + return GetProto(1).tagged_type().tag(); + } + + bool ForwardStep() { + auto& idx = Path_.back().Idx; + const google::protobuf::Message* nextPtr = nullptr; + bool hasIdx = true; + + switch (GetKind()) { + case ETypeKind::Optional: + nextPtr = &GetProto().optional_type().item(); + break; + + case ETypeKind::List: + nextPtr = &GetProto().list_type().item(); + break; + + case ETypeKind::Struct: { + if (idx >= 0) { + auto& structType = GetProto().struct_type(); + if (idx >= structType.members_size()) { + idx = structType.members_size() - 1; + hasIdx = false; + } + if (idx >= 0) { + nextPtr = &structType.members(idx).type(); + } + } else { + nextPtr = &GetProto(); + } + break; + } + + case ETypeKind::Tuple: { + if (idx >= 0) { + auto& tupleType = GetProto().tuple_type(); + if (idx >= tupleType.elements_size()) { + idx = tupleType.elements_size() - 1; + hasIdx = false; + } + if (idx >= 0) { + nextPtr = &tupleType.elements(idx); + } + } else { + nextPtr = &GetProto(); + } + break; + } + + case ETypeKind::Dict: { + if (idx == 0) { + nextPtr = &GetProto().dict_type().key(); + } else if (idx == 1) { + nextPtr = &GetProto().dict_type().payload(); + } else { + nextPtr = &GetProto(); + } + break; + } + + case ETypeKind::Variant: { + const Ydb::VariantType& variantType = GetProto().variant_type(); + auto wrappedVariant = std::make_unique(); + switch (variantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + *wrappedVariant->mutable_tuple_type() = variantType.tuple_items(); + break; + } + case Ydb::VariantType::kStructItems: { + *wrappedVariant->mutable_struct_type() = variantType.struct_items(); + break; + } + default: { + FatalError(TStringBuilder() << "Unexpected variant type kind: " << variantType); + break; + } + } + WrappedVariants_.emplace_back(std::move(wrappedVariant)); + nextPtr = WrappedVariants_.back().get(); + break; + } + + case ETypeKind::Tagged: { + nextPtr = &GetProto().tagged_type().type(); + break; + } + + default: + FatalError(TStringBuilder() << "Unexpected type kind: " << GetKind()); + break; + } + + Path_.emplace_back(TProtoPosition{nextPtr, -1}); + return hasIdx; + } + +private: + void CheckKind(ETypeKind kind, const std::string& method) const { + NYdb::CheckKind(GetKind(), kind, method); + } + + void CheckPreviousKind(ETypeKind kind, const std::string method) const { + if (Path_.size() < 2) { + FatalError("Expected container type."); + return; + } + + NYdb::CheckKind(GetKind(1), kind, method); + } + + const Ydb::Type& GetProto(ui32 offset = 0) const { + return *static_cast(Path_[Path_.size() - (offset + 1)].Ptr); + } + + void BackwardStep() { + Path_.pop_back(); + } + + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TTypeParser: " << msg); + } + +private: + struct TProtoPosition { + const google::protobuf::Message* Ptr; + i32 Idx; + }; + +private: + TType Type_; + TStackVec Path_; + TStackVec, 8> WrappedVariants_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TTypeParser::TTypeParser(TTypeParser&&) = default; +TTypeParser::~TTypeParser() = default; + +TTypeParser::TTypeParser(const TType& type) + : Impl_(new TImpl(type)) {} + +TTypeParser::ETypeKind TTypeParser::GetKind() const { + return Impl_->GetKind(); +} + +EPrimitiveType TTypeParser::GetPrimitive() const { + return Impl_->GetPrimitive(); +} + +TDecimalType TTypeParser::GetDecimal() const { + return Impl_->GetDecimal(); +} + +TPgType TTypeParser::GetPg() const { + return Impl_->GetPg(); +} + +void TTypeParser::OpenOptional() { + Impl_->Open(); +} + +void TTypeParser::CloseOptional() { + Impl_->Close(); +} + +void TTypeParser::OpenList() { + Impl_->Open(); +} + +void TTypeParser::CloseList() { + Impl_->Close(); +} + +void TTypeParser::OpenStruct() { + Impl_->Open(); +} + +void TTypeParser::CloseStruct() { + Impl_->Close(); +} + +const std::string& TTypeParser::GetMemberName() { + return Impl_->GetMemberName(); +} + +bool TTypeParser::TryNextMember() { + return Impl_->TryNext(); +} + +void TTypeParser::OpenTuple() { + Impl_->Open(); +} + +void TTypeParser::CloseTuple() { + Impl_->Close(); +} + +bool TTypeParser::TryNextElement() { + return Impl_->TryNext(); +} + +void TTypeParser::OpenDict() { + Impl_->Open(); +} + +void TTypeParser::CloseDict() { + Impl_->Close(); +} + +void TTypeParser::DictKey() { + Impl_->DictKey(); +} + +void TTypeParser::DictPayload() { + Impl_->DictPayload(); +} + +void TTypeParser::OpenVariant(size_t index) { + Impl_->OpenVariant(index); +} + +void TTypeParser::OpenVariant() { + Impl_->Open(); +} + +void TTypeParser::CloseVariant() { + Impl_->Close(); +} + +void TTypeParser::OpenTagged() { + Impl_->Open(); +} + +const std::string& TTypeParser::GetTag() { + return Impl_->GetTag(); +} + +void TTypeParser::CloseTagged() { + Impl_->Close(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { + using namespace std::literals; + switch (parser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + out << parser.GetPrimitive(); + break; + + case TTypeParser::ETypeKind::Decimal: { + auto decimal = parser.GetDecimal(); + out << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")"; + break; + } + + case TTypeParser::ETypeKind::Pg: { + auto pg = parser.GetPg(); + out << "Pg('"sv << pg.TypeName << "\',\'" << pg.TypeModifier << "\'," + << pg.Oid << "," << pg.Typlen << ',' << pg.Typmod << ')'; + break; + } + + case TTypeParser::ETypeKind::Optional: + parser.OpenOptional(); + FormatTypeInternal(parser, out); + parser.CloseOptional(); + out << '?'; + break; + + case TTypeParser::ETypeKind::Tagged: + parser.OpenTagged(); + out << "Tagged<"; + FormatTypeInternal(parser, out); + out << ",\'" << parser.GetTag() << "\'>"; + parser.CloseTagged(); + break; + + case TTypeParser::ETypeKind::List: + out << "List<"; + parser.OpenList(); + FormatTypeInternal(parser, out); + parser.CloseList(); + out << '>'; + break; + + case TTypeParser::ETypeKind::Tuple: { + out << "Tuple<"; + parser.OpenTuple(); + bool needsComma = false; + while (parser.TryNextElement()) { + if (needsComma) { + out << ','; + } + FormatTypeInternal(parser, out); + needsComma = true; + } + parser.CloseTuple(); + out << '>'; + break; + } + + case TTypeParser::ETypeKind::Struct: { + out << "Struct<"; + parser.OpenStruct(); + bool needsComma = false; + while (parser.TryNextMember()) { + if (needsComma) { + out << ','; + } + out << '\'' << parser.GetMemberName() << "\':"; + FormatTypeInternal(parser, out); + needsComma = true; + } + parser.CloseStruct(); + out << '>'; + break; + } + + case TTypeParser::ETypeKind::Dict: + out << "Dict<"; + parser.OpenDict(); + parser.DictKey(); + FormatTypeInternal(parser, out); + out << ','; + parser.DictPayload(); + FormatTypeInternal(parser, out); + parser.CloseDict(); + out << '>'; + break; + + case TTypeParser::ETypeKind::Variant: + // TODO: Variant + break; + + case TTypeParser::ETypeKind::Void: + out << "Void"sv; + break; + + case TTypeParser::ETypeKind::Null: + out << "Null"sv; + break; + + default: + ThrowFatalError(TStringBuilder() + << "Unexpected type kind: " << parser.GetKind()); + } +} + +std::string FormatType(const TType& type) { + TTypeParser parser(type); + TStringStream out; + FormatTypeInternal(parser, out); + return out.Str(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TTypeBuilder::TImpl { + using ETypeKind = TTypeParser::ETypeKind; + +public: + TImpl() + { + Path_.emplace_back(TProtoPosition{&ProtoType_}); + } + + TImpl(Ydb::Type& type) + { + Path_.emplace_back(TProtoPosition{&type}); + } + + TType Build() { + if (Path_.size() > 1) { + FatalError("Invalid Build() call, type is incomplete."); + } + + Ydb::Type type; + type.Swap(&ProtoType_); + return TType(std::move(type)); + } + + void Primitive(const EPrimitiveType& primitiveType) { + GetProto().set_type_id(Ydb::Type::PrimitiveTypeId(primitiveType)); + } + + void Decimal(const TDecimalType& decimalType) { + auto& decimal = *GetProto().mutable_decimal_type(); + decimal.set_precision(decimalType.Precision); + decimal.set_scale(decimalType.Scale); + } + + void Pg(const TPgType& pgType) { + auto& pg = *GetProto().mutable_pg_type(); + pg.set_type_name(TStringType{pgType.TypeName}); + pg.set_type_modifier(TStringType{pgType.TypeModifier}); + } + + void BeginOptional() { + AddPosition(GetProto().mutable_optional_type()->mutable_item()); + } + + void EndOptional() { + CloseContainer(); + } + + void Optional(const TType& itemType) { + GetProto().mutable_optional_type()->mutable_item()->CopyFrom(itemType.GetProto()); + } + + void BeginList() { + AddPosition(GetProto().mutable_list_type()->mutable_item()); + } + + void EndList() { + CloseContainer(); + } + + void List(const TType& itemType) { + GetProto().mutable_list_type()->mutable_item()->CopyFrom(itemType.GetProto()); + } + + void BeginStruct() { + GetProto().mutable_struct_type(); + AddPosition(&GetProto()); + } + + void EndStruct() { + CloseContainer(); + } + + void AddMember(const std::string& memberName) { + CheckPreviousKind(ETypeKind::Struct, "AddMember"); + PopPosition(); + auto member = GetProto().mutable_struct_type()->add_members(); + member->set_name(TStringType{memberName}); + AddPosition(member->mutable_type()); + } + + void AddMember(const std::string& memberName, const TType& memberType) { + AddMember(memberName); + GetProto().CopyFrom(memberType.GetProto()); + } + + void SelectMember(size_t index) { + CheckPreviousKind(ETypeKind::Struct, "SelectMember"); + PopPosition(); + auto member = GetProto().mutable_struct_type()->mutable_members(index); + AddPosition(member->mutable_type()); + } + + void BeginTuple() { + GetProto().mutable_tuple_type(); + AddPosition(&GetProto()); + } + + void EndTuple() { + CloseContainer(); + } + + void AddElement() { + CheckPreviousKind(ETypeKind::Tuple, "AddElement"); + PopPosition(); + AddPosition(GetProto().mutable_tuple_type()->add_elements()); + } + + void AddElement(const TType& elementType) { + AddElement(); + GetProto().CopyFrom(elementType.GetProto()); + } + + void SelectElement(size_t index) { + CheckPreviousKind(ETypeKind::Tuple, "SelectElement"); + PopPosition(); + AddPosition(GetProto().mutable_tuple_type()->mutable_elements(index)); + } + + void BeginDict() { + GetProto().mutable_dict_type(); + AddPosition(&GetProto()); + } + + void EndDict() { + CloseContainer(); + } + + void DictKey() { + CheckPreviousKind(ETypeKind::Dict, "DictKey"); + PopPosition(); + AddPosition(GetProto().mutable_dict_type()->mutable_key()); + } + + void DictKey(const TType& keyType) { + DictKey(); + GetProto().CopyFrom(keyType.GetProto()); + } + + void DictPayload() { + CheckPreviousKind(ETypeKind::Dict, "DictPayload"); + PopPosition(); + AddPosition(GetProto().mutable_dict_type()->mutable_payload()); + } + + void DictPayload(const TType& payloadType) { + DictPayload(); + GetProto().CopyFrom(payloadType.GetProto()); + } + + void BeginTagged(const std::string& tag) { + GetProto().mutable_tagged_type()->set_tag(TStringType{tag}); + AddPosition(GetProto().mutable_tagged_type()->mutable_type()); + } + + void EndTagged() { + CloseContainer(); + } + + void Tagged(const std::string& tag, const TType& itemType) { + auto taggedType = GetProto().mutable_tagged_type(); + taggedType->set_tag(TStringType{tag}); + taggedType->mutable_type()->CopyFrom(itemType.GetProto()); + } + + Ydb::Type& GetProto(ui32 offset = 0) { + return *static_cast(Path_[Path_.size() - (offset + 1)].Ptr); + } + + void SetType(const TType& type) { + GetProto().CopyFrom(type.GetProto()); + } + + void SetType(TType&& type) { + GetProto() = std::move(type.GetProto()); + } + +private: + void AddPosition(Ydb::Type* type) { + Path_.emplace_back(TProtoPosition{type}); + } + + void PopPosition() { + Path_.pop_back(); + } + + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TTypeBuilder: " << msg); + } + + void CheckKind(ETypeKind kind, const std::string& method) { + NYdb::CheckKind(GetKind(), kind, method); + } + + void CheckPreviousKind(ETypeKind kind, const std::string& method) { + NYdb::CheckKind(GetKind(1), kind, method); + } + + ETypeKind GetKind(ui32 offset = 0) { + return NYdb::GetKind(GetProto(offset)); + } + + template + void CloseContainer() { + auto sz = Path_.size(); + if (sz < 2) { + FatalError("No opened container to close"); + return; + } + CheckPreviousKind(kind, "End"); + PopPosition(); + } + +private: + struct TProtoPosition { + google::protobuf::Message* Ptr; + }; + +private: + Ydb::Type ProtoType_; + TStackVec Path_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TTypeBuilder::TTypeBuilder(TTypeBuilder&&) = default; +TTypeBuilder::~TTypeBuilder() = default; + +TTypeBuilder::TTypeBuilder() + : Impl_(new TImpl()) {} + +TType TTypeBuilder::Build() { + return Impl_->Build(); +} + +TTypeBuilder& TTypeBuilder::Primitive(const EPrimitiveType& primitiveType) { + Impl_->Primitive(primitiveType); + return *this; +} + +TTypeBuilder& TTypeBuilder::Decimal(const TDecimalType& decimalType) { + Impl_->Decimal(decimalType); + return *this; +} + +TTypeBuilder& TTypeBuilder::Pg(const TPgType& pgType) { + Impl_->Pg(pgType); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginOptional() { + Impl_->BeginOptional(); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndOptional() { + Impl_->EndOptional(); + return *this; +} + +TTypeBuilder& TTypeBuilder::Optional(const TType& itemType) { + Impl_->Optional(itemType); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginList() { + Impl_->BeginList(); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndList() { + Impl_->EndList(); + return *this; +} + +TTypeBuilder& TTypeBuilder::List(const TType& itemType) { + Impl_->List(itemType); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginStruct() { + Impl_->BeginStruct(); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndStruct() { + Impl_->EndStruct(); + return *this; +} + +TTypeBuilder& TTypeBuilder::AddMember(const std::string& memberName) { + Impl_->AddMember(memberName); + return *this; +} + +TTypeBuilder& TTypeBuilder::AddMember(const std::string& memberName, const TType& memberType) { + Impl_->AddMember(memberName, memberType); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginTuple() { + Impl_->BeginTuple(); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndTuple() { + Impl_->EndTuple(); + return *this; +} + +TTypeBuilder& TTypeBuilder::AddElement() { + Impl_->AddElement(); + return *this; +} + +TTypeBuilder& TTypeBuilder::AddElement(const TType& elementType) { + Impl_->AddElement(elementType); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginDict() { + Impl_->BeginDict(); + return *this; +} + +TTypeBuilder& TTypeBuilder::DictKey() { + Impl_->DictKey(); + return *this; +} + +TTypeBuilder& TTypeBuilder::DictKey(const TType& keyType) { + Impl_->DictKey(keyType); + return *this; +} + +TTypeBuilder& TTypeBuilder::DictPayload() { + Impl_->DictPayload(); + return *this; +} + +TTypeBuilder& TTypeBuilder::DictPayload(const TType& payloadType) { + Impl_->DictPayload(payloadType); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndDict() { + Impl_->EndDict(); + return *this; +} + +TTypeBuilder& TTypeBuilder::BeginTagged(const std::string& tag) { + Impl_->BeginTagged(tag); + return *this; +} + +TTypeBuilder& TTypeBuilder::EndTagged() { + Impl_->EndTagged(); + return *this; +} + +TTypeBuilder& TTypeBuilder::Tagged(const std::string& tag, const TType& itemType) { + Impl_->Tagged(tag, itemType); + return *this; +} + +//////////////////////////////////////////////////////////////////////////////// + +TDecimalValue::TDecimalValue(const Ydb::Value& valueProto, const TDecimalType& decimalType) + : DecimalType_(decimalType) + , Low_(valueProto.low_128()) + , Hi_(valueProto.high_128()) +{} + +TDecimalValue::TDecimalValue(const std::string& decimalString, ui8 precision, ui8 scale) + : DecimalType_(precision, scale) +{ + NYdb::NDecimal::TInt128 val = NYdb::NDecimal::FromString(decimalString, precision, scale); + static_assert(sizeof(val) == 16, "wrong TInt128 size"); + char* buf = reinterpret_cast(&val); + Low_ = *(uint64_t*)buf; + Hi_ = *(int64_t*)(buf + 8); +} + +std::string TDecimalValue::ToString() const { + NYdb::NDecimal::TInt128 val = NYdb::NDecimal::FromHalfs(Low_, Hi_); + return NYdb::NDecimal::ToString(val, DecimalType_.Precision, DecimalType_.Scale); +} + +//////////////////////////////////////////////////////////////////////////////// + +TPgValue::TPgValue(const Ydb::Value& pgValueProto, const TPgType& pgType) + : PgType_(pgType) +{ + if (pgValueProto.value_case() == Ydb::Value::kTextValue) { + Kind_ = VK_TEXT; + Content_ = pgValueProto.text_value(); + return; + } + + if (pgValueProto.value_case() == Ydb::Value::kBytesValue) { + Kind_ = VK_BINARY; + Content_ = pgValueProto.bytes_value(); + return; + } + + Kind_ = VK_NULL; +} + +TPgValue::TPgValue(EPgValueKind kind, const std::string& content, const TPgType& pgType) + : PgType_(pgType) + , Kind_(kind) + , Content_(content) +{ +} + +bool TPgValue::IsNull() const { + return Kind_ == VK_NULL; +} + +bool TPgValue::IsText() const { + return Kind_ == VK_TEXT; +} + +//////////////////////////////////////////////////////////////////////////////// + +TUuidValue::TUuidValue(uint64_t low_128, uint64_t high_128) { + Buf_.Halfs[0] = low_128; + Buf_.Halfs[1] = high_128; +} + +TUuidValue::TUuidValue(const Ydb::Value& valueProto) { + Buf_.Halfs[0] = valueProto.low_128(); + Buf_.Halfs[1] = valueProto.high_128(); +} + +TUuidValue::TUuidValue(const std::string& uuidString) { + ui16 dw[8]; + if (!NUuid::ParseUuidToArray(uuidString, dw, false)) { + ThrowFatalError(TStringBuilder() << "Unable to parse string as uuid"); + } + static_assert(sizeof(dw) == sizeof(Buf_.Bytes)); + // TODO: check output on big-endian machines here and everywhere. + std::memcpy(Buf_.Bytes, dw, sizeof(dw)); +} + +std::string TUuidValue::ToString() const { + TStringStream s; + ui16 dw[8]; + static_assert(sizeof(dw) == sizeof(Buf_.Bytes)); + std::memcpy(dw, Buf_.Bytes, sizeof(dw)); + NUuid::UuidToString(dw, s); + return s.Str(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TValue::TImpl { +public: + TImpl(const TType& type, const Ydb::Value& valueProto) + : Type_(type) + , ProtoValue_(valueProto) {} + + TImpl(const TType& type, Ydb::Value&& valueProto) + : Type_(type) + , ProtoValue_(std::move(valueProto)) {} + + TType Type_; + Ydb::Value ProtoValue_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TValue::TValue(const TType& type, const Ydb::Value& valueProto) + : Impl_(new TImpl(type, valueProto)) {} + +TValue::TValue(const TType& type, Ydb::Value&& valueProto) + : Impl_(new TImpl(type, std::move(valueProto))) {} + +const TType& TValue::GetType() const { + return Impl_->Type_; +} + +TType & TValue::GetType() { + return Impl_->Type_; +} + +const Ydb::Value& TValue::GetProto() const { + return Impl_->ProtoValue_; +} + +Ydb::Value& TValue::GetProto() { + return Impl_->ProtoValue_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TValueParser::TImpl { + using ETypeKind = TTypeParser::ETypeKind; + + enum class EParseKind { + Value, + Null, + Items, + Pairs, + Pair + }; + + struct TProtoPosition { + EParseKind Kind; + const google::protobuf::Message* Ptr; + i32 Idx; + }; + +public: + TImpl(const TValue& value) + : Value_(value.Impl_) + , TypeParser_(value.GetType()) + { + Reset(Value_->ProtoValue_); + } + + TImpl(const TType& type) + : TypeParser_(type) {} + + void Reset(const Ydb::Value& value) { + TypeParser_.Impl_->Reset(); + Path_.clear(); + AddPath(EParseKind::Value, &value); + } + + ETypeKind GetKind() const { + return TypeParser_.GetKind(); + } + + EPrimitiveType GetPrimitiveType() const { + CheckKind(ETypeKind::Primitive, "GetPrimitiveType"); + return TypeParser_.GetPrimitive(); + } + + bool GetBool() const { + CheckPrimitive(NYdb::EPrimitiveType::Bool); + return GetProto().bool_value(); + } + + i8 GetInt8() const { + CheckPrimitive(NYdb::EPrimitiveType::Int8); + return GetProto().int32_value(); + } + + ui8 GetUint8() const { + CheckPrimitive(NYdb::EPrimitiveType::Uint8); + return GetProto().uint32_value(); + } + + i16 GetInt16() const { + CheckPrimitive(NYdb::EPrimitiveType::Int16); + return GetProto().int32_value(); + } + + ui16 GetUint16() const { + CheckPrimitive(NYdb::EPrimitiveType::Uint16); + return GetProto().uint32_value(); + } + + i32 GetInt32() const { + CheckPrimitive(NYdb::EPrimitiveType::Int32); + return GetProto().int32_value(); + } + + ui32 GetUint32() const { + CheckPrimitive(NYdb::EPrimitiveType::Uint32); + return GetProto().uint32_value(); + } + + int64_t GetInt64() const { + CheckPrimitive(NYdb::EPrimitiveType::Int64); + return GetProto().int64_value(); + } + + uint64_t GetUint64() const { + CheckPrimitive(NYdb::EPrimitiveType::Uint64); + return GetProto().uint64_value(); + } + + float GetFloat() const { + CheckPrimitive(NYdb::EPrimitiveType::Float); + return GetProto().float_value(); + } + + double GetDouble() const { + CheckPrimitive(NYdb::EPrimitiveType::Double); + return GetProto().double_value(); + } + + TInstant GetDate() const { + CheckPrimitive(NYdb::EPrimitiveType::Date); + return TInstant::Days(GetProto().uint32_value()); + } + + TInstant GetDatetime() const { + CheckPrimitive(NYdb::EPrimitiveType::Datetime); + return TInstant::Seconds(GetProto().uint32_value()); + } + + TInstant GetTimestamp() const { + CheckPrimitive(NYdb::EPrimitiveType::Timestamp); + return TInstant::MicroSeconds(GetProto().uint64_value()); + } + + int64_t GetInterval() const { + CheckPrimitive(NYdb::EPrimitiveType::Interval); + return GetProto().int64_value(); + } + + i32 GetDate32() const { + CheckPrimitive(NYdb::EPrimitiveType::Date32); + return GetProto().int32_value(); + } + + int64_t GetDatetime64() const { + CheckPrimitive(NYdb::EPrimitiveType::Datetime64); + return GetProto().int64_value(); + } + + int64_t GetTimestamp64() const { + CheckPrimitive(NYdb::EPrimitiveType::Timestamp64); + return GetProto().int64_value(); + } + + int64_t GetInterval64() const { + CheckPrimitive(NYdb::EPrimitiveType::Interval64); + return GetProto().int64_value(); + } + + const std::string& GetTzDate() const { + CheckPrimitive(NYdb::EPrimitiveType::TzDate); + return GetProto().text_value(); + } + + const std::string& GetTzDatetime() const { + CheckPrimitive(NYdb::EPrimitiveType::TzDatetime); + return GetProto().text_value(); + } + + const std::string& GetTzTimestamp() const { + CheckPrimitive(NYdb::EPrimitiveType::TzTimestamp); + return GetProto().text_value(); + } + + const std::string& GetString() const { + CheckPrimitive(NYdb::EPrimitiveType::String); + return GetProto().bytes_value(); + } + + const std::string& GetUtf8() const { + CheckPrimitive(NYdb::EPrimitiveType::Utf8); + return GetProto().text_value(); + } + + const std::string& GetYson() const { + CheckPrimitive(NYdb::EPrimitiveType::Yson); + return GetProto().bytes_value(); + } + + const std::string& GetJson() const { + CheckPrimitive(NYdb::EPrimitiveType::Json); + return GetProto().text_value(); + } + + TUuidValue GetUuid() const { + CheckPrimitive(NYdb::EPrimitiveType::Uuid); + return TUuidValue(GetProto()); + } + + const std::string& GetJsonDocument() const { + CheckPrimitive(NYdb::EPrimitiveType::JsonDocument); + return GetProto().text_value(); + } + + const std::string& GetDyNumber() const { + CheckPrimitive(NYdb::EPrimitiveType::DyNumber); + return GetProto().text_value(); + } + + TDecimalValue GetDecimal() const { + CheckDecimal(); + return TDecimalValue(GetProto(), TypeParser_.GetDecimal()); + } + + TPgValue GetPg() const { + CheckPg(); + return TPgValue(GetProto(), TypeParser_.GetPg()); + } + + void OpenOptional() { + TypeParser_.OpenOptional(); + + if (GetProto().value_case() == Ydb::Value::kNestedValue) { + AddPath(EParseKind::Value, &GetProto().nested_value()); + } else if (GetProto().value_case() == Ydb::Value::kNullFlagValue) { + AddPath(EParseKind::Null, &GetProto()); + } else { + AddPath(EParseKind::Value, &GetProto()); + } + } + + bool IsNull() const { + return GetPathBack().Kind == EParseKind::Null; + } + + void CloseOptional() { + PopPath(); + TypeParser_.CloseOptional(); + } + + void OpenList() { + TypeParser_.OpenList(); + OpenItems(); + } + + bool TryNextListItem() { + return NextItems(); + } + + void CloseList() { + CloseItems(); + TypeParser_.CloseList(); + } + + void OpenStruct() { + TypeParser_.OpenStruct(); + OpenItems(); + } + + bool TryNextMember() { + if (TypeParser_.TryNextMember()) { + if (NextItems()) { + return true; + } + + FatalError(TStringBuilder() << "Missing struct 'items' value at index: " << GetPathBack().Idx); + } + + return false; + } + + const std::string& GetMemberName() { + return TypeParser_.GetMemberName(); + } + + void CloseStruct() { + CloseItems(); + TypeParser_.CloseStruct(); + } + + void OpenTuple() { + TypeParser_.OpenTuple(); + OpenItems(); + } + + bool TryNextElement() { + if (TypeParser_.TryNextElement()) { + if (NextItems()) { + return true; + } + + FatalError(TStringBuilder() << "Missing tuple 'items' value at index: " << GetPathBack().Idx); + } + + return false; + } + + void CloseTuple() { + CloseItems(); + TypeParser_.CloseTuple(); + } + + void OpenDict() { + TypeParser_.OpenDict(); + OpenPairs(); + } + + bool TryNextDictItem() { + return NextPairs(); + } + + void DictKey() { + TypeParser_.DictKey(); + + if (GetPathBack().Kind != EParseKind::Pair) { + PopPath(); + } + + AddPath(EParseKind::Value, &GetProtoPair().key()); + } + + void DictPayload() { + TypeParser_.DictPayload(); + + if (GetPathBack().Kind != EParseKind::Pair) { + PopPath(); + } + + AddPath(EParseKind::Value, &GetProtoPair().payload()); + } + + void CloseDict() { + ClosePairs(); + TypeParser_.CloseDict(); + } + + void OpenVariant() { + auto variantIndex = GetProto().variant_index(); + TypeParser_.OpenVariant(variantIndex); + if (GetProto().value_case() == Ydb::Value::kNestedValue) { + AddPath(EParseKind::Value, &GetProto().nested_value()); + } else { + FatalError(TStringBuilder() << "No nested value for variant type."); + } + } + + void CloseVariant() { + PopPath(); + TypeParser_.CloseVariant(); + } + + void OpenTagged() { + TypeParser_.OpenTagged(); + } + + const std::string& GetTag() { + return TypeParser_.GetTag(); + } + + void CloseTagged() { + TypeParser_.CloseTagged(); + } + +private: + const TProtoPosition& GetPathBack() const { + if (Path_.empty()) { + FatalError(TStringBuilder() << "Bad parser state, no open value."); + } + + return Path_.back(); + } + + void PopPath() { + if (Path_.empty()) { + FatalError(TStringBuilder() << "Bad parser state, no open value."); + } + + Path_.pop_back(); + } + + void AddPath(EParseKind kind, const google::protobuf::Message* message, i32 idx = -1) { + if (!Path_.empty()) { + if (Path_.back().Kind == EParseKind::Null) { + FatalError(TStringBuilder() << "Can't parse inside NULL value"); + return; + } + + switch (Path_.back().Kind) { + case EParseKind::Value: + break; + + case EParseKind::Pair: + if (kind != EParseKind::Value) { + FatalError(TStringBuilder() << "Bad parser state, expected dict pair."); + return; + } + break; + + default: + FatalError(TStringBuilder() << "Bad parser state, no value to parse."); + return; + } + } + + Path_.emplace_back(TProtoPosition{kind, message, idx}); + } + + void OpenItems(i32 idx = -1) { + if (idx < 0) { + AddPath(EParseKind::Items, &GetProto(), idx); + } else { + AddPath(EParseKind::Value, &GetProto().items(idx), idx); + } + } + + void CloseItems() { + PopPath(); + } + + bool NextItems() { + auto idx = GetPathBack().Idx; + idx++; + + PopPath(); + + bool isEnd = (idx == static_cast(GetProto().items_size())); + if (isEnd) { + idx--; + } + + OpenItems(idx); + + return !isEnd; + } + + void OpenPairs(i32 idx = -1) { + if (idx < 0) { + AddPath(EParseKind::Pairs, &GetProto(), idx); + } else { + AddPath(EParseKind::Pair, &GetProto().pairs(idx), idx); + } + } + + bool NextPairs() { + if (GetPathBack().Kind != EParseKind::Pairs && GetPathBack().Kind != EParseKind::Pair) { + PopPath(); + } + + auto idx = GetPathBack().Idx; + idx++; + + PopPath(); + + bool isEnd = (idx == static_cast(GetProto().pairs_size())); + if (isEnd) { + idx--; + } + + OpenPairs(idx); + + return !isEnd; + } + + void ClosePairs() { + if (GetPathBack().Kind != EParseKind::Pairs && GetPathBack().Kind != EParseKind::Pair) { + PopPath(); + } + + PopPath(); + } + + void CheckKind(ETypeKind kind, const std::string& method) const { + NYdb::CheckKind(TypeParser_.GetKind(), kind, method); + } + + void CheckTransportKind(Ydb::Value::ValueCase expectedCase) const { + if (expectedCase != GetProto().value_case()) { + FatalError(TStringBuilder() << "Transport value case mismatch, requested: " << (ui32)expectedCase + << ", actual: " << (ui32)GetProto().value_case()); + } + } + + void CheckPrimitive(EPrimitiveType primitiveType) const { + CheckKind(ETypeKind::Primitive, "Get"); + + if (primitiveType != TypeParser_.GetPrimitive()) { + FatalError(TStringBuilder() << "Type mismatch, requested: " << primitiveType + << ", actual: " << TypeParser_.GetPrimitive()); + } + + CheckTransportKind(GetPrimitiveValueCase(primitiveType)); + } + + void CheckDecimal() const { + CheckKind(ETypeKind::Decimal, "Get"); + CheckTransportKind(Ydb::Value::kLow128); + } + + void CheckPg() const { + CheckKind(ETypeKind::Pg, "Get"); + } + + const Ydb::Value& GetProto() const { + return *static_cast(GetPathBack().Ptr); + } + + const Ydb::ValuePair& GetProtoPair() const { + if (GetPathBack().Kind != EParseKind::Pair) { + FatalError(TStringBuilder() << "Bad parser state, expected dict pair"); + } + + return *static_cast(GetPathBack().Ptr); + } + + Ydb::Value::ValueCase GetPrimitiveValueCase(NYdb::EPrimitiveType primitiveTypeId) const { + switch (primitiveTypeId) { + case NYdb::EPrimitiveType::Bool: + return Ydb::Value::kBoolValue; + case NYdb::EPrimitiveType::Int8: + return Ydb::Value::kInt32Value; + case NYdb::EPrimitiveType::Uint8: + return Ydb::Value::kUint32Value; + case NYdb::EPrimitiveType::Int16: + return Ydb::Value::kInt32Value; + case NYdb::EPrimitiveType::Uint16: + return Ydb::Value::kUint32Value; + case NYdb::EPrimitiveType::Int32: + return Ydb::Value::kInt32Value; + case NYdb::EPrimitiveType::Uint32: + return Ydb::Value::kUint32Value; + case NYdb::EPrimitiveType::Int64: + return Ydb::Value::kInt64Value; + case NYdb::EPrimitiveType::Uint64: + return Ydb::Value::kUint64Value; + case NYdb::EPrimitiveType::Float: + return Ydb::Value::kFloatValue; + case NYdb::EPrimitiveType::Double: + return Ydb::Value::kDoubleValue; + case NYdb::EPrimitiveType::Date: + case NYdb::EPrimitiveType::Datetime: + return Ydb::Value::kUint32Value; + case NYdb::EPrimitiveType::Date32: + return Ydb::Value::kInt32Value; + case NYdb::EPrimitiveType::Timestamp: + return Ydb::Value::kUint64Value; + case NYdb::EPrimitiveType::Interval: + case NYdb::EPrimitiveType::Interval64: + case NYdb::EPrimitiveType::Timestamp64: + case NYdb::EPrimitiveType::Datetime64: + return Ydb::Value::kInt64Value; + case NYdb::EPrimitiveType::TzDate: + case NYdb::EPrimitiveType::TzDatetime: + case NYdb::EPrimitiveType::TzTimestamp: + return Ydb::Value::kTextValue; + case NYdb::EPrimitiveType::String: + return Ydb::Value::kBytesValue; + case NYdb::EPrimitiveType::Utf8: + return Ydb::Value::kTextValue; + case NYdb::EPrimitiveType::Yson: + return Ydb::Value::kBytesValue; + case NYdb::EPrimitiveType::Json: + case NYdb::EPrimitiveType::JsonDocument: + case NYdb::EPrimitiveType::DyNumber: + return Ydb::Value::kTextValue; + case NYdb::EPrimitiveType::Uuid: + return Ydb::Value::kLow128; + default: + FatalError(TStringBuilder() << "Unexpected primitive type: " << primitiveTypeId); + return Ydb::Value::kBytesValue; + } + } + + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TValueParser: " << msg); + } + +private: + std::shared_ptr Value_; + TTypeParser TypeParser_; + TStackVec Path_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TValueParser::TValueParser(TValueParser&&) = default; +TValueParser::~TValueParser() = default; + +TValueParser::TValueParser(const TValue& value) + : Impl_(new TImpl(value)) {} + +TValueParser::TValueParser(const TType& type) + : Impl_(new TImpl(type)) {} + +void TValueParser::Reset(const Ydb::Value& value) { + Impl_->Reset(value); +} + +TTypeParser::ETypeKind TValueParser::GetKind() const { + return Impl_->GetKind(); +} + +EPrimitiveType TValueParser::GetPrimitiveType() const { + return Impl_->GetPrimitiveType(); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TValueParser::GetBool() const { + return Impl_->GetBool(); +} + +i8 TValueParser::GetInt8() const { + return Impl_->GetInt8(); +} + +ui8 TValueParser::GetUint8() const { + return Impl_->GetUint8(); +} + +i16 TValueParser::GetInt16() const { + return Impl_->GetInt16(); +} + +ui16 TValueParser::GetUint16() const { + return Impl_->GetUint16(); +} + +i32 TValueParser::GetInt32() const { + return Impl_->GetInt32(); +} + +ui32 TValueParser::GetUint32() const { + return Impl_->GetUint32(); +} + +int64_t TValueParser::GetInt64() const { + return Impl_->GetInt64(); +} + +uint64_t TValueParser::GetUint64() const { + return Impl_->GetUint64(); +} + +float TValueParser::GetFloat() const { + return Impl_->GetFloat(); +} + +double TValueParser::GetDouble() const { + return Impl_->GetDouble(); +} + +TInstant TValueParser::GetDate() const { + return Impl_->GetDate(); +} + +TInstant TValueParser::GetDatetime() const { + return Impl_->GetDatetime(); +} + +TInstant TValueParser::GetTimestamp() const { + return Impl_->GetTimestamp(); +} + +int64_t TValueParser::GetInterval() const { + return Impl_->GetInterval(); +} + +i32 TValueParser::GetDate32() const { + return Impl_->GetDate32(); +} + +int64_t TValueParser::GetDatetime64() const { + return Impl_->GetDatetime64(); +} + +int64_t TValueParser::GetTimestamp64() const { + return Impl_->GetTimestamp64(); +} + +int64_t TValueParser::GetInterval64() const { + return Impl_->GetInterval64(); +} + +const std::string& TValueParser::GetTzDate() const { + return Impl_->GetTzDate(); +} + +const std::string& TValueParser::GetTzDatetime() const { + return Impl_->GetTzDatetime(); +} + +const std::string& TValueParser::GetTzTimestamp() const { + return Impl_->GetTzTimestamp(); +} + +const std::string& TValueParser::GetString() const { + return Impl_->GetString(); +} + +const std::string& TValueParser::GetUtf8() const { + return Impl_->GetUtf8(); +} + +const std::string& TValueParser::GetYson() const { + return Impl_->GetYson(); +} + +const std::string& TValueParser::GetJson() const { + return Impl_->GetJson(); +} + +TUuidValue TValueParser::GetUuid() const { + return Impl_->GetUuid(); +} + +const std::string& TValueParser::GetJsonDocument() const { + return Impl_->GetJsonDocument(); +} + +const std::string& TValueParser::GetDyNumber() const { + return Impl_->GetDyNumber(); +} + +TDecimalValue TValueParser::GetDecimal() const { + return Impl_->GetDecimal(); +} + +TPgValue TValueParser::GetPg() const { + return Impl_->GetPg(); +} + +//////////////////////////////////////////////////////////////////////////////// + +#define RET_OPT_VALUE(Type, Name) \ + Impl_->OpenOptional(); \ + auto value = Impl_->IsNull() ? std::optional() : Impl_->Get##Name(); \ + Impl_->CloseOptional(); \ + return value; + +std::optional TValueParser::GetOptionalBool() const { + RET_OPT_VALUE(bool, Bool); +} + +std::optional TValueParser::GetOptionalInt8() const { + RET_OPT_VALUE(i8, Int8); +} + +std::optional TValueParser::GetOptionalUint8() const { + RET_OPT_VALUE(ui8, Uint8); +} + +std::optional TValueParser::GetOptionalInt16() const { + RET_OPT_VALUE(i16, Int16); +} + +std::optional TValueParser::GetOptionalUint16() const { + RET_OPT_VALUE(ui16, Uint16); +} + +std::optional TValueParser::GetOptionalInt32() const { + RET_OPT_VALUE(i32, Int32); +} + +std::optional TValueParser::GetOptionalUint32() const { + RET_OPT_VALUE(ui32, Uint32); +} + +std::optional TValueParser::GetOptionalInt64() const { + RET_OPT_VALUE(int64_t, Int64); +} + +std::optional TValueParser::GetOptionalUint64() const { + RET_OPT_VALUE(uint64_t, Uint64); +} + +std::optional TValueParser::GetOptionalFloat() const { + RET_OPT_VALUE(float, Float); +} + +std::optional TValueParser::GetOptionalDouble() const { + RET_OPT_VALUE(double, Double); +} + +std::optional TValueParser::GetOptionalDate() const { + RET_OPT_VALUE(TInstant, Date); +} + +std::optional TValueParser::GetOptionalDatetime() const { + RET_OPT_VALUE(TInstant, Datetime); +} + +std::optional TValueParser::GetOptionalTimestamp() const { + RET_OPT_VALUE(TInstant, Timestamp); +} + +std::optional TValueParser::GetOptionalInterval() const { + RET_OPT_VALUE(int64_t, Interval); +} + +std::optional TValueParser::GetOptionalDate32() const { + RET_OPT_VALUE(int64_t, Date32); +} + +std::optional TValueParser::GetOptionalDatetime64() const { + RET_OPT_VALUE(int64_t, Datetime64); +} + +std::optional TValueParser::GetOptionalTimestamp64() const { + RET_OPT_VALUE(int64_t, Timestamp64); +} + +std::optional TValueParser::GetOptionalInterval64() const { + RET_OPT_VALUE(int64_t, Interval64); +} + +std::optional TValueParser::GetOptionalTzDate() const { + RET_OPT_VALUE(std::string, TzDate); +} + +std::optional TValueParser::GetOptionalTzDatetime() const { + RET_OPT_VALUE(std::string, TzDatetime); +} + +std::optional TValueParser::GetOptionalTzTimestamp() const { + RET_OPT_VALUE(std::string, TzTimestamp); +} + +std::optional TValueParser::GetOptionalString() const { + RET_OPT_VALUE(std::string, String); +} + +std::optional TValueParser::GetOptionalUtf8() const { + RET_OPT_VALUE(std::string, Utf8); +} + +std::optional TValueParser::GetOptionalYson() const { + RET_OPT_VALUE(std::string, Yson); +} + +std::optional TValueParser::GetOptionalJson() const { + RET_OPT_VALUE(std::string, Json); +} + +std::optional TValueParser::GetOptionalUuid() const { + RET_OPT_VALUE(TUuidValue, Uuid); +} + +std::optional TValueParser::GetOptionalJsonDocument() const { + RET_OPT_VALUE(std::string, JsonDocument); +} + +std::optional TValueParser::GetOptionalDyNumber() const { + RET_OPT_VALUE(std::string, DyNumber); +} + +std::optional TValueParser::GetOptionalDecimal() const { + RET_OPT_VALUE(TDecimalValue, Decimal); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TValueParser::OpenOptional() { + Impl_->OpenOptional(); +} + +bool TValueParser::IsNull() const { + return Impl_->IsNull(); +} + +void TValueParser::CloseOptional() { + Impl_->CloseOptional(); +} + +void TValueParser::OpenList() { + Impl_->OpenList(); +} + +bool TValueParser::TryNextListItem() { + return Impl_->TryNextListItem(); +} + +void TValueParser::CloseList() { + Impl_->CloseList(); +} + +void TValueParser::OpenStruct() { + Impl_->OpenStruct(); +} + +bool TValueParser::TryNextMember() { + return Impl_->TryNextMember(); +} + +const std::string& TValueParser::GetMemberName() const { + return Impl_->GetMemberName(); +} + +void TValueParser::CloseStruct() { + Impl_->CloseStruct(); +} + +void TValueParser::OpenTuple() { + Impl_->OpenTuple(); +} + +bool TValueParser::TryNextElement() { + return Impl_->TryNextElement(); +} + +void TValueParser::CloseTuple() { + Impl_->CloseTuple(); +} + +void TValueParser::OpenDict() { + Impl_->OpenDict(); +} + +bool TValueParser::TryNextDictItem() { + return Impl_->TryNextDictItem(); +} + +void TValueParser::DictKey() { + Impl_->DictKey(); +} + +void TValueParser::DictPayload() { + Impl_->DictPayload(); +} + +void TValueParser::CloseDict() { + Impl_->CloseDict(); +} + +void TValueParser::OpenVariant() { + Impl_->OpenVariant(); +} + +void TValueParser::CloseVariant() { + Impl_->CloseVariant(); +} + +void TValueParser::OpenTagged() { + Impl_->OpenTagged(); +} + +const std::string& TValueParser::GetTag() const { + return Impl_->GetTag(); +} + +void TValueParser::CloseTagged() { + Impl_->CloseTagged(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TValueBuilderImpl { + using ETypeKind = TTypeParser::ETypeKind; + using TMembersMap = std::map; + + struct TProtoPosition { + Ydb::Value& Value; + bool BuildType = false; + ui32 OptLevel = 0; + + TProtoPosition(Ydb::Value& value) + : Value(value) {} + }; + + struct TStructPosition { + const TMembersMap* MembersMap = nullptr; + TDynBitMap FilledMembers; + + TStructPosition(const TMembersMap* membersMap) + : MembersMap(membersMap) + , FilledMembers() {} + }; + +public: + TValueBuilderImpl() + : TypeBuilder_() + { + PushPath(ProtoValue_); + } + + TValueBuilderImpl(const TType& type) + : TypeBuilder_() + { + PushPath(ProtoValue_); + GetType().CopyFrom(type.GetProto()); + } + + TValueBuilderImpl(Ydb::Type& type, Ydb::Value& value) + : TypeBuilder_(type) + { + PushPath(value); + } + + void CheckValue() { + if (Path_.size() > 1) { + FatalError("Invalid Build() call, value is incomplete."); + return; + } + } + + TValue BuildValue() { + CheckValue(); + + Ydb::Value value; + value.Swap(&ProtoValue_); + + return TValue(TypeBuilder_.Build(), std::move(value)); + } + + void Bool(bool value) { + FillPrimitiveType(EPrimitiveType::Bool); + GetValue().set_bool_value(value); + } + + void Int8(i8 value) { + FillPrimitiveType(EPrimitiveType::Int8); + GetValue().set_int32_value(value); + } + + void Uint8(ui8 value) { + FillPrimitiveType(EPrimitiveType::Uint8); + GetValue().set_uint32_value(value); + } + + void Int16(i16 value) { + FillPrimitiveType(EPrimitiveType::Int16); + GetValue().set_int32_value(value); + } + + void Uint16(ui16 value) { + FillPrimitiveType(EPrimitiveType::Uint16); + GetValue().set_uint32_value(value); + } + + void Int32(i32 value) { + FillPrimitiveType(EPrimitiveType::Int32); + GetValue().set_int32_value(value); + } + + void Uint32(ui32 value) { + FillPrimitiveType(EPrimitiveType::Uint32); + GetValue().set_uint32_value(value); + } + + void Int64(int64_t value) { + FillPrimitiveType(EPrimitiveType::Int64); + GetValue().set_int64_value(value); + } + + void Uint64(uint64_t value) { + FillPrimitiveType(EPrimitiveType::Uint64); + GetValue().set_uint64_value(value); + } + + void Float(float value) { + FillPrimitiveType(EPrimitiveType::Float); + GetValue().set_float_value(value); + } + + void Double(double value) { + FillPrimitiveType(EPrimitiveType::Double); + GetValue().set_double_value(value); + } + + void Date(const TInstant& value) { + FillPrimitiveType(EPrimitiveType::Date); + GetValue().set_uint32_value(value.Days()); + } + + void Datetime(const TInstant& value) { + FillPrimitiveType(EPrimitiveType::Datetime); + GetValue().set_uint32_value(value.Seconds()); + } + + void Timestamp(const TInstant& value) { + FillPrimitiveType(EPrimitiveType::Timestamp); + GetValue().set_uint64_value(value.MicroSeconds()); + } + + void Interval(int64_t value) { + FillPrimitiveType(EPrimitiveType::Interval); + GetValue().set_int64_value(value); + } + + void Date32(const i32 value) { + FillPrimitiveType(EPrimitiveType::Date32); + GetValue().set_int32_value(value); + } + + void Datetime64(const int64_t value) { + FillPrimitiveType(EPrimitiveType::Datetime64); + GetValue().set_int64_value(value); + } + + void Timestamp64(const int64_t value) { + FillPrimitiveType(EPrimitiveType::Timestamp64); + GetValue().set_int64_value(value); + } + + void Interval64(const int64_t value) { + FillPrimitiveType(EPrimitiveType::Interval64); + GetValue().set_int64_value(value); + } + + void TzDate(const std::string& value) { + FillPrimitiveType(EPrimitiveType::TzDate); + GetValue().set_text_value(TStringType{value}); + } + + void TzDatetime(const std::string& value) { + FillPrimitiveType(EPrimitiveType::TzDatetime); + GetValue().set_text_value(TStringType{value}); + } + + void TzTimestamp(const std::string& value) { + FillPrimitiveType(EPrimitiveType::TzTimestamp); + GetValue().set_text_value(TStringType{value}); + } + + void String(const std::string& value) { + FillPrimitiveType(EPrimitiveType::String); + GetValue().set_bytes_value(TStringType{value}); + } + + void Utf8(const std::string& value) { + FillPrimitiveType(EPrimitiveType::Utf8); + GetValue().set_text_value(TStringType{value}); + } + + void Yson(const std::string& value) { + FillPrimitiveType(EPrimitiveType::Yson); + GetValue().set_bytes_value(TStringType{value}); + } + + void Json(const std::string& value) { + FillPrimitiveType(EPrimitiveType::Json); + GetValue().set_text_value(TStringType{value}); + } + + void Uuid(const TUuidValue& value) { + FillPrimitiveType(EPrimitiveType::Uuid); + GetValue().set_low_128(value.Buf_.Halfs[0]); + GetValue().set_high_128(value.Buf_.Halfs[1]); + } + + void JsonDocument(const std::string& value) { + FillPrimitiveType(EPrimitiveType::JsonDocument); + GetValue().set_text_value(TStringType{value}); + } + + void DyNumber(const std::string& value) { + FillPrimitiveType(EPrimitiveType::DyNumber); + GetValue().set_text_value(TStringType{value}); + } + + void Decimal(const TDecimalValue& value) { + FillDecimalType(value.DecimalType_); + GetValue().set_low_128(value.Low_); + GetValue().set_high_128(value.Hi_); + } + + void Pg(const TPgValue& value) { + FillPgType(value.PgType_); + if (value.IsNull()) { + GetValue().set_null_flag_value(::google::protobuf::NULL_VALUE); + } else if (value.IsText()) { + GetValue().set_text_value(TStringType{value.Content_}); + } else { + GetValue().set_bytes_value(TStringType{value.Content_}); + } + } + + void BeginOptional() { + SetBuildType(!CheckType(ETypeKind::Optional)); + + auto optLevel = PathTop().OptLevel; + if (optLevel == 0) { + PushPath(GetValue()); + } + + TypeBuilder_.BeginOptional(); + + ++PathTop().OptLevel; + } + + void EndOptional() { + CheckContainerKind(ETypeKind::Optional); + + TypeBuilder_.EndOptional(); + + if (!PathTop().OptLevel) { + FatalError(TStringBuilder() << "No opened optional"); + } + + --PathTop().OptLevel; + + if (!PathTop().OptLevel) { + PopPath(); + } + } + + void BeginOptional(const TType& itemType) { + BeginOptional(); + if (!CheckType(itemType)) { + TypeBuilder_.SetType(itemType); + } + } + + void EmptyOptional(const TType& itemType) { + BeginOptional(itemType); + NestEmptyOptional(); + EndOptional(); + } + + void EmptyOptional(EPrimitiveType itemType) { + BeginOptional(); + FillPrimitiveType(itemType); + NestEmptyOptional(); + EndOptional(); + } + + void EmptyOptional() { + BeginOptional(); + if (!CheckType()) { + FatalError(TStringBuilder() << "EmptyOptional: unknown item type"); + return; + } + NestEmptyOptional(); + EndOptional(); + } + + void BeginList(const TType& itemType) { + BeginList(); + + if (!CheckType(itemType)) { + TypeBuilder_.SetType(itemType); + } + } + + void BeginList() { + SetBuildType(!CheckType(ETypeKind::List)); + + TypeBuilder_.BeginList(); + PushPath(GetValue()); + } + + void EndList() { + CheckContainerKind(ETypeKind::List); + + TypeBuilder_.EndList(); + PopPath(); + } + + void AddListItem() { + CheckContainerKind(ETypeKind::List); + PopPath(); + PushPath(*GetValue().add_items()); + } + + void AddListItem(const TValue& itemValue) { + CheckContainerKind(ETypeKind::List); + PopPath(); + PushPath(*GetValue().add_items()); + + if (!CheckType(itemValue.GetType())) { + TypeBuilder_.SetType(itemValue.GetType()); + } + SetProtoValue(itemValue); + } + + void AddListItem(TValue&& itemValue) { + CheckContainerKind(ETypeKind::List); + PopPath(); + PushPath(*GetValue().add_items()); + + if (!CheckType(itemValue.GetType())) { + TypeBuilder_.SetType(std::move(itemValue.GetType())); + } + SetProtoValue(std::move(itemValue)); + } + + void EmptyList(const TType& itemType) { + BeginList(itemType); + EndList(); + } + + void EmptyList() { + BeginList(); + if (!CheckType()) { + FatalError(TStringBuilder() << "EmptyList: unknown item type"); + return; + } + EndList(); + } + + void BeginStruct() { + SetBuildType(!CheckType(ETypeKind::Struct)); + if (!GetBuildType()) { + auto& membersMap = GetMembersMap(GetType().mutable_struct_type()); + GetValue().mutable_items()->Reserve(membersMap.size()); + for (ui32 i = 0; i < membersMap.size(); ++i) { + GetValue().mutable_items()->Add(); + } + + PushStructsPath(&membersMap); + } + + TypeBuilder_.BeginStruct(); + PushPath(GetValue()); + } + + void AddMember(const std::string& memberName) { + CheckContainerKind(ETypeKind::Struct); + + PopPath(); + + if (GetBuildType()) { + TypeBuilder_.AddMember(memberName); + PushPath(*GetValue().add_items()); + } else { + auto membersMap = StructsPathTop().MembersMap; + if (!membersMap) { + FatalError(TStringBuilder() << "Missing struct members info."); + return; + } + + auto memberIndex = MapFindPtr(*membersMap, memberName); + if (!memberIndex) { + FatalError(TStringBuilder() << "Struct member not found: " << memberName); + return; + } + + TypeBuilder_.SelectMember(*memberIndex); + PushPath(*GetValue().mutable_items(*memberIndex)); + + StructsPathTop().FilledMembers.Set(*memberIndex); + } + } + + void AddMember(const std::string& memberName, const TValue& memberValue) { + AddMember(memberName); + + if (!CheckType(memberValue.GetType())) { + TypeBuilder_.SetType(memberValue.GetType()); + } + + SetProtoValue(memberValue); + } + + void AddMember(const std::string& memberName, TValue&& memberValue) { + AddMember(memberName); + + if (!CheckType(memberValue.GetType())) { + TypeBuilder_.SetType(std::move(memberValue.GetType())); + } + + SetProtoValue(std::move(memberValue)); + } + + void EndStruct() { + CheckContainerKind(ETypeKind::Struct); + + PopPath(); + TypeBuilder_.EndStruct(); + + if (!GetBuildType()) { + auto& membersMap = *StructsPathTop().MembersMap; + auto& filledMembers = StructsPathTop().FilledMembers; + + if (filledMembers.Count() < membersMap.size()) { + filledMembers.Flip(); + auto index = filledMembers.FirstNonZeroBit(); + auto it = std::find_if(membersMap.begin(), membersMap.end(), + [index](const auto& pair) { return pair.second == index; }); + + FatalError(TStringBuilder() << "No value given for struct member: " << it->first); + } + + PopStructsPath(); + } + } + + void BeginTuple() { + SetBuildType(!CheckType(ETypeKind::Tuple)); + TypeBuilder_.BeginTuple(); + PushPath(GetValue()); + } + + void AddElement() { + CheckContainerKind(ETypeKind::Tuple); + + PopPath(); + + if (GetBuildType()) { + TypeBuilder_.AddElement(); + } else { + auto index = GetValue().items_size(); + auto tuple_size = GetType(1).tuple_type().elements_size(); + if (index >= tuple_size) { + FatalError(TStringBuilder() << "Tuple elements count mismatch, expected: " + << tuple_size << ", actual: " << index + 1); + return; + } + TypeBuilder_.SelectElement(index); + } + + PushPath(*GetValue().add_items()); + } + + void AddElement(const TValue& elementValue) { + AddElement(); + + if (!CheckType(elementValue.GetType())) { + TypeBuilder_.SetType(elementValue.GetType()); + } + + SetProtoValue(elementValue); + } + + void EndTuple() { + CheckContainerKind(ETypeKind::Tuple); + + PopPath(); + TypeBuilder_.EndTuple(); + + if (!GetBuildType()) { + auto expectedElements = GetType().tuple_type().elements_size(); + auto actualIElements = GetValue().items_size(); + if (expectedElements != actualIElements) { + FatalError(TStringBuilder() << "Tuple elements count mismatch, expected: " << expectedElements + << ", actual: " << actualIElements); + } + } + } + + void BeginDict() { + SetBuildType(!CheckType(ETypeKind::Dict)); + TypeBuilder_.BeginDict(); + PushPath(GetValue()); + } + + void BeginDict(const TType& keyType, const TType& payloadType) { + BeginDict(); + + TypeBuilder_.DictKey(); + if (!CheckType(keyType)) { + TypeBuilder_.SetType(keyType); + } + + TypeBuilder_.DictPayload(); + if (!CheckType(payloadType)) { + TypeBuilder_.SetType(payloadType); + } + + TypeBuilder_.EndDict(); + TypeBuilder_.BeginDict(); + } + + void AddDictItem() { + CheckContainerKind(ETypeKind::Dict); + PopPath(); + + GetValue().add_pairs(); + PushPath(GetValue()); + } + + void DictKey() { + CheckContainerKind(ETypeKind::Dict); + PopPath(); + + TypeBuilder_.DictKey(); + PushPath(*GetValue().mutable_pairs()->rbegin()->mutable_key()); + } + + void DictKey(const TValue& keyValue) { + DictKey(); + + if (!CheckType(keyValue.GetType())) { + TypeBuilder_.SetType(keyValue.GetType()); + } + + SetProtoValue(keyValue); + } + + void DictPayload() { + CheckContainerKind(ETypeKind::Dict); + PopPath(); + + TypeBuilder_.DictPayload(); + PushPath(*GetValue().mutable_pairs()->rbegin()->mutable_payload()); + } + + void DictPayload(const TValue& payloadValue) { + DictPayload(); + + if (!CheckType(payloadValue.GetType())) { + TypeBuilder_.SetType(payloadValue.GetType()); + } + + SetProtoValue(payloadValue); + } + + void EndDict() { + CheckContainerKind(ETypeKind::Dict); + + PopPath(); + TypeBuilder_.EndDict(); + } + + void EmptyDict(const TType& keyType, const TType& payloadType) { + BeginDict(keyType, payloadType); + EndDict(); + } + + void EmptyDict() { + BeginDict(); + + TypeBuilder_.DictKey(); + if (!CheckType()) { + FatalError(TStringBuilder() << "EmptyDict: unknown key type"); + return; + } + + TypeBuilder_.DictPayload(); + if (!CheckType()) { + FatalError(TStringBuilder() << "EmptyDict: unknown payload type"); + return; + } + + EndDict(); + } + + void BeginTagged(const std::string& tag) { + SetBuildType(!CheckType(ETypeKind::Tagged)); + TypeBuilder_.BeginTagged(tag); + PushPath(GetValue()); + } + + void EndTagged() { + CheckContainerKind(ETypeKind::Tagged); + + PopPath(); + TypeBuilder_.EndTagged(); + } + +private: + Ydb::Type& GetType(size_t offset = 0) { + return TypeBuilder_.GetProto(offset); + } + + Ydb::Value& GetValue() { + return PathTop().Value; + } + + void SetProtoValue(const TValue& value) { + GetValue().CopyFrom(value.GetProto()); + } + + void SetProtoValue(TValue&& value) { + GetValue() = std::move(value.GetProto()); + } + + bool GetBuildType() { + return PathTop().BuildType; + } + + void SetBuildType(bool value) { + PathTop().BuildType = value; + } + + void FillPrimitiveType(EPrimitiveType type) { + if (!CheckPrimitiveType(type)) { + TypeBuilder_.Primitive(type); + } + } + + void FillDecimalType(const TDecimalType& type) { + if (!CheckDecimalType()) { + TypeBuilder_.Decimal(type); + } + } + + void FillPgType(const TPgType& type) { + if (!CheckPgType()) { + TypeBuilder_.Pg(type); + } + } + + bool CheckType() { + if (!GetType().type_case()) { + return false; + } + + return true; + } + + bool CheckType(ETypeKind kind) { + if (!CheckType()) { + return false; + } + + auto expectedKind = GetKind(GetType()); + if (expectedKind != kind) { + FatalError(TStringBuilder() << "Type mismatch, expected: " << expectedKind + << ", actual: " << kind); + return false; + } + + return true; + } + + bool CheckType(const TType& type) { + if (!CheckType()) { + return false; + } + + if (!TypesEqual(GetType(), type.GetProto())) { + FatalError(TStringBuilder() << "Type mismatch, expected: " << FormatType(GetType()) + << ", actual: " << FormatType(type)); + return false; + } + + return true; + } + + bool CheckPrimitiveType(EPrimitiveType type) { + if (!CheckType(ETypeKind::Primitive)) { + return false; + } + + auto expectedType = EPrimitiveType(GetType().type_id()); + if (expectedType != type) { + FatalError(TStringBuilder() << "Primitive type mismatch, expected: " << expectedType + << ", actual: " << type); + return false; + } + + return true; + } + + bool CheckDecimalType() { + return CheckType(ETypeKind::Decimal); + } + + bool CheckPgType() { + return CheckType(ETypeKind::Pg); + } + + void CheckContainerKind(ETypeKind kind) { + if (Path_.size() < 2) { + FatalError(TStringBuilder() << "No opened container"); + } + + auto actualKind = GetKind(GetType(1)); + if (actualKind != kind) { + FatalError(TStringBuilder() << "Container type mismatch, expected: " << kind + << ", actual: " << actualKind); + } + } + + void NestEmptyOptional() { + ui32 optLevel = PathTop().OptLevel - 1; + for (ui32 i = 0; i < optLevel; ++i) { + PushPath(*GetValue().mutable_nested_value()); + } + + GetValue().set_null_flag_value(::google::protobuf::NULL_VALUE); + + for (ui32 i = 0; i < optLevel; ++i) { + PopPath(); + } + } + + TProtoPosition& PathTop() { + return Path_.back(); + } + + void PushPath(Ydb::Value& value) { + Path_.emplace_back(value); + } + + void PopPath() { + Path_.pop_back(); + } + + void PushStructsPath(const TMembersMap* membersMap) { + StructsPath_.emplace_back(membersMap); + } + + void PopStructsPath() { + StructsPath_.pop_back(); + } + + TStructPosition& StructsPathTop() { + return StructsPath_.back(); + } + + TMembersMap& GetMembersMap(const Ydb::StructType* structType) { + auto it = StructsMap_.find(structType); + if (it == StructsMap_.end()) { + TMembersMap membersMap; + for (size_t i = 0; i < (size_t)structType->members_size(); ++i) { + membersMap.emplace(std::make_pair(structType->members(i).name(), i)); + } + auto result = StructsMap_.emplace(std::make_pair(structType, std::move(membersMap))); + it = result.first; + } + + return it->second; + } + + void FatalError(const std::string& msg) const { + ThrowFatalError(TStringBuilder() << "TValueBuilder: " << msg); + } + +private: + + //TTypeBuilder TypeBuilder_; + TTypeBuilder::TImpl TypeBuilder_; + Ydb::Value ProtoValue_; + std::map StructsMap_; + + TStackVec Path_; + TStackVec StructsPath_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template +TValueBuilderBase::TValueBuilderBase(TValueBuilderBase&&) = default; + +template +TValueBuilderBase::~TValueBuilderBase() = default; + +template +TValueBuilderBase::TValueBuilderBase() + : Impl_(new TValueBuilderImpl()) {} + +template +TValueBuilderBase::TValueBuilderBase(const TType& type) + : Impl_(new TValueBuilderImpl(type)) {} + +template +TValueBuilderBase::TValueBuilderBase(Ydb::Type& type, Ydb::Value& value) + : Impl_(new TValueBuilderImpl(type, value)) {} + +template +void TValueBuilderBase::CheckValue() { + return Impl_->CheckValue(); +} + +template +TDerived& TValueBuilderBase::Bool(bool value) { + Impl_->Bool(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Int8(i8 value) { + Impl_->Int8(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Uint8(ui8 value) { + Impl_->Uint8(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Int16(i16 value) { + Impl_->Int16(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Uint16(ui16 value) { + Impl_->Uint16(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Int32(i32 value) { + Impl_->Int32(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Uint32(ui32 value) { + Impl_->Uint32(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Int64(int64_t value) { + Impl_->Int64(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Uint64(uint64_t value) { + Impl_->Uint64(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Float(float value) { + Impl_->Float(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Double(double value) { + Impl_->Double(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Date(const TInstant& value) { + Impl_->Date(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Datetime(const TInstant& value) { + Impl_->Datetime(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Timestamp(const TInstant& value) { + Impl_->Timestamp(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Interval(int64_t value) { + Impl_->Interval(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Date32(const i32 value) { + Impl_->Date32(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Datetime64(const int64_t value) { + Impl_->Datetime64(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Timestamp64(const int64_t value) { + Impl_->Timestamp64(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Interval64(const int64_t value) { + Impl_->Interval64(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::TzDate(const std::string& value) { + Impl_->TzDate(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::TzDatetime(const std::string& value) { + Impl_->TzDatetime(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::TzTimestamp(const std::string& value) { + Impl_->TzTimestamp(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::String(const std::string& value) { + Impl_->String(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Utf8(const std::string& value) { + Impl_->Utf8(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Yson(const std::string& value) { + Impl_->Yson(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Json(const std::string& value) { + Impl_->Json(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Uuid(const TUuidValue& value) { + Impl_->Uuid(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::JsonDocument(const std::string& value) { + Impl_->JsonDocument(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::DyNumber(const std::string& value) { + Impl_->DyNumber(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Decimal(const TDecimalValue& value) { + Impl_->Decimal(value); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::Pg(const TPgValue& value) { + Impl_->Pg(value); + return static_cast(*this); +} + +#define SET_OPT_VALUE_MAYBE(Name) \ + if (value) { \ + Impl_->BeginOptional(); \ + Impl_->Name(*value); \ + Impl_->EndOptional(); \ + } else { \ + Impl_->EmptyOptional(EPrimitiveType::Name); \ + } \ + return static_cast(*this); + +#define SET_OPT_VALUE(Name) \ + Impl_->BeginOptional(); \ + Impl_->Name(value); \ + Impl_->EndOptional(); \ + return static_cast(*this); + +template +TDerived& TValueBuilderBase::OptionalBool(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Bool); +} + +template +TDerived& TValueBuilderBase::OptionalInt8(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Int8); +} + +template +TDerived& TValueBuilderBase::OptionalUint8(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Uint8); +} + +template +TDerived& TValueBuilderBase::OptionalInt16(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Int16); +} + +template +TDerived& TValueBuilderBase::OptionalUint16(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Uint16); +} + +template +TDerived& TValueBuilderBase::OptionalInt32(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Int32); +} + +template +TDerived& TValueBuilderBase::OptionalUint32(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Uint32); +} + +template +TDerived& TValueBuilderBase::OptionalInt64(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Int64); +} + +template +TDerived& TValueBuilderBase::OptionalUint64(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Uint64); +} + +template +TDerived& TValueBuilderBase::OptionalFloat(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Float); +} + +template +TDerived& TValueBuilderBase::OptionalDouble(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Double); +} + +template +TDerived& TValueBuilderBase::OptionalDate(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Date); +} + +template +TDerived& TValueBuilderBase::OptionalDatetime(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Datetime); +} + +template +TDerived& TValueBuilderBase::OptionalTimestamp(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Timestamp); +} + +template +TDerived& TValueBuilderBase::OptionalInterval(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Interval); +} + +template +TDerived& TValueBuilderBase::OptionalDate32(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Date32); +} + +template +TDerived& TValueBuilderBase::OptionalDatetime64(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Datetime64); +} + +template +TDerived& TValueBuilderBase::OptionalTimestamp64(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Timestamp64); +} + +template +TDerived& TValueBuilderBase::OptionalInterval64(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Interval64); +} + +template +TDerived& TValueBuilderBase::OptionalTzDate(const std::optional& value) { + SET_OPT_VALUE_MAYBE(TzDate); +} + +template +TDerived& TValueBuilderBase::OptionalTzDatetime(const std::optional& value) { + SET_OPT_VALUE_MAYBE(TzDatetime); +} + +template +TDerived& TValueBuilderBase::OptionalTzTimestamp(const std::optional& value) { + SET_OPT_VALUE_MAYBE(TzTimestamp); +} + +template +TDerived& TValueBuilderBase::OptionalString(const std::optional& value) { + SET_OPT_VALUE_MAYBE(String); +} + +template +TDerived& TValueBuilderBase::OptionalUtf8(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Utf8); +} + +template +TDerived& TValueBuilderBase::OptionalYson(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Yson); +} + +template +TDerived& TValueBuilderBase::OptionalJson(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Json); +} + +template +TDerived& TValueBuilderBase::OptionalUuid(const std::optional& value) { + SET_OPT_VALUE_MAYBE(Uuid); +} + +template +TDerived& TValueBuilderBase::OptionalJsonDocument(const std::optional& value) { + SET_OPT_VALUE_MAYBE(JsonDocument); +} + +template +TDerived& TValueBuilderBase::OptionalDyNumber(const std::optional& value) { + SET_OPT_VALUE_MAYBE(DyNumber); +} + + +template +TDerived& TValueBuilderBase::BeginOptional() { + Impl_->BeginOptional(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndOptional() { + Impl_->EndOptional(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyOptional(const TType& itemType) { + Impl_->EmptyOptional(itemType); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyOptional(EPrimitiveType itemType) { + Impl_->EmptyOptional(itemType); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyOptional() { + Impl_->EmptyOptional(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::BeginList() { + Impl_->BeginList(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndList() { + Impl_->EndList(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddListItem() { + Impl_->AddListItem(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddListItem(const TValue& itemValue) { + Impl_->AddListItem(itemValue); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddListItem(TValue&& itemValue) { + Impl_->AddListItem(std::move(itemValue)); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyList(const TType& itemType) { + Impl_->EmptyList(itemType); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyList() { + Impl_->EmptyList(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::BeginStruct() { + Impl_->BeginStruct(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddMember(const std::string& memberName) { + Impl_->AddMember(memberName); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddMember(const std::string& memberName, const TValue& memberValue) { + Impl_->AddMember(memberName, memberValue); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddMember(const std::string& memberName, TValue&& memberValue) { + Impl_->AddMember(memberName, std::move(memberValue)); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndStruct() { + Impl_->EndStruct(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::BeginTuple() { + Impl_->BeginTuple(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddElement() { + Impl_->AddElement(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddElement(const TValue& elementValue) { + Impl_->AddElement(elementValue); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndTuple() { + Impl_->EndTuple(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::BeginDict() { + Impl_->BeginDict(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::AddDictItem() { + Impl_->AddDictItem(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::DictKey() { + Impl_->DictKey(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::DictKey(const TValue& keyValue) { + Impl_->DictKey(keyValue); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::DictPayload() { + Impl_->DictPayload(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::DictPayload(const TValue& payloadValue) { + Impl_->DictPayload(payloadValue); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndDict() { + Impl_->EndDict(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyDict(const TType& keyType, const TType& payloadType) { + Impl_->EmptyDict(keyType, payloadType); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EmptyDict() { + Impl_->EmptyDict(); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::BeginTagged(const std::string& tag) { + Impl_->BeginTagged(tag); + return static_cast(*this); +} + +template +TDerived& TValueBuilderBase::EndTagged() { + Impl_->EndTagged(); + return static_cast(*this); +} + +//////////////////////////////////////////////////////////////////////////////// + +template class TValueBuilderBase; +template class TValueBuilderBase; + +//////////////////////////////////////////////////////////////////////////////// + +TValueBuilder::TValueBuilder() + : TValueBuilderBase() {} + +TValueBuilder::TValueBuilder(const TType& type) + : TValueBuilderBase(type) {} + +TValue TValueBuilder::Build() { + return Impl_->BuildValue(); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/value/ya.make b/ydb/public/sdk/cpp/src/client/value/ya.make new file mode 100644 index 000000000000..4a98d9d5e8f9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/value/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + out.cpp + value.cpp +) + +GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h) + +PEERDIR( + library/cpp/containers/stack_vector + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/impl/ydb_internal/value_helpers + ydb/public/sdk/cpp/src/client/types/fatal_error_handlers + ydb/public/sdk/cpp/src/library/decimal + ydb/public/sdk/cpp/src/library/uuid +) + +END() diff --git a/ydb/public/sdk/cpp/src/client/ya.make b/ydb/public/sdk/cpp/src/client/ya.make new file mode 100644 index 000000000000..67f3fe957add --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/ya.make @@ -0,0 +1,52 @@ +RECURSE( + bsconfig + draft + extensions + helpers + impl/ydb_endpoints + impl/ydb_internal/common + impl/ydb_internal/db_driver_state + impl/ydb_internal/grpc_connections + impl/ydb_internal/logger + impl/ydb_internal/make_request + impl/ydb_internal/plain_status + impl/ydb_internal/thread_pool + impl/ydb_internal/value_helpers + impl/ydb_stats + resources + common_client + common_client/impl + coordination + datastreams + debug + discovery + driver + export + extension_common + federated_topic + federated_topic/impl + import + operation + params + persqueue_public + persqueue_public/impl + proto + query + query/impl + rate_limiter + result + scheme + table + table/impl + table/query_stats + topic + topic/codecs + topic/impl + types + types/credentials + types/exceptions + types/fatal_error_handlers + types/operation + types/status + value +) diff --git a/ydb/public/sdk/cpp/src/library/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/CMakeLists.txt new file mode 100644 index 000000000000..33f373a28d93 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/CMakeLists.txt @@ -0,0 +1,18 @@ +if (YDB_SDK_TESTS) + add_subdirectory(benchmark) +endif() + +add_subdirectory(decimal) +add_subdirectory(grpc/client) +add_subdirectory(issue) +add_subdirectory(jwt) +add_subdirectory(malloc/api) +add_subdirectory(operation_id) +add_subdirectory(persqueue/obfuscate) +add_subdirectory(persqueue/topic_parser_public) +add_subdirectory(proto_output) +add_subdirectory(retry) +add_subdirectory(string_utils/base64) +add_subdirectory(string_utils/helpers) +add_subdirectory(string_utils/misc) +add_subdirectory(uuid) diff --git a/ydb/public/sdk/cpp/src/library/benchmark/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/benchmark/CMakeLists.txt new file mode 100644 index 000000000000..90a15166bf95 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/benchmark/CMakeLists.txt @@ -0,0 +1 @@ +_ydb_sdk_add_library(benchmark-clobber INTERFACE) diff --git a/ydb/public/sdk/cpp/src/library/benchmark/clobber.h b/ydb/public/sdk/cpp/src/library/benchmark/clobber.h new file mode 100644 index 000000000000..f8ce28a3f1d6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/benchmark/clobber.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace NBench { + +/** + * Functions that states "I can read and write everywhere in memory". + * + * Use it to prevent optimizer from reordering or discarding memory writes prior + * to it's call, and force memory reads after it's call. + */ +Y_FORCE_INLINE void Clobber() { +#if defined(__GNUC__) + asm volatile("" + : + : + : "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#else + // Otherwise, do nothing +#endif +} + +} // namespace NBench diff --git a/ydb/public/sdk/cpp/src/library/decimal/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/decimal/CMakeLists.txt new file mode 100644 index 000000000000..edbf8f0917bd --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/decimal/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(yql-public-decimal) + +target_link_libraries(yql-public-decimal PUBLIC + yutil +) + +target_sources(yql-public-decimal + PRIVATE + yql_decimal.cpp +) + +_ydb_sdk_install_targets(TARGETS yql-public-decimal) diff --git a/ydb/public/sdk/cpp/src/library/decimal/ya.make b/ydb/public/sdk/cpp/src/library/decimal/ya.make new file mode 100644 index 000000000000..15f41e160575 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/decimal/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + yql_decimal.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.cpp b/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.cpp new file mode 100644 index 000000000000..0eb36911ff76 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.cpp @@ -0,0 +1,241 @@ +#include "yql_decimal.h" + +#include +#include +#include + +namespace NYdb::NDecimal { + +static const TUint128 Ten(10U); + +TUint128 GetDivider(ui8 scale) { + TUint128 d(1U); + while (scale--) + d *= Ten; + return d; +} + +bool IsError(TInt128 v) { + return v > Nan() || v < -Inf(); +} + +bool IsNan(TInt128 v) { + return v == Nan(); +} + +bool IsInf(TInt128 v) { + return v == Inf() || v == -Inf(); +} + +bool IsNormal(TInt128 v) { + return v < Inf() && v > -Inf(); +} + +const char* ToString(TInt128 val, ui8 precision, ui8 scale) { + if (!precision || precision > MaxPrecision || scale > precision) { + return ""; + } + + if (val == Inf()) + return "inf"; + if (val == -Inf()) + return "-inf"; + if (val == Nan()) + return "nan"; + + if (!IsNormal(val)) { + return nullptr; + } + + if (!val) { + return "0"; + } + + const bool neg = val < 0; + TUint128 v = neg ? -val : val; + + // log_{10}(2^120) ~= 36.12, 37 decimal places + // plus dot, zero before dot, sign and zero byte at the end + static thread_local char str[40]; + auto end = str + sizeof(str); + *--end = 0; + + auto s = end; + + do { + if (!precision--) { + return ""; + } + + + const auto digit = ui8(v % Ten); + if (digit || !scale || s != end) { + *--s = "0123456789"[digit]; + } + + if (scale && !--scale && s != end) { + *--s = '.'; + } + } while (v /= Ten); + + if (scale) { + do { + if (!precision--) { + return nullptr; + } + + *--s = '0'; + } while (--scale); + + *--s = '.'; + } + + if (*s == '.') { + *--s = '0'; + } + + if (neg) { + *--s = '-'; + } + + return s; +} + +namespace { + bool IsNan(const char* s) { + return (s[0] == 'N' || s[0] == 'n') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n'); + } + + bool IsInf(const char* s) { + return (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'F' || s[2] == 'f'); + } +} + + +TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale) { + if (scale > precision) + return Err(); + + auto s = str.data(); + auto l = str.size(); + + if (!s || !l) + return Err(); + + const bool neg = '-' == *s; + if (neg || '+' == *s) { + ++s; + --l; + } + + if (3U == l) { + if (IsInf(s)) + return neg ? -Inf() : Inf(); + if (IsNan(s)) + return Nan(); + } + + TUint128 v = 0U; + auto integral = precision - scale; + + for (bool dot = false; l; --l) { + if (*s == '.') { + if (dot) + return Err(); + + ++s; + dot = true; + continue; + } + + if (dot) { + if (scale) + --scale; + else + break; + } + + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + v *= Ten; + v += c - '0'; + + if (!dot && v && !integral--) { + return neg ? -Inf() : Inf(); + } + } + + if (l--) { + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + bool plus = c > '5'; + if (!plus && c == '5') { + for (plus = v & 1; !plus && l; --l) { + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + plus = c != '0'; + } + } + + while (l--) + if (!std::isdigit(*s++)) + return Err(); + + if (plus) + if (++v >= GetDivider(precision)) + v = Inf(); + } + + while (scale--) + v *= Ten; + + return neg ? -v : v; +} + +TInt128 FromHalfs(ui64 lo, i64 hi) { + ui64 half[2] = {lo, static_cast(hi)}; + TInt128 val128; + std::memcpy(&val128, half, sizeof(val128)); + return val128; +} + +bool IsValid(const std::string_view& str) { + auto s = str.data(); + auto l = str.size(); + + if (!s || !l) + return false; + + if ('-' == *s || '+' == *s) { + ++s; + --l; + } + + if (3U == l && (IsInf(s) || IsNan(s))) { + return true; + } + + for (bool dot = false; l--;) { + const char c = *s++; + if (c == '.') { + if (dot) + return false; + + dot = true; + continue; + } + + if (!std::isdigit(c)) + return false; + } + + return true; +} + +} diff --git a/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.h b/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.h new file mode 100644 index 000000000000..d768a1e1b3f8 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/decimal/yql_decimal.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include "yql_wide_int.h" + +#include +#include +#include + +namespace NYdb::NDecimal { + +#ifdef _win_ +#ifndef DONT_USE_NATIVE_INT128 +#define DONT_USE_NATIVE_INT128 +#endif +#endif + +#ifdef DONT_USE_NATIVE_INT128 +using TInt128 = TWide; +using TUint128 = TWide; +#else +using TInt128 = signed __int128; +using TUint128 = unsigned __int128; +#endif + +constexpr ui8 MaxPrecision = 35; + +static_assert(sizeof(TInt128) == 16, "Wrong size of TInt128, expected 16"); + +inline constexpr TInt128 Inf() { + return TInt128(100000000000000000ULL) * TInt128(1000000000000000000ULL); +} + +inline constexpr TInt128 Nan() { + return Inf() + TInt128(1); +} + +inline constexpr TInt128 Err() { + return Nan() + TInt128(1); +} + +bool IsError(TInt128 v); +bool IsNan(TInt128 v); +bool IsInf(TInt128 v); + +bool IsNormal(TInt128 v); + +const char* ToString(TInt128 v, ui8 precision, ui8 scale = 0); +TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale = 0); +TInt128 FromHalfs(ui64 lo, i64 hi); + +bool IsValid(const std::string_view& str); + +} diff --git a/ydb/public/sdk/cpp/src/library/decimal/yql_wide_int.h b/ydb/public/sdk/cpp/src/library/decimal/yql_wide_int.h new file mode 100644 index 000000000000..00a9cbe575a0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/decimal/yql_wide_int.h @@ -0,0 +1,344 @@ +#pragma once + +#include + +#include +#include +#include + +namespace NYdb::NDecimal { + +#ifndef _win_ +typedef __int128 i128_t; +typedef unsigned __int128 ui128_t; +#endif + +template, typename TUnsignedPart = std::make_unsigned_t> +class TWide { +private: + using THalf = TOneHalf; + using TPart = TUnsignedPart; + + using TIsSigned = std::is_same; + using TIsUnsigned = std::is_same; + + static_assert(TIsSigned::value || TIsUnsigned::value, "Invalid using of TWide."); + static_assert(sizeof(TSignedPart) == sizeof(TUnsignedPart), "Different sizes of TWide parts."); + + TPart Lo; + THalf Hi; + + static constexpr auto FullBitSize = sizeof(TPart) << 4U; + static constexpr auto PartBitSize = sizeof(TPart) << 3U; + static constexpr auto QuarterBitSize = sizeof(TPart) << 2U; + + static constexpr TPart GetUpperQuarter(TPart h) { + return h >> QuarterBitSize; + } + + static constexpr TPart GetLowerQuarter(TPart h) { + const auto mask = TPart(~TPart(0U)) >> QuarterBitSize; + return h & mask; + } + + template + constexpr std::enable_if_t CastImpl() const { + return static_cast(Lo); + } + + template + constexpr std::enable_if_t CastImpl() const { + return *reinterpret_cast(this); + } + + template + constexpr std::enable_if_t CastImpl() const { + return (T(std::make_unsigned_t(Hi) << PartBitSize)) | Lo; + } + + constexpr size_t GetBits() const { + size_t out = Hi ? PartBitSize : 0U; + for (auto up = TPart(out ? Hi : Lo); up; up >>= 1U) { + ++out; + } + return out; + } + + using TUnsigned = TWide; + + using TSibling = std::conditional_t::value, + TWide, TWide>; + friend TSibling; +public: + constexpr TWide() = default; + constexpr TWide(const TWide& rhs) = default; + constexpr TWide(TWide&& rhs) = default; + + constexpr TWide& operator=(const TWide& rhs) = default; + constexpr TWide& operator=(TWide&& rhs) = default; + + constexpr TWide(const TSibling& rhs) + : Lo(rhs.Lo), Hi(rhs.Hi) + {} + + template + constexpr TWide(const U upper_rhs, const L lower_rhs) + : Lo(lower_rhs), Hi(upper_rhs) + {} + + template ::value && sizeof(THalf) < sizeof(T), size_t> Shift = PartBitSize> + constexpr TWide(const T rhs) + : Lo(rhs), Hi(rhs >> Shift) + {} + + template ::value && sizeof(T) <= sizeof(THalf), bool> Signed = std::is_signed::value> + constexpr TWide(const T rhs) + : Lo(rhs), Hi(Signed && rhs < 0 ? ~0 : 0) + {} + + template ::value && std::is_same::value, THalf>> + constexpr explicit TWide(const T& rhs) + : Lo(rhs), Hi(TIsSigned::value && rhs < 0 ? ~0 : 0) + {} + + template ::value && sizeof(THalf) < sizeof(T), size_t> Shift = PartBitSize> + constexpr TWide& operator=(const T rhs) { + Hi = rhs >> Shift; + Lo = rhs; + return *this; + } + + template ::value && sizeof(T) <= sizeof(THalf), bool> Signed = std::is_signed::value> + constexpr TWide& operator=(const T rhs) { + Hi = Signed && rhs < 0 ? ~0 : 0; + Lo = rhs; + return *this; + } + + constexpr explicit operator bool() const { return bool(Hi) || bool(Lo); } + + constexpr explicit operator i8() const { return CastImpl(); } + constexpr explicit operator ui8() const { return CastImpl(); } + constexpr explicit operator i16() const { return CastImpl(); } + constexpr explicit operator ui16() const { return CastImpl(); } + constexpr explicit operator i32() const { return CastImpl(); } + constexpr explicit operator ui32() const { return CastImpl(); } + constexpr explicit operator i64() const { return CastImpl(); } + constexpr explicit operator ui64() const { return CastImpl(); } +#ifndef _win_ + constexpr explicit operator i128_t() const { return CastImpl(); } + constexpr explicit operator ui128_t() const { return CastImpl(); } +#endif + constexpr explicit operator float() const { + return TIsSigned::value && Hi < 0 ? -float(-*this) : std::fma(float(Hi), std::exp2(float(PartBitSize)), float(Lo)); + } + + constexpr explicit operator double() const { + return TIsSigned::value && Hi < 0 ? -double(-*this) : std::fma(double(Hi), std::exp2(double(PartBitSize)), double(Lo)); + } + + constexpr TWide operator~() const { + return TWide(~Hi, ~Lo); + } + + constexpr TWide operator+() const { + return TWide(Hi, Lo); + } + + constexpr TWide operator-() const { + const auto sign = THalf(1) << PartBitSize - 1U; + if (TIsSigned::value && !Lo && Hi == sign) + return *this; + return ++~*this; + } + + constexpr TWide& operator++() { + if (!++Lo) ++Hi; + return *this; + } + + constexpr TWide& operator--() { + if (!Lo--) --Hi; + return *this; + } + + constexpr TWide operator++(int) { + const TWide r(*this); + ++*this; + return r; + } + + constexpr TWide operator--(int) { + const TWide r(*this); + --*this; + return r; + } + + constexpr TWide& operator&=(const TWide& rhs) { + Hi &= rhs.Hi; + Lo &= rhs.Lo; + return *this; + } + + constexpr TWide& operator|=(const TWide& rhs) { + Hi |= rhs.Hi; + Lo |= rhs.Lo; + return *this; + } + + constexpr TWide& operator^=(const TWide& rhs) { + Hi ^= rhs.Hi; + Lo ^= rhs.Lo; + return *this; + } + + constexpr TWide& operator+=(const TWide& rhs) { + const auto l = Lo; + Lo += rhs.Lo; + Hi += rhs.Hi; + if (l > Lo) ++Hi; + return *this; + } + + constexpr TWide& operator-=(const TWide& rhs) { + const auto l = Lo; + Lo -= rhs.Lo; + Hi -= rhs.Hi; + if (l < Lo) --Hi; + return *this; + } + + constexpr TWide& operator<<=(const TWide& rhs) { + if (const auto shift = size_t(rhs.Lo) % FullBitSize) { + if (shift < PartBitSize) { + Hi = TPart(Hi) << shift; + Hi |= Lo >> (PartBitSize - shift); + Lo <<= shift; + } else { + Hi = Lo << (shift - PartBitSize); + Lo = 0; + } + } + + return *this; + } + + constexpr TWide& operator>>=(const TWide& rhs) { + if (const auto shift = size_t(rhs.Lo) % FullBitSize) { + if (shift < PartBitSize) { + Lo >>= shift; + Lo |= TPart(Hi) << (PartBitSize - shift); + Hi >>= shift; + } else { + Lo = Hi >> (shift - PartBitSize); + Hi = TIsSigned::value && Hi < 0 ? ~0 : 0; + } + } + + return *this; + } + + constexpr TWide& operator*=(const TWide& rhs) { return *this = Mul(*this, rhs); } + constexpr TWide& operator/=(const TWide& rhs) { return *this = DivMod(*this, rhs).first; } + constexpr TWide& operator%=(const TWide& rhs) { return *this = DivMod(*this, rhs).second; } + + constexpr TWide operator&(const TWide& rhs) const { return TWide(*this) &= rhs; } + constexpr TWide operator|(const TWide& rhs) const { return TWide(*this) |= rhs; } + constexpr TWide operator^(const TWide& rhs) const { return TWide(*this) ^= rhs; } + constexpr TWide operator+(const TWide& rhs) const { return TWide(*this) += rhs; } + constexpr TWide operator-(const TWide& rhs) const { return TWide(*this) -= rhs; } + constexpr TWide operator*(const TWide& rhs) const { return Mul(*this, rhs); } + constexpr TWide operator/(const TWide& rhs) const { return DivMod(*this, rhs).first; } + constexpr TWide operator%(const TWide& rhs) const { return DivMod(*this, rhs).second; } + constexpr TWide operator<<(const TWide& rhs) const { return TWide(*this) <<= rhs; } + constexpr TWide operator>>(const TWide& rhs) const { return TWide(*this) >>= rhs; } + + template + constexpr std::enable_if_t::value, T> operator&(const T rhs) const { return T(*this) & rhs; } + + constexpr bool operator==(const TWide& rhs) const { return std::tie(Hi, Lo) == std::tie(rhs.Hi, rhs.Lo); } + constexpr bool operator!=(const TWide& rhs) const { return std::tie(Hi, Lo) != std::tie(rhs.Hi, rhs.Lo); } + constexpr bool operator>=(const TWide& rhs) const { return std::tie(Hi, Lo) >= std::tie(rhs.Hi, rhs.Lo); } + constexpr bool operator<=(const TWide& rhs) const { return std::tie(Hi, Lo) <= std::tie(rhs.Hi, rhs.Lo); } + constexpr bool operator>(const TWide& rhs) const { return std::tie(Hi, Lo) > std::tie(rhs.Hi, rhs.Lo); } + constexpr bool operator<(const TWide& rhs) const { return std::tie(Hi, Lo) < std::tie(rhs.Hi, rhs.Lo); } + +private: + static constexpr TWide Mul(const TWide& lhs, const TWide& rhs) { + const TPart lq[] = {GetLowerQuarter(lhs.Lo), GetUpperQuarter(lhs.Lo), GetLowerQuarter(lhs.Hi), GetUpperQuarter(lhs.Hi)}; + const TPart rq[] = {GetLowerQuarter(rhs.Lo), GetUpperQuarter(rhs.Lo), GetLowerQuarter(rhs.Hi), GetUpperQuarter(rhs.Hi)}; + + const TPart prod0[] = {TPart(lq[0] * rq[0])}; + const TPart prod1[] = {TPart(lq[0] * rq[1]), TPart(lq[1] * rq[0])}; + const TPart prod2[] = {TPart(lq[0] * rq[2]), TPart(lq[1] * rq[1]), TPart(lq[2] * rq[0])}; + const TPart prod3[] = {TPart(lq[0] * rq[3]), TPart(lq[1] * rq[2]), TPart(lq[2] * rq[1]), TPart(lq[3] * rq[0])}; + + const TPart fourthQ = GetLowerQuarter(prod0[0]); + const TPart thirdQ = GetUpperQuarter(prod0[0]) + + GetLowerQuarter(prod1[0]) + GetLowerQuarter(prod1[1]); + const TPart secondQ = GetUpperQuarter(thirdQ) + + GetUpperQuarter(prod1[0]) + GetUpperQuarter(prod1[1]) + + GetLowerQuarter(prod2[0]) + GetLowerQuarter(prod2[1]) + GetLowerQuarter(prod2[2]); + const TPart firstQ = GetUpperQuarter(secondQ) + + GetUpperQuarter(prod2[0]) + GetUpperQuarter(prod2[1]) + GetUpperQuarter(prod2[2]) + + GetLowerQuarter(prod3[0]) + GetLowerQuarter(prod3[1]) + GetLowerQuarter(prod3[2]) + GetLowerQuarter(prod3[3]); + + return TWide((firstQ << QuarterBitSize) | GetLowerQuarter(secondQ), (thirdQ << QuarterBitSize) | fourthQ); + } + + static constexpr std::pair DivMod(const TWide& lhs, const TWide& rhs) { + const bool nl = TIsSigned::value && lhs.Hi < 0; + const bool nr = TIsSigned::value && rhs.Hi < 0; + + const TUnsigned l = nl ? -lhs : +lhs, r = nr ? -rhs : +rhs; + + TUnsigned div = 0, mod = 0; + + for (auto x = l.GetBits(); x;) { + mod <<= 1; + div <<= 1; + + if (--x < PartBitSize ? l.Lo & (TPart(1U) << x) : l.Hi & (TPart(1U) << x - PartBitSize)) { + ++mod; + } + + if (mod >= r) { + mod -= r; + ++div; + } + } + + if (nl) mod = -mod; + if (nr != nl) div = -div; + + return {div, mod}; + } +}; + +template struct THalfOf; +template<> struct THalfOf { typedef i8 Type; }; +template<> struct THalfOf { typedef ui8 Type; }; +template<> struct THalfOf { typedef i16 Type; }; +template<> struct THalfOf { typedef ui16 Type; }; +template<> struct THalfOf { typedef i32 Type; }; +template<> struct THalfOf { typedef ui32 Type; }; +#ifndef _win_ +template<> struct THalfOf { typedef i64 Type; }; +template<> struct THalfOf { typedef ui64 Type; }; +#endif +template struct THalfOf {}; + +template struct TPairOf; +template<> struct TPairOf { typedef i16 Type; }; +template<> struct TPairOf { typedef ui16 Type; }; +template<> struct TPairOf { typedef i32 Type; }; +template<> struct TPairOf { typedef ui32 Type; }; +template<> struct TPairOf { typedef i64 Type; }; +template<> struct TPairOf { typedef ui64 Type; }; +#ifndef _win_ +template<> struct TPairOf { typedef i128_t Type; }; +template<> struct TPairOf { typedef ui128_t Type; }; +#endif +template struct TPairOf {}; + +} diff --git a/ydb/public/sdk/cpp/src/library/grpc/client/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/grpc/client/CMakeLists.txt new file mode 100644 index 000000000000..aef53794904a --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/grpc/client/CMakeLists.txt @@ -0,0 +1,15 @@ +_ydb_sdk_add_library(grpc-client) + +target_link_libraries(grpc-client + PUBLIC + yutil + gRPC::grpc++ +) + +target_sources(grpc-client PRIVATE + grpc_client_low.cpp +) + +target_compile_definitions(grpc-client PRIVATE YDB_DISABLE_GRPC_SOCKET_MUTATOR) + +_ydb_sdk_install_targets(TARGETS grpc-client) diff --git a/ydb/library/grpc/client/grpc_client_low.cpp b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.cpp similarity index 95% rename from ydb/library/grpc/client/grpc_client_low.cpp rename to ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.cpp index 687b4b6e45a9..a2b959c31176 100644 --- a/ydb/library/grpc/client/grpc_client_low.cpp +++ b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.cpp @@ -1,10 +1,9 @@ #include "grpc_client_low.h" -#include -#include + +#include #include -#include #include #include @@ -15,6 +14,12 @@ #include #endif +#if !defined(YDB_DISABLE_GRPC_SOCKET_MUTATOR) +#include +#endif + +#include + namespace NYdbGrpc { void EnableGRpcTracing() { @@ -31,6 +36,7 @@ void EnableGRpcTracing() { gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); } +#if !defined(YDB_DISABLE_GRPC_SOCKET_MUTATOR) class TGRpcKeepAliveSocketMutator : public grpc_socket_mutator { public: TGRpcKeepAliveSocketMutator(int idle, int count, int interval) @@ -51,20 +57,20 @@ class TGRpcKeepAliveSocketMutator : public grpc_socket_mutator { } bool SetOption(int fd) { if (!SetOption(fd, SOL_SOCKET, SO_KEEPALIVE, 1)) { - Cerr << Sprintf("Failed to set SO_KEEPALIVE option: %s", strerror(errno)) << Endl; + std::cerr << std::format("Failed to set SO_KEEPALIVE option: {}", strerror(errno)) << std::endl; return false; } #ifdef _linux_ if (Idle_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPIDLE, Idle_)) { - Cerr << Sprintf("Failed to set TCP_KEEPIDLE option: %s", strerror(errno)) << Endl; + std::cerr << std::format("Failed to set TCP_KEEPIDLE option: {}", strerror(errno)) << std::endl; return false; } if (Count_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPCNT, Count_)) { - Cerr << Sprintf("Failed to set TCP_KEEPCNT option: %s", strerror(errno)) << Endl; + std::cerr << std::format("Failed to set TCP_KEEPCNT option: {}", strerror(errno)) << std::endl; return false; } if (Interval_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPINTVL, Interval_)) { - Cerr << Sprintf("Failed to set TCP_KEEPINTVL option: %s", strerror(errno)) << Endl; + std::cerr << std::format("Failed to set TCP_KEEPINTVL option: {}", strerror(errno)) << std::endl; return false; } #endif @@ -102,6 +108,7 @@ grpc_socket_mutator_vtable TGRpcKeepAliveSocketMutator::VTable = &TGRpcKeepAliveSocketMutator::Destroy, &TGRpcKeepAliveSocketMutator::Mutate2 }; +#endif TChannelPool::TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, const TDuration& expireTime) : TcpKeepAliveSettings_(tcpKeepAliveSettings) @@ -110,7 +117,7 @@ TChannelPool::TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, co {} void TChannelPool::GetStubsHolderLocked( - const TString& channelId, + const std::string& channelId, const TGRpcClientConfig& config, std::function cb) { @@ -148,7 +155,7 @@ void TChannelPool::GetStubsHolderLocked( } } -void TChannelPool::DeleteChannel(const TString& channelId) { +void TChannelPool::DeleteChannel(const std::string& channelId) { std::unique_lock writeLock(RWMutex_); auto poolIt = Pool_.find(channelId); if (poolIt != Pool_.end()) { @@ -166,7 +173,7 @@ void TChannelPool::DeleteExpiredStubsHolders() { LastUsedQueue_.erase(LastUsedQueue_.begin(), lastExpired); } -void TChannelPool::EraseFromQueueByTime(const TInstant& lastUseTime, const TString& channelId) { +void TChannelPool::EraseFromQueueByTime(const TInstant& lastUseTime, const std::string& channelId) { auto [begin, end] = LastUsedQueue_.equal_range(lastUseTime); auto pos = std::find_if(begin, end, [&](auto a){return a.second == channelId;}); Y_ABORT_UNLESS(pos != LastUsedQueue_.end(), "data corruption at TChannelPool"); @@ -305,7 +312,7 @@ class TGRpcClientLow::TContextImpl final } // Call directly subscribed callbacks - if (callbacks) { + if (!callbacks.empty()) { RunCallbacksNoExcept(callbacks); } @@ -457,7 +464,7 @@ void TGRpcClientLow::Stop(bool wait) { void TGRpcClientLow::StopInternal(bool silent) { bool shutdown; - TVector cancelQueue; + std::vector cancelQueue; { std::unique_lock guard(Mtx_); @@ -582,6 +589,7 @@ void TGRpcClientLow::ForgetContext(TContextImpl* context) { } grpc_socket_mutator* NImpl::CreateGRpcKeepAliveSocketMutator(const TTcpKeepAliveSettings& TcpKeepAliveSettings_) { +#if !defined(YDB_DISABLE_GRPC_SOCKET_MUTATOR) TGRpcKeepAliveSocketMutator* mutator = nullptr; if (TcpKeepAliveSettings_.Enabled) { mutator = new TGRpcKeepAliveSocketMutator( @@ -591,6 +599,8 @@ grpc_socket_mutator* NImpl::CreateGRpcKeepAliveSocketMutator(const TTcpKeepAlive ); } return mutator; +#endif + return nullptr; } } // namespace NGRpc diff --git a/ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.h b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.h new file mode 100644 index 000000000000..72d355783b38 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_client_low.h @@ -0,0 +1,1440 @@ +#pragma once + +#include "grpc_common.h" + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This file contains low level logic for grpc + * This file should not be used in high level code without special reason + */ +namespace NYdbGrpc { + +const size_t DEFAULT_NUM_THREADS = 2; + +//////////////////////////////////////////////////////////////////////////////// + +void EnableGRpcTracing(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TTcpKeepAliveSettings { + bool Enabled; + size_t Idle; + size_t Count; + size_t Interval; +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Common interface used to execute action from grpc cq routine +class IQueueClientEvent { +public: + virtual ~IQueueClientEvent() = default; + + //! Execute an action defined by implementation + virtual bool Execute(bool ok) = 0; + + //! Finish and destroy event + virtual void Destroy() = 0; +}; + +// Implementation of IQueueClientEvent that reduces allocations +template +class TQueueClientFixedEvent : private IQueueClientEvent { + using TCallback = void (TSelf::*)(bool); + +public: + TQueueClientFixedEvent(TSelf* self, TCallback callback) + : Self(self) + , Callback(callback) + { } + + IQueueClientEvent* Prepare() { + Self->Ref(); + return this; + } + +private: + bool Execute(bool ok) override { + ((*Self).*Callback)(ok); + return false; + } + + void Destroy() override { + Self->UnRef(); + } + +private: + TSelf* const Self; + TCallback const Callback; +}; + +class IQueueClientContext; +using IQueueClientContextPtr = std::shared_ptr; + +// Provider of IQueueClientContext instances +class IQueueClientContextProvider { +public: + virtual ~IQueueClientContextProvider() = default; + + virtual IQueueClientContextPtr CreateContext() = 0; +}; + +// Activity context for a low-level client +class IQueueClientContext : public IQueueClientContextProvider { +public: + virtual ~IQueueClientContext() = default; + + //! Returns CompletionQueue associated with the client + virtual grpc::CompletionQueue* CompletionQueue() = 0; + + //! Returns true if context has been cancelled + virtual bool IsCancelled() const = 0; + + //! Tries to cancel context, calling all registered callbacks + virtual bool Cancel() = 0; + + //! Subscribes callback to cancellation + // + // Note there's no way to unsubscribe, if subscription is temporary + // make sure you create a new context with CreateContext and release + // it as soon as it's no longer needed. + virtual void SubscribeCancel(std::function callback) = 0; + + //! Subscribes callback to cancellation + // + // This alias is for compatibility with older code. + void SubscribeStop(std::function callback) { + SubscribeCancel(std::move(callback)); + } +}; + +// Represents grpc status and error message string +struct TGrpcStatus { + std::string Msg; + std::string Details; + int GRpcStatusCode; + bool InternalError; + std::multimap ServerTrailingMetadata; + + TGrpcStatus() + : GRpcStatusCode(grpc::StatusCode::OK) + , InternalError(false) + { } + + TGrpcStatus(std::string msg, int statusCode, bool internalError) + : Msg(std::move(msg)) + , GRpcStatusCode(statusCode) + , InternalError(internalError) + { } + + TGrpcStatus(grpc::StatusCode status, std::string msg, std::string details = {}) + : Msg(std::move(msg)) + , Details(std::move(details)) + , GRpcStatusCode(status) + , InternalError(false) + { } + + TGrpcStatus(const grpc::Status& status) + : TGrpcStatus(status.error_code(), std::string(status.error_message()), std::string(status.error_details())) + { } + + TGrpcStatus& operator=(const grpc::Status& status) { + Msg = std::string(status.error_message()); + Details = std::string(status.error_details()); + GRpcStatusCode = status.error_code(); + InternalError = false; + return *this; + } + + static TGrpcStatus Internal(std::string msg) { + return { std::move(msg), -1, true }; + } + + bool Ok() const { + return !InternalError && GRpcStatusCode == grpc::StatusCode::OK; + } + + TStringBuilder ToDebugString() const { + TStringBuilder ret; + ret << "gRpcStatusCode: " << GRpcStatusCode; + if(!Ok()) + ret << ", Msg: " << Msg << ", Details: " << Details << ", InternalError: " << InternalError; + return ret; + } +}; + +bool inline IsGRpcStatusGood(const TGrpcStatus& status) { + return status.Ok(); +} + +// Response callback type - this callback will be called when request is finished +// (or after getting each chunk in case of streaming mode) +template +using TResponseCallback = std::function; + +template +using TAdvancedResponseCallback = std::function; + +// Call associated metadata +struct TCallMeta { + std::shared_ptr CallCredentials; + std::vector> Aux; + std::variant Timeout; // timeout as duration from now or time point in future +}; + +class TGRpcRequestProcessorCommon { +protected: + void ApplyMeta(const TCallMeta& meta) { + for (const auto& rec : meta.Aux) { + Context.AddMetadata(NYdb::TStringType{rec.first}, NYdb::TStringType{rec.second}); + } + if (meta.CallCredentials) { + Context.set_credentials(meta.CallCredentials); + } + if (const TDuration* timeout = std::get_if(&meta.Timeout)) { + if (*timeout) { + auto deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(timeout->MicroSeconds(), GPR_TIMESPAN)); + Context.set_deadline(deadline); + } + } else if (const TInstant* deadline = std::get_if(&meta.Timeout)) { + if (*deadline) { + Context.set_deadline(gpr_time_from_micros(deadline->MicroSeconds(), GPR_CLOCK_MONOTONIC)); + } + } + } + + void GetInitialMetadata(std::unordered_multimap* metadata) { + for (const auto& [key, value] : Context.GetServerInitialMetadata()) { + metadata->emplace( + std::string(key.begin(), key.end()), + std::string(value.begin(), value.end()) + ); + } + } + + grpc::Status Status; + grpc::ClientContext Context; + std::shared_ptr LocalContext; +}; + +template +class TSimpleRequestProcessor + : public TThrRefBase + , public IQueueClientEvent + , public TGRpcRequestProcessorCommon { + using TAsyncReaderPtr = std::unique_ptr>; + template friend class TServiceConnection; +public: + using TPtr = TIntrusivePtr; + using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); + + explicit TSimpleRequestProcessor(TResponseCallback&& callback) + : Callback_(std::move(callback)) + { } + + ~TSimpleRequestProcessor() { + if (!Replied_ && Callback_) { + Callback_(TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + Callback_ = nullptr; // free resources as early as possible + } + } + + bool Execute(bool ok) override { + { + std::unique_lock guard(Mutex_); + LocalContext.reset(); + } + TGrpcStatus status; + if (ok) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + Replied_ = true; + Callback_(std::move(status), std::move(Reply_)); + Callback_ = nullptr; // free resources as early as possible + return false; + } + + void Destroy() override { + UnRef(); + } + +private: + IQueueClientEvent* FinishedEvent() { + Ref(); + return this; + } + + void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + Replied_ = true; + Callback_(TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + Callback_ = nullptr; + return; + } + { + std::unique_lock guard(Mutex_); + LocalContext = context; + Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); + Reader_->Finish(&Reply_, &Status, FinishedEvent()); + } + context->SubscribeStop([self = TPtr(this)] { + self->Stop(); + }); + } + + void Stop() { + Context.TryCancel(); + } + + TResponseCallback Callback_; + TResponse Reply_; + std::mutex Mutex_; + TAsyncReaderPtr Reader_; + + bool Replied_ = false; +}; + +template +class TAdvancedRequestProcessor + : public TThrRefBase + , public IQueueClientEvent + , public TGRpcRequestProcessorCommon { + using TAsyncReaderPtr = std::unique_ptr>; + template friend class TServiceConnection; +public: + using TPtr = TIntrusivePtr; + using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); + + explicit TAdvancedRequestProcessor(TAdvancedResponseCallback&& callback) + : Callback_(std::move(callback)) + { } + + ~TAdvancedRequestProcessor() { + if (!Replied_ && Callback_) { + Callback_(Context, TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + Callback_ = nullptr; // free resources as early as possible + } + } + + bool Execute(bool ok) override { + { + std::unique_lock guard(Mutex_); + LocalContext.reset(); + } + TGrpcStatus status; + if (ok) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + Replied_ = true; + Callback_(Context, std::move(status), std::move(Reply_)); + Callback_ = nullptr; // free resources as early as possible + return false; + } + + void Destroy() override { + UnRef(); + } + +private: + IQueueClientEvent* FinishedEvent() { + Ref(); + return this; + } + + void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + Replied_ = true; + Callback_(Context, TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + Callback_ = nullptr; + return; + } + { + std::unique_lock guard(Mutex_); + LocalContext = context; + Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); + Reader_->Finish(&Reply_, &Status, FinishedEvent()); + } + context->SubscribeStop([self = TPtr(this)] { + self->Stop(); + }); + } + + void Stop() { + Context.TryCancel(); + } + + TAdvancedResponseCallback Callback_; + TResponse Reply_; + std::mutex Mutex_; + TAsyncReaderPtr Reader_; + + bool Replied_ = false; +}; + +class IStreamRequestCtrl : public TThrRefBase { +public: + using TPtr = TIntrusivePtr; + + /** + * Asynchronously cancel the request + */ + virtual void Cancel() = 0; +}; + +template +class IStreamRequestReadProcessor : public IStreamRequestCtrl { +public: + using TPtr = TIntrusivePtr; + using TReadCallback = std::function; + + /** + * Scheduled initial server metadata read from the stream + */ + virtual void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) = 0; + + /** + * Scheduled response read from the stream + * Callback will be called with the status if it failed + * Only one Read or Finish call may be active at a time + */ + virtual void Read(TResponse* response, TReadCallback callback) = 0; + + /** + * Stop reading and gracefully finish the stream + * Only one Read or Finish call may be active at a time + */ + virtual void Finish(TReadCallback callback) = 0; + + /** + * Additional callback to be called when stream has finished + */ + virtual void AddFinishedCallback(TReadCallback callback) = 0; +}; + +template +class IStreamRequestReadWriteProcessor : public IStreamRequestReadProcessor { +public: + using TPtr = TIntrusivePtr; + using TWriteCallback = std::function; + + /** + * Scheduled request write to the stream + */ + virtual void Write(TRequest&& request, TWriteCallback callback = { }) = 0; +}; + +class TGRpcKeepAliveSocketMutator; + +namespace NImpl { +grpc_socket_mutator* CreateGRpcKeepAliveSocketMutator(const TTcpKeepAliveSettings& TcpKeepAliveSettings_); +} // NImpl + +// Class to hold stubs allocated on channel. +// It is poor documented part of grpc. See KIKIMR-6109 and comment to this commit + +// Stub holds shared_ptr, so we can destroy this holder even if +// request processor using stub +class TStubsHolder : public TNonCopyable { + using TypeInfoRef = std::reference_wrapper; + + struct THasher { + std::size_t operator()(TypeInfoRef code) const { + return code.get().hash_code(); + } + }; + + struct TEqualTo { + bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { + return lhs.get() == rhs.get(); + } + }; +public: + TStubsHolder(std::shared_ptr channel) + : ChannelInterface_(channel) + {} + + // Returns true if channel can't be used to perform request now + bool IsChannelBroken() const { + auto state = ChannelInterface_->GetState(false); + return state == GRPC_CHANNEL_SHUTDOWN || + state == GRPC_CHANNEL_TRANSIENT_FAILURE; + } + + template + std::shared_ptr GetOrCreateStub() { + const auto& stubId = typeid(TStub); + { + std::shared_lock readGuard(RWMutex_); + const auto it = Stubs_.find(stubId); + if (it != Stubs_.end()) { + return std::static_pointer_cast(it->second); + } + } + { + std::unique_lock writeGuard(RWMutex_); + auto it = Stubs_.emplace(stubId, nullptr); + if (!it.second) { + return std::static_pointer_cast(it.first->second); + } else { + it.first->second = std::make_shared(ChannelInterface_); + return std::static_pointer_cast(it.first->second); + } + } + } + + const TInstant& GetLastUseTime() const { + return LastUsed_; + } + + void SetLastUseTime(const TInstant& time) { + LastUsed_ = time; + } +private: + TInstant LastUsed_ = Now(); + std::shared_mutex RWMutex_; + std::unordered_map, THasher, TEqualTo> Stubs_; + std::shared_ptr ChannelInterface_; +}; + +class TChannelPool { +public: + TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, const TDuration& expireTime = TDuration::Minutes(6)); + //Allows to CreateStub from TStubsHolder under lock + //The callback will be called just during GetStubsHolderLocked call + void GetStubsHolderLocked(const std::string& channelId, const TGRpcClientConfig& config, std::function cb); + void DeleteChannel(const std::string& channelId); + void DeleteExpiredStubsHolders(); +private: + std::shared_mutex RWMutex_; + std::unordered_map Pool_; + std::multimap LastUsedQueue_; + [[maybe_unused]] TTcpKeepAliveSettings TcpKeepAliveSettings_; + TDuration ExpireTime_; + TDuration UpdateReUseTime_; + void EraseFromQueueByTime(const TInstant& lastUseTime, const std::string& channelId); +}; + +template +using TStreamReaderCallback = std::function::TPtr)>; + +template +class TStreamRequestReadProcessor + : public IStreamRequestReadProcessor + , public TGRpcRequestProcessorCommon { + template friend class TServiceConnection; +public: + using TSelf = TStreamRequestReadProcessor; + using TAsyncReaderPtr = std::unique_ptr>; + using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*, void*); + using TReaderCallback = TStreamReaderCallback; + using TPtr = TIntrusivePtr; + using TBase = IStreamRequestReadProcessor; + using TReadCallback = typename TBase::TReadCallback; + + explicit TStreamRequestReadProcessor(TReaderCallback&& callback) + : Callback(std::move(callback)) + { + Y_ABORT_UNLESS(Callback, "Missing connected callback"); + } + + void Cancel() override { + Context.TryCancel(); + + { + std::unique_lock guard(Mutex); + Cancelled = true; + if (Started && !ReadFinished) { + if (!ReadActive) { + ReadFinished = true; + } + if (ReadFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } + } + } + + void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished && !HasInitialMetadata) { + ReadActive = true; + ReadCallback = std::move(callback); + InitialMetadata = metadata; + if (!ReadFinished) { + Stream->ReadInitialMetadata(OnReadDoneTag.Prepare()); + } + return; + } + if (!HasInitialMetadata) { + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } else { + GetInitialMetadata(metadata); + } + } + + callback(std::move(status)); + } + + void Read(TResponse* message, TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + ReadCallback = std::move(callback); + if (!ReadFinished) { + Stream->Read(message, OnReadDoneTag.Prepare()); + } + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + } + + callback(std::move(status)); + } + + void Finish(TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + FinishCallback = std::move(callback); + if (!ReadFinished) { + ReadFinished = true; + } + Stream->Finish(&Status, OnFinishedTag.Prepare()); + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + callback(std::move(status)); + } + + void AddFinishedCallback(TReadCallback callback) override { + Y_ABORT_UNLESS(callback, "Unexpected empty callback"); + + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + if (!Finished) { + FinishedCallbacks.emplace_back().swap(callback); + return; + } + + if (FinishedOk) { + status = Status; + } else if (Cancelled) { + status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + callback(std::move(status)); + } + +private: + void Start(TStub& stub, const TRequest& request, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + auto callback = std::move(Callback); + TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); + callback(std::move(status), nullptr); + return; + } + + { + std::unique_lock guard(Mutex); + LocalContext = context; + Stream = (stub.*asyncRequest)(&Context, request, context->CompletionQueue(), OnStartDoneTag.Prepare()); + } + + context->SubscribeStop([self = TPtr(this)] { + self->Cancel(); + }); + } + + void OnReadDone(bool ok) { + TGrpcStatus status; + TReadCallback callback; + std::unordered_multimap* initialMetadata = nullptr; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(ReadActive, "Unexpected Read done callback"); + Y_ABORT_UNLESS(!ReadFinished, "Unexpected ReadFinished flag"); + + if (!ok || Cancelled) { + ReadFinished = true; + + Stream->Finish(&Status, OnFinishedTag.Prepare()); + if (!ok) { + // Keep ReadActive=true, so callback is called + // after the call is finished with an error + return; + } + } + + callback = std::move(ReadCallback); + ReadCallback = nullptr; + ReadActive = false; + initialMetadata = InitialMetadata; + InitialMetadata = nullptr; + HasInitialMetadata = true; + } + + if (initialMetadata) { + GetInitialMetadata(initialMetadata); + } + + callback(std::move(status)); + } + + void OnStartDone(bool ok) { + TReaderCallback callback; + + { + std::unique_lock guard(Mutex); + Started = true; + if (!ok || Cancelled) { + ReadFinished = true; + Stream->Finish(&Status, OnFinishedTag.Prepare()); + return; + } + callback = std::move(Callback); + Callback = nullptr; + } + + callback({ }, typename TBase::TPtr(this)); + } + + void OnFinished(bool ok) { + TGrpcStatus status; + std::vector finishedCallbacks; + TReaderCallback startCallback; + TReadCallback readCallback; + TReadCallback finishCallback; + + { + std::unique_lock guard(Mutex); + + Finished = true; + FinishedOk = ok; + LocalContext.reset(); + + if (ok) { + status = Status; + } else if (Cancelled) { + status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + + finishedCallbacks.swap(FinishedCallbacks); + + if (Callback) { + Y_ABORT_UNLESS(!ReadActive); + startCallback = std::move(Callback); + Callback = nullptr; + } else if (ReadActive) { + if (ReadCallback) { + readCallback = std::move(ReadCallback); + ReadCallback = nullptr; + } else { + finishCallback = std::move(FinishCallback); + FinishCallback = nullptr; + } + ReadActive = false; + } + } + + for (auto& finishedCallback : finishedCallbacks) { + auto statusCopy = status; + finishedCallback(std::move(statusCopy)); + } + + if (startCallback) { + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); + } + startCallback(std::move(status), nullptr); + } else if (readCallback) { + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + for (const auto& [name, value] : Context.GetServerTrailingMetadata()) { + status.ServerTrailingMetadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } + } + readCallback(std::move(status)); + } else if (finishCallback) { + finishCallback(std::move(status)); + } + } + + TReaderCallback Callback; + TAsyncReaderPtr Stream; + using TFixedEvent = TQueueClientFixedEvent; + std::mutex Mutex; + TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; + TFixedEvent OnStartDoneTag = { this, &TSelf::OnStartDone }; + TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; + + TReadCallback ReadCallback; + TReadCallback FinishCallback; + std::vector FinishedCallbacks; + std::unordered_multimap* InitialMetadata = nullptr; + bool Started = false; + bool HasInitialMetadata = false; + bool ReadActive = false; + bool ReadFinished = false; + bool Finished = false; + bool Cancelled = false; + bool FinishedOk = false; +}; + +template +using TStreamConnectedCallback = std::function::TPtr)>; + +template +class TStreamRequestReadWriteProcessor + : public IStreamRequestReadWriteProcessor + , public TGRpcRequestProcessorCommon { +public: + using TSelf = TStreamRequestReadWriteProcessor; + using TBase = IStreamRequestReadWriteProcessor; + using TPtr = TIntrusivePtr; + using TConnectedCallback = TStreamConnectedCallback; + using TReadCallback = typename TBase::TReadCallback; + using TWriteCallback = typename TBase::TWriteCallback; + using TAsyncReaderWriterPtr = std::unique_ptr>; + using TAsyncRequest = TAsyncReaderWriterPtr (TStub::*)(grpc::ClientContext*, grpc::CompletionQueue*, void*); + + explicit TStreamRequestReadWriteProcessor(TConnectedCallback&& callback) + : ConnectedCallback(std::move(callback)) + { + Y_ABORT_UNLESS(ConnectedCallback, "Missing connected callback"); + } + + void Cancel() override { + Context.TryCancel(); + + { + std::unique_lock guard(Mutex); + Cancelled = true; + if (Started && !(ReadFinished && WriteFinished)) { + if (!ReadActive) { + ReadFinished = true; + } + if (!WriteActive) { + WriteFinished = true; + } + if (ReadFinished && WriteFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } + } + } + + void Write(TRequest&& request, TWriteCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + if (Cancelled || ReadFinished || WriteFinished) { + status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); + } else if (WriteActive) { + auto& item = WriteQueue.emplace_back(); + item.Callback.swap(callback); + item.Request.Swap(&request); + } else { + WriteActive = true; + WriteCallback.swap(callback); + Stream->Write(request, OnWriteDoneTag.Prepare()); + } + } + + if (!status.Ok() && callback) { + callback(std::move(status)); + } + } + + void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished && !HasInitialMetadata) { + ReadActive = true; + ReadCallback = std::move(callback); + InitialMetadata = metadata; + if (!ReadFinished) { + Stream->ReadInitialMetadata(OnReadDoneTag.Prepare()); + } + return; + } + if (!HasInitialMetadata) { + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } else { + GetInitialMetadata(metadata); + } + } + + callback(std::move(status)); + } + + void Read(TResponse* message, TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + ReadCallback = std::move(callback); + if (!ReadFinished) { + Stream->Read(message, OnReadDoneTag.Prepare()); + } + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + } + + callback(std::move(status)); + } + + void Finish(TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + FinishCallback = std::move(callback); + if (!ReadFinished) { + ReadFinished = true; + if (!WriteActive) { + WriteFinished = true; + } + if (WriteFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + callback(std::move(status)); + } + + void AddFinishedCallback(TReadCallback callback) override { + Y_ABORT_UNLESS(callback, "Unexpected empty callback"); + + TGrpcStatus status; + + { + std::unique_lock guard(Mutex); + if (!Finished) { + FinishedCallbacks.emplace_back().swap(callback); + return; + } + + if (FinishedOk) { + status = Status; + } else if (Cancelled) { + status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + callback(std::move(status)); + } + +private: + template friend class TServiceConnection; + + void Start(TStub& stub, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + auto callback = std::move(ConnectedCallback); + TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); + callback(std::move(status), nullptr); + return; + } + + { + std::unique_lock guard(Mutex); + LocalContext = context; + Stream = (stub.*asyncRequest)(&Context, context->CompletionQueue(), OnConnectedTag.Prepare()); + } + + context->SubscribeStop([self = TPtr(this)] { + self->Cancel(); + }); + } + +private: + void OnConnected(bool ok) { + TConnectedCallback callback; + + { + std::unique_lock guard(Mutex); + Started = true; + if (!ok || Cancelled) { + ReadFinished = true; + WriteFinished = true; + Stream->Finish(&Status, OnFinishedTag.Prepare()); + return; + } + + callback = std::move(ConnectedCallback); + ConnectedCallback = nullptr; + } + + callback({ }, typename TBase::TPtr(this)); + } + + void OnReadDone(bool ok) { + TGrpcStatus status; + TReadCallback callback; + std::unordered_multimap* initialMetadata = nullptr; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(ReadActive, "Unexpected Read done callback"); + Y_ABORT_UNLESS(!ReadFinished, "Unexpected ReadFinished flag"); + + if (!ok || Cancelled) { + ReadFinished = true; + if (!WriteActive) { + WriteFinished = true; + } + if (WriteFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + if (!ok) { + // Keep ReadActive=true, so callback is called + // after the call is finished with an error + return; + } + } + + callback = std::move(ReadCallback); + ReadCallback = nullptr; + ReadActive = false; + initialMetadata = InitialMetadata; + InitialMetadata = nullptr; + HasInitialMetadata = true; + } + + if (initialMetadata) { + GetInitialMetadata(initialMetadata); + } + + callback(std::move(status)); + } + + void OnWriteDone(bool ok) { + TWriteCallback okCallback; + + { + std::unique_lock guard(Mutex); + Y_ABORT_UNLESS(WriteActive, "Unexpected Write done callback"); + Y_ABORT_UNLESS(!WriteFinished, "Unexpected WriteFinished flag"); + + if (ok) { + okCallback.swap(WriteCallback); + } else if (WriteCallback) { + // Put callback back on the queue until OnFinished + auto& item = WriteQueue.emplace_front(); + item.Callback.swap(WriteCallback); + } + + if (!ok || Cancelled) { + WriteActive = false; + WriteFinished = true; + if (ReadFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } else if (!WriteQueue.empty()) { + WriteCallback.swap(WriteQueue.front().Callback); + Stream->Write(WriteQueue.front().Request, OnWriteDoneTag.Prepare()); + WriteQueue.pop_front(); + } else { + WriteActive = false; + if (ReadFinished) { + WriteFinished = true; + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } + } + + if (okCallback) { + okCallback(TGrpcStatus()); + } + } + + void OnFinished(bool ok) { + TGrpcStatus status; + std::deque writesDropped; + std::vector finishedCallbacks; + TConnectedCallback connectedCallback; + TReadCallback readCallback; + TReadCallback finishCallback; + + { + std::unique_lock guard(Mutex); + Finished = true; + FinishedOk = ok; + LocalContext.reset(); + + if (ok) { + status = Status; + } else if (Cancelled) { + status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + + writesDropped.swap(WriteQueue); + finishedCallbacks.swap(FinishedCallbacks); + + if (ConnectedCallback) { + Y_ABORT_UNLESS(!ReadActive); + connectedCallback = std::move(ConnectedCallback); + ConnectedCallback = nullptr; + } else if (ReadActive) { + if (ReadCallback) { + readCallback = std::move(ReadCallback); + ReadCallback = nullptr; + } else { + finishCallback = std::move(FinishCallback); + FinishCallback = nullptr; + } + ReadActive = false; + } + } + + for (auto& item : writesDropped) { + if (item.Callback) { + TGrpcStatus writeStatus = status; + if (writeStatus.Ok()) { + writeStatus = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); + } + item.Callback(std::move(writeStatus)); + } + } + + for (auto& finishedCallback : finishedCallbacks) { + TGrpcStatus statusCopy = status; + finishedCallback(std::move(statusCopy)); + } + + if (connectedCallback) { + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); + } + connectedCallback(std::move(status), nullptr); + } else if (readCallback) { + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + for (const auto& [name, value] : Context.GetServerTrailingMetadata()) { + status.ServerTrailingMetadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } + } + readCallback(std::move(status)); + } else if (finishCallback) { + finishCallback(std::move(status)); + } + } + +private: + struct TWriteItem { + TWriteCallback Callback; + TRequest Request; + }; + +private: + using TFixedEvent = TQueueClientFixedEvent; + + TFixedEvent OnConnectedTag = { this, &TSelf::OnConnected }; + TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; + TFixedEvent OnWriteDoneTag = { this, &TSelf::OnWriteDone }; + TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; + +private: + std::mutex Mutex; + TAsyncReaderWriterPtr Stream; + TConnectedCallback ConnectedCallback; + TReadCallback ReadCallback; + TReadCallback FinishCallback; + std::vector FinishedCallbacks; + std::deque WriteQueue; + TWriteCallback WriteCallback; + std::unordered_multimap* InitialMetadata = nullptr; + bool Started = false; + bool HasInitialMetadata = false; + bool ReadActive = false; + bool ReadFinished = false; + bool WriteActive = false; + bool WriteFinished = false; + bool Finished = false; + bool Cancelled = false; + bool FinishedOk = false; +}; + +class TGRpcClientLow; + +template +class TServiceConnection { + using TStub = typename TGRpcService::Stub; + friend class TGRpcClientLow; + +public: + /* + * Start simple request + */ + template + void DoRequest(const TRequest& request, + TResponseCallback callback, + typename TSimpleRequestProcessor::TAsyncRequest asyncRequest, + const TCallMeta& metas = { }, + IQueueClientContextProvider* provider = nullptr) + { + auto processor = MakeIntrusive>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); + } + + /* + * Start simple request + */ + template + void DoAdvancedRequest(const TRequest& request, + TAdvancedResponseCallback callback, + typename TAdvancedRequestProcessor::TAsyncRequest asyncRequest, + const TCallMeta& metas = { }, + IQueueClientContextProvider* provider = nullptr) + { + auto processor = MakeIntrusive>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); + } + + /* + * Start bidirectional streamming + */ + template + void DoStreamRequest(TStreamConnectedCallback callback, + typename TStreamRequestReadWriteProcessor::TAsyncRequest asyncRequest, + const TCallMeta& metas = { }, + IQueueClientContextProvider* provider = nullptr) + { + auto processor = MakeIntrusive>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, std::move(asyncRequest), provider ? provider : Provider_); + } + + /* + * Start streaming response reading (one request, many responses) + */ + template + void DoStreamRequest(const TRequest& request, + TStreamReaderCallback callback, + typename TStreamRequestReadProcessor::TAsyncRequest asyncRequest, + const TCallMeta& metas = { }, + IQueueClientContextProvider* provider = nullptr) + { + auto processor = MakeIntrusive>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, request, std::move(asyncRequest), provider ? provider : Provider_); + } + +private: + TServiceConnection(std::shared_ptr ci, + IQueueClientContextProvider* provider) + : Stub_(TGRpcService::NewStub(ci)) + , Provider_(provider) + { + Y_ABORT_UNLESS(Provider_, "Connection does not have a queue provider"); + } + + TServiceConnection(TStubsHolder& holder, + IQueueClientContextProvider* provider) + : Stub_(holder.GetOrCreateStub()) + , Provider_(provider) + { + Y_ABORT_UNLESS(Provider_, "Connection does not have a queue provider"); + } + + std::shared_ptr Stub_; + IQueueClientContextProvider* Provider_; +}; + +class TGRpcClientLow + : public IQueueClientContextProvider +{ + class TContextImpl; + friend class TContextImpl; + + enum ECqState : int { + WORKING = 0, + STOP_SILENT = 1, + STOP_EXPLICIT = 2, + }; + +public: + explicit TGRpcClientLow(size_t numWorkerThread = DEFAULT_NUM_THREADS, bool useCompletionQueuePerThread = false); + ~TGRpcClientLow(); + + // Tries to stop all currently running requests (via their stop callbacks) + // Will shutdown CQ and drain events once all requests have finished + // No new requests may be started after this call + void Stop(bool wait = false); + + // Waits until all currently running requests finish execution + void WaitIdle(); + + inline bool IsStopping() const { + switch (GetCqState()) { + case WORKING: + return false; + case STOP_SILENT: + case STOP_EXPLICIT: + return true; + } + + Y_UNREACHABLE(); + } + + IQueueClientContextPtr CreateContext() override; + + template + std::unique_ptr> CreateGRpcServiceConnection(const TGRpcClientConfig& config) { + return std::unique_ptr>(new TServiceConnection(CreateChannelInterface(config), this)); + } + + template + std::unique_ptr> CreateGRpcServiceConnection(const TGRpcClientConfig& config, const TTcpKeepAliveSettings& keepAlive) { + auto mutator = NImpl::CreateGRpcKeepAliveSocketMutator(keepAlive); + // will be destroyed inside grpc + return std::unique_ptr>(new TServiceConnection(CreateChannelInterface(config, mutator), this)); + } + + template + std::unique_ptr> CreateGRpcServiceConnection(TStubsHolder& holder) { + return std::unique_ptr>(new TServiceConnection(holder, this)); + } + + // Tests only, not thread-safe + void AddWorkerThreadForTest(); + +private: + using IThreadRef = std::unique_ptr; + using CompletionQueueRef = std::unique_ptr; + void Init(size_t numWorkerThread); + + inline ECqState GetCqState() const { + return static_cast(CqState_.load()); + } + + inline void SetCqState(ECqState state) { + CqState_.store(state); + } + + void StopInternal(bool silent); + void WaitInternal(); + + void ForgetContext(TContextImpl* context); + +private: + bool UseCompletionQueuePerThread_; + std::vector CQS_; + std::vector WorkerThreads_; + std::atomic CqState_ = -1; + + std::mutex Mtx_; + std::condition_variable ContextsEmpty_; + std::unordered_set Contexts_; + + std::mutex JoinMutex_; +}; + +} // namespace NGRpc diff --git a/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.h b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.h new file mode 100644 index 000000000000..bea1671c00cb --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include +#include + +namespace NYdbGrpc { + +struct TGRpcClientConfig { + std::string Locator; // format host:port + TDuration Timeout = TDuration::Max(); // request timeout + ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size + ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests + ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests + ui32 MaxInFlight = 0; + bool EnableSsl = false; + grpc::SslCredentialsOptions SslCredentials; + grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; + ui64 MemQuota = 0; + std::unordered_map StringChannelParams; + std::unordered_map IntChannelParams; + std::string LoadBalancingPolicy = { }; + std::string SslTargetNameOverride = { }; + + TGRpcClientConfig() = default; + TGRpcClientConfig(const TGRpcClientConfig&) = default; + TGRpcClientConfig(TGRpcClientConfig&&) = default; + TGRpcClientConfig& operator=(const TGRpcClientConfig&) = default; + TGRpcClientConfig& operator=(TGRpcClientConfig&&) = default; + + TGRpcClientConfig(const std::string& locator, TDuration timeout = TDuration::Max(), + ui64 maxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, const std::string& caCert = "", const std::string& clientCert = "", + const std::string& clientPrivateKey = "", grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) + : Locator(locator) + , Timeout(timeout) + , MaxMessageSize(maxMessageSize) + , MaxInFlight(maxInFlight) + , EnableSsl(enableSsl) + , SslCredentials{.pem_root_certs = NYdb::TStringType{caCert}, + .pem_private_key = NYdb::TStringType{clientPrivateKey}, + .pem_cert_chain = NYdb::TStringType{clientCert}} + , CompressionAlgoritm(compressionAlgorithm) + {} +}; + +inline std::shared_ptr CreateChannelInterface(const TGRpcClientConfig& config, grpc_socket_mutator* mutator = nullptr){ + grpc::ChannelArguments args; + args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); + args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); + args.SetCompressionAlgorithm(config.CompressionAlgoritm); + + for (const auto& kvp: config.StringChannelParams) { + args.SetString(NYdb::TStringType{kvp.first}, NYdb::TStringType{kvp.second}); + } + + for (const auto& kvp: config.IntChannelParams) { + args.SetInt(NYdb::TStringType{kvp.first}, kvp.second); + } + + if (config.MemQuota) { + grpc::ResourceQuota quota; + quota.Resize(config.MemQuota); + args.SetResourceQuota(quota); + } + if (mutator) { + args.SetSocketMutator(mutator); + } + if (!config.LoadBalancingPolicy.empty()) { + args.SetLoadBalancingPolicyName(NYdb::TStringType{config.LoadBalancingPolicy}); + } + if (!config.SslTargetNameOverride.empty()) { + args.SetSslTargetNameOverride(NYdb::TStringType{config.SslTargetNameOverride}); + } + if (config.EnableSsl || !config.SslCredentials.pem_root_certs.empty()) { + return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::SslCredentials(config.SslCredentials), args); + } else { + return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::InsecureChannelCredentials(), args); + } +} + +} // namespace NGRpc diff --git a/ydb/public/sdk/cpp/src/library/grpc/client/ya.make b/ydb/public/sdk/cpp/src/library/grpc/client/ya.make new file mode 100644 index 000000000000..8b479d117df2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/grpc/client/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + grpc_client_low.cpp +) + +PEERDIR( + contrib/libs/grpc + library/cpp/containers/stack_vector +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/grpc/common/constants.h b/ydb/public/sdk/cpp/src/library/grpc/common/constants.h new file mode 100644 index 000000000000..14ff7105342e --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/grpc/common/constants.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdbGrpc { + +constexpr uint64_t DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; + +} diff --git a/ydb/public/sdk/cpp/src/library/issue/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/issue/CMakeLists.txt new file mode 100644 index 000000000000..2d81c602eba5 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/CMakeLists.txt @@ -0,0 +1,21 @@ +_ydb_sdk_add_library(yql-public-issue) + +target_link_libraries(yql-public-issue + PUBLIC + yutil + protobuf::libprotobuf + colorizer + resource + api-protos + enum_serialization_runtime + string_utils-misc +) + +target_sources(yql-public-issue + PRIVATE + utf8.cpp + yql_issue_message.cpp + yql_issue.cpp +) + +_ydb_sdk_install_targets(TARGETS yql-public-issue) diff --git a/ydb/public/sdk/cpp/src/library/issue/utf8.cpp b/ydb/public/sdk/cpp/src/library/issue/utf8.cpp new file mode 100644 index 000000000000..39ca0eec4ebe --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/utf8.cpp @@ -0,0 +1,58 @@ +#include "utf8.h" + +#include + +namespace NYdb::NIssue { + +namespace { + +unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; +} + +} + +bool IsUtf8(const std::string_view& str) { + for (auto it = str.cbegin(); str.cend() != it;) { +#define COPY() if (str.cend() != it) { c = *it++; } else { return false; } +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + auto c = *it++; + if (!(c & 0x80)) + continue; + + bool result = true; + switch (GetRange(static_cast(c))) { + case 2: TAIL(); break; + case 3: TAIL(); TAIL(); break; + case 4: COPY(); TRANS(0x50); TAIL(); break; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); break; + case 6: TAIL(); TAIL(); TAIL(); break; + case 10: COPY(); TRANS(0x20); TAIL(); break; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); break; + default: return false; + } + + if (!result) return false; +#undef COPY +#undef TRANS +#undef TAIL + } + return true; +} + +} diff --git a/ydb/public/sdk/cpp/src/library/issue/utf8.h b/ydb/public/sdk/cpp/src/library/issue/utf8.h new file mode 100644 index 000000000000..66141bf29984 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/utf8.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdb::NIssue { + +bool IsUtf8(const std::string_view& str); + +} diff --git a/ydb/public/sdk/cpp/src/library/issue/ya.make b/ydb/public/sdk/cpp/src/library/issue/ya.make new file mode 100644 index 000000000000..fbaf3f720579 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + utf8.cpp + yql_issue.cpp + yql_issue_message.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/colorizer + ydb/public/api/protos + ydb/public/sdk/cpp/src/library/string_utils/helpers +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/issue/yql_issue.cpp b/ydb/public/sdk/cpp/src/library/issue/yql_issue.cpp new file mode 100644 index 000000000000..ef139fd58851 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/yql_issue.cpp @@ -0,0 +1,302 @@ +#include "utf8.h" + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace NYdb::NIssue { + +std::string SeverityToString(ESeverity severity) { + switch (severity) { + case ESeverity::Fatal: + return "Fatal"; + case ESeverity::Error: + return "Error"; + case ESeverity::Warning: + return "Warning"; + case ESeverity::Info: + return "Info"; + } +} + +void SanitizeNonAscii(std::string& s) { + if (!NYdb::NIssue::IsUtf8(s)) { + std::string escaped; + escaped.reserve(s.size()); + const unsigned char* i = reinterpret_cast(s.data()); + const unsigned char* end = i + s.size(); + while (i < end) { + char32_t rune; + size_t runeLen; + const RECODE_RESULT result = SafeReadUTF8Char(rune, runeLen, i, end); + if (result == RECODE_OK) { + escaped.insert(escaped.end(), reinterpret_cast(i), reinterpret_cast(i + runeLen)); + i += runeLen; + } else { + escaped.push_back('?'); + ++i; + } + } + s = escaped; + } +} + +uint64_t TIssue::Hash() const noexcept { + return CombineHashes( + CombineHashes( + (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), + std::hash{}(Position.File) + ), + (size_t)CombineHashes((size_t)IntHash(static_cast(IssueCode)), std::hash{}(Message))); +} + +void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { + out << Range() << ": " << SeverityToString(GetSeverity()) << ": "; + if (oneLine) { + std::string message = StripString(Message); + SubstGlobal(message, '\n', ' '); + out << message; + } else { + out << Message; + } + if (GetCode()) { + out << ", code: " << GetCode(); + } +} + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function fn, std::function afterChildrenFn) { + enum class EFnType { + Main, + AfterChildren, + }; + + const bool hasAfterChildrenFn = bool(afterChildrenFn); + TStack> issuesStack; + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::Main)); + while (!issuesStack.empty()) { + auto level = std::get<0>(issuesStack.top()); + const auto& curIssue = *std::get<1>(issuesStack.top()); + const EFnType fnType = std::get<2>(issuesStack.top()); + issuesStack.pop(); + if (!leafOnly || curIssue.GetSubIssues().empty()) { + if (fnType == EFnType::Main) { + fn(curIssue, level); + } else { + afterChildrenFn(curIssue, level); + } + } + if (fnType == EFnType::Main) { + level++; + const auto& subIssues = curIssue.GetSubIssues(); + for (int i = subIssues.size() - 1; i >= 0; i--) { + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::Main)); + } + } + } +} + +namespace { + +Y_NO_INLINE void Indent(IOutputStream& out, uint32_t indentation) { + char* whitespaces = reinterpret_cast(alloca(indentation)); + memset(whitespaces, ' ', indentation); + out.Write(whitespaces, indentation); +} + +void ProgramLinesWithErrors( + const std::string& programText, + const std::vector& errors, + std::map& lines) +{ + std::vector rows; + for (const auto& topIssue: errors) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t /*level*/) { + for (uint32_t row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { + rows.push_back(row); + } + }); + } + std::sort(rows.begin(), rows.end()); + + auto prog = StringSplitter(programText).Split('\n'); + auto progIt = prog.begin(), progEnd = prog.end(); + uint32_t progRow = 1; + + for (uint32_t row: rows) { + while (progRow < row && progIt != progEnd) { + ++progRow; + ++progIt; + } + if (progIt != progEnd) { + lines[row] = progIt->Token(); + } + } +} + +} // namspace + +void TIssues::PrintTo(IOutputStream& out, bool oneLine) const +{ + if (oneLine) { + bool printWithSpace = false; + if (Issues_.size() > 1) { + printWithSpace = true; + out << "["; + } + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { + if (level > 0) { + out << " subissue: { "; + } else { + out << (printWithSpace ? " { " : "{ "); + } + issue.PrintTo(out, true); + }, + [&](const TIssue&, uint16_t) { + out << " }"; + }); + } + if (Issues_.size() > 1) { + out << " ]"; + } + } else { + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { + auto shift = level * 4; + Indent(out, shift); + out << issue << Endl; + }); + } + } +} + +void TIssues::PrintWithProgramTo( + IOutputStream& out, + const std::string& programFilename, + const std::string& programText) const +{ + using namespace NColorizer; + + std::map lines; + ProgramLinesWithErrors(programText, Issues_, lines); + + for (const TIssue& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { + auto shift = level * 4; + Indent(out, shift); + out << DarkGray() << programFilename << Old() << ':'; + out << Purple() << issue.Range() << Old(); + auto color = (issue.GetSeverity() == ESeverity::Warning) ? Yellow() : LightRed(); + auto severityName = SeverityToString(issue.GetSeverity()); + out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n'; + Indent(out, shift); + if (issue.Position.HasValue()) { + out << '\t' << lines[issue.Position.Row] << '\n'; + out << '\t'; + if (issue.Position.Column > 0) { + Indent(out, issue.Position.Column - 1); + } + out << '^'; + } + out << Endl; + }); + } +} + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { + std::string_view messageBuf = e.what(); + auto parsedPos = TryParseTerminationMessage(messageBuf); + auto issue = TIssue(parsedPos.value_or(pos), messageBuf); + const TErrorException* errorException = dynamic_cast(&e); + if (errorException) { + issue.SetCode(errorException->GetCode(), ESeverity::Error); + } else { + issue.SetCode(UNEXPECTED_ERROR, ESeverity::Fatal); + } + return issue; +} + +static constexpr std::string_view TerminationMessageMarker = "Terminate was called, reason("; + +std::optional TryParseTerminationMessage(std::string_view& message) { + size_t len = 0; + size_t startPos = message.find(TerminationMessageMarker); + size_t endPos = 0; + if (startPos != std::string::npos) { + endPos = message.find(')', startPos + TerminationMessageMarker.size()); + if (endPos != std::string::npos) { + std::string_view lenText = message.substr(startPos + TerminationMessageMarker.size()) + .substr(0, endPos - startPos - TerminationMessageMarker.size()); + auto result = std::from_chars(lenText.data(), lenText.data() + lenText.size(), len); + if (result.ec != std::errc{} || result.ptr != lenText.data() + lenText.size()) { + len = 0; + } + } + } + + if (len) { + message = message.substr(endPos + 3).substr(0, len); + auto s = message; + std::optional file; + std::optional row; + std::optional column; + NUtils::GetNext(s, ':', file); + NUtils::GetNext(s, ':', row); + NUtils::GetNext(s, ':', column); + uint32_t rowValue, columnValue; + if (file && row && column && TryFromString(*row, rowValue) && TryFromString(*column, columnValue)) { + message = StripStringLeft(s); + return TPosition(columnValue, rowValue, std::string(*file)); + } + } + + return std::nullopt; +} + +} // namspace NYql + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TPosition& pos) { + out << (pos.File.empty() ? "

" : pos.File); + if (pos) { + out << ":" << pos.Row << ':' << pos.Column; + } +} + +template<> +void Out(IOutputStream & out, const NYdb::NIssue::TRange & range) { + if (range.IsRange()) { + out << '[' << range.Position << '-' << range.EndPosition << ']'; + } else { + out << range.Position; + } +} + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TIssue& error) { + error.PrintTo(out); +} + +template <> +void Out(IOutputStream& out, const NYdb::NIssue::TIssues& error) { + error.PrintTo(out); +} diff --git a/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.cpp b/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.cpp new file mode 100644 index 000000000000..65fae800443e --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.cpp @@ -0,0 +1,93 @@ +#include "yql_issue_message.h" + +#include + +#include +#include +#include + +#include + +namespace NYdb::NIssue { + +TIssue IssueFromMessage(const Ydb::Issue::IssueMessage& issueMessage) { + TIssue topIssue; + std::deque> queue; + queue.push_front(std::make_pair(&topIssue, &issueMessage)); + while (!queue.empty()) { + TIssue& issue = *queue.back().first; + const auto& message = *queue.back().second; + queue.pop_back(); + TPosition position(message.position().column(), message.position().row(), message.position().file()); + TPosition endPosition(message.end_position().column(), message.end_position().row()); + if (position.HasValue()) { + if (endPosition.HasValue()) { + issue = TIssue(position, endPosition, message.message()); + } else { + issue = TIssue(position, message.message()); + } + } else { + issue = TIssue(message.message()); + } + + for (const auto& subMessage : message.issues()) { + auto subIssue = new TIssue(); + issue.AddSubIssue(subIssue); + queue.push_front(std::make_pair(subIssue, &subMessage)); + } + + issue.SetCode(message.issue_code(), static_cast(message.severity())); + } + return topIssue; +} + +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField &message, TIssues &issues) { + issues.Clear(); + if (message.size()) { + issues.Reserve(message.size()); + for (auto &x : message) { + issues.AddIssue(IssueFromMessage(x)); + } + } +} + +void IssueToMessage(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage) { + std::deque> queue; + queue.push_front(std::make_pair(&topIssue, issueMessage)); + while (!queue.empty()) { + const TIssue& issue = *queue.back().first; + auto& message = *queue.back().second; + queue.pop_back(); + if (issue.Position) { + auto& position = *message.mutable_position(); + position.set_row(issue.Position.Row); + position.set_column(issue.Position.Column); + position.set_file(NYdb::TStringType{issue.Position.File}); + } + if (issue.EndPosition) { + auto& endPosition = *message.mutable_end_position(); + endPosition.set_row(issue.EndPosition.Row); + endPosition.set_column(issue.EndPosition.Column); + } + message.set_message(NYdb::TStringType{issue.GetMessage()}); + message.set_issue_code(issue.GetCode()); + message.set_severity(static_cast(issue.GetSeverity())); + + for (auto subIssue : issue.GetSubIssues()) { + Ydb::Issue::IssueMessage* subMessage = message.add_issues(); + queue.push_front(std::make_pair(subIssue.Get(), subMessage)); + } + } +} + +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField *message) { + message->Clear(); + if (!issues) + return; + message->Reserve(issues.Size()); + for (const auto &issue : issues) { + IssueToMessage(issue, message->Add()); + } +} + +} diff --git a/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.h b/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.h new file mode 100644 index 000000000000..c6ee864101e2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/issue/yql_issue_message.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace NYdb::NIssue { + +TIssue IssueFromMessage(const Ydb::Issue::IssueMessage& issueMessage); +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message, TIssues& issues); + +void IssueToMessage(const TIssue& topIssue, Ydb::Issue::IssueMessage* message); +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField* message); + +} diff --git a/ydb/public/sdk/cpp/src/library/jwt/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/jwt/CMakeLists.txt new file mode 100644 index 000000000000..99a1153efc6a --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/jwt/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(library-jwt) + +target_link_libraries(library-jwt PUBLIC + yutil + jwt-cpp::jwt-cpp + json + impl-ydb_internal-common +) + +target_sources(library-jwt PRIVATE + jwt.cpp +) + +_ydb_sdk_install_targets(TARGETS library-jwt) diff --git a/ydb/public/lib/jwt/jwt.cpp b/ydb/public/sdk/cpp/src/library/jwt/jwt.cpp similarity index 86% rename from ydb/public/lib/jwt/jwt.cpp rename to ydb/public/sdk/cpp/src/library/jwt/jwt.cpp index f5ee7145e8aa..5f11c7d67ac9 100644 --- a/ydb/public/lib/jwt/jwt.cpp +++ b/ydb/public/sdk/cpp/src/library/jwt/jwt.cpp @@ -1,16 +1,16 @@ -#include +#include -#include "jwt.h" +#include #include #include namespace NYdb { -TStringType MakeSignedJwt(const TJwtParams& params, const TDuration& lifetime) { +std::string MakeSignedJwt(const TJwtParams& params, const TDuration& lifetime) { // this constant works on all envs (internal IAM, preprod cloud, prod cloud) // according to potamus@, it's recommended audience - static const TStringType AUDIENCE{"https://iam.api.cloud.yandex.net/iam/v1/tokens"}; + static const std::string AUDIENCE{"https://iam.api.cloud.yandex.net/iam/v1/tokens"}; const auto now = std::chrono::system_clock::now(); const auto expire = now + std::chrono::milliseconds(lifetime.MilliSeconds()); const auto token = jwt::create() @@ -20,10 +20,10 @@ TStringType MakeSignedJwt(const TJwtParams& params, const TDuration& lifetime) { .set_audience(AUDIENCE) .set_expires_at(expire) .sign(jwt::algorithm::ps256(params.PubKey, params.PrivKey)); - return TStringType{token}; + return std::string{token}; } -TJwtParams ParseJwtParams(const TStringType& jsonParamsStr) { +TJwtParams ParseJwtParams(const std::string& jsonParamsStr) { NJson::TJsonValue json; NJson::ReadJsonTree(jsonParamsStr, &json, true); auto map = json.GetMap(); diff --git a/ydb/public/sdk/cpp/src/library/jwt/ya.make b/ydb/public/sdk/cpp/src/library/jwt/ya.make new file mode 100644 index 000000000000..3bd60ca6d015 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/jwt/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + jwt.cpp +) + +PEERDIR( + contrib/libs/jwt-cpp + library/cpp/json +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/malloc/api/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/malloc/api/CMakeLists.txt new file mode 100644 index 000000000000..6121ef27cdad --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/malloc/api/CMakeLists.txt @@ -0,0 +1,5 @@ +_ydb_sdk_add_library(malloc-api) + +target_sources(malloc-api PRIVATE + malloc.cpp +) diff --git a/ydb/public/sdk/cpp/src/library/malloc/api/malloc.cpp b/ydb/public/sdk/cpp/src/library/malloc/api/malloc.cpp new file mode 100644 index 000000000000..6f5a3514d56f --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/malloc/api/malloc.cpp @@ -0,0 +1,43 @@ +#include +#include + +#include "malloc.h" + +namespace { + bool SetEmptyParam(const char*, const char*) { + return false; + } + + const char* GetEmptyParam(const char*) { + return nullptr; + } + + bool CheckEmptyParam(const char*, bool defaultValue) { + return defaultValue; + } +} + +namespace NMalloc { + volatile bool IsAllocatorCorrupted = false; + + TMallocInfo::TMallocInfo() + : Name() + , SetParam(SetEmptyParam) + , GetParam(GetEmptyParam) + , CheckParam(CheckEmptyParam) + { + } + + void AbortFromCorruptedAllocator(const char* errorMessage) { + errorMessage = errorMessage ? errorMessage : ""; + fprintf(stderr, "Allocator error: %s\n", errorMessage); + IsAllocatorCorrupted = true; + abort(); + } + + TMallocInfo MallocInfo() { + TMallocInfo r; + r.Name = "system"; + return r; + } +} diff --git a/ydb/public/sdk/cpp/src/library/malloc/api/malloc.h b/ydb/public/sdk/cpp/src/library/malloc/api/malloc.h new file mode 100644 index 000000000000..ebd545d6dd92 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/malloc/api/malloc.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace NMalloc { + struct TMallocInfo { + TMallocInfo(); + + const char* Name; + + bool (*SetParam)(const char* param, const char* value); + const char* (*GetParam)(const char* param); + + bool (*CheckParam)(const char* param, bool defaultValue); + }; + + extern volatile bool IsAllocatorCorrupted; + void AbortFromCorruptedAllocator(const char* errorMessage = nullptr); + + // this function should be implemented by malloc implementations + TMallocInfo MallocInfo(); + + struct TAllocHeader { + void* Block; + size_t AllocSize; + void Y_FORCE_INLINE Encode(void* block, size_t size, size_t signature) { + Block = block; + AllocSize = size | signature; + } + }; +} diff --git a/ydb/public/sdk/cpp/src/library/operation_id/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/operation_id/CMakeLists.txt new file mode 100644 index 000000000000..3d525a37c150 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/operation_id/CMakeLists.txt @@ -0,0 +1,17 @@ +add_subdirectory(protos) + +_ydb_sdk_add_library(library-operation_id) + +target_link_libraries(library-operation_id + PRIVATE + yutil + cgiparam + uri + lib-operation_id-protos +) + +target_sources(library-operation_id PRIVATE + operation_id.cpp +) + +_ydb_sdk_install_targets(TARGETS library-operation_id) diff --git a/ydb/public/sdk/cpp/src/library/operation_id/operation_id.cpp b/ydb/public/sdk/cpp/src/library/operation_id/operation_id.cpp new file mode 100644 index 000000000000..9d8e880332d6 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/operation_id/operation_id.cpp @@ -0,0 +1,309 @@ +#include +#include + +#include + +#include +#include + +#include +#include + +namespace NKikimr { +namespace NOperationId { + +using namespace NUri; + +static const std::string QueryIdPrefix = "ydb://preparedqueryid/4?id="; + +std::string FormatPreparedQueryIdCompat(const std::string& in) { + return QueryIdPrefix + in; +} + +bool DecodePreparedQueryIdCompat(const std::string& in, std::string& out) { + if (in.size() <= QueryIdPrefix.size()) { + ythrow yexception() << "Unable to parse input string"; + } + if (in.compare(0, QueryIdPrefix.size(), QueryIdPrefix) == 0) { + out = in.substr(QueryIdPrefix.size()); + return true; + } + return false; +} + +std::string ProtoToString(const Ydb::TOperationId& proto) { + using namespace ::google::protobuf; + const Reflection& reflection = *proto.GetReflection(); + std::vector fields; + reflection.ListFields(proto, &fields); + TStringStream res; + switch (proto.kind()) { + case Ydb::TOperationId::OPERATION_DDL: + case Ydb::TOperationId::OPERATION_DML: + res << "ydb://operation"; + break; + case Ydb::TOperationId::SESSION_YQL: + res << "ydb://session"; + break; + case Ydb::TOperationId::PREPARED_QUERY_ID: + res << "ydb://preparedqueryid"; + break; + case Ydb::TOperationId::CMS_REQUEST: + res << "ydb://cmsrequest"; + break; + case Ydb::TOperationId::EXPORT: + res << "ydb://export"; + break; + case Ydb::TOperationId::IMPORT: + res << "ydb://import"; + break; + case Ydb::TOperationId::BUILD_INDEX: + res << "ydb://buildindex"; + break; + case Ydb::TOperationId::SCRIPT_EXECUTION: + res << "ydb://scriptexec"; + break; + default: + Y_ABORT_UNLESS(false, "unexpected kind"); + } + // According to protobuf documentation: + // Fields (both normal fields and extension fields) will be listed ordered by field number, + // so we can rely on it to build url string + for (const FieldDescriptor* field : fields) { + Y_ASSERT(field != nullptr); + if (field) { + if (field->is_repeated()) { + int size = reflection.FieldSize(proto, field); + if (size) { + res << "?"; + } + for (int i = 0; i < size; i++) { + const auto& message = reflection.GetRepeatedMessage(proto, field, i); + const auto& data = dynamic_cast(message); + TUri::ReEncode(res, data.key()); + res << "="; + TUri::ReEncode(res, data.value()); + if (i < size - 1) { + res << "&"; + } + } + } else { + res << "/"; + const FieldDescriptor::CppType type = field->cpp_type(); + switch (type) { + case FieldDescriptor::CPPTYPE_ENUM: + res << reflection.GetEnumValue(proto, field); + break; + default: + Y_ABORT_UNLESS(false, "unexpected protobuf field type"); + break; + } + } + } + } + return res.Str(); +} + +class TOperationId::TImpl { +public: + TImpl() { + Proto.set_kind(Ydb::TOperationId::UNUSED); + } + + TImpl(const std::string &string, bool allowEmpty) { + if (allowEmpty && string.empty()) { + Proto.set_kind(Ydb::TOperationId::UNUSED); + return; + } + + TUri uri; + TState::EParsed er = uri.Parse(string, TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible); + if (er != TState::ParsedOK) { + ythrow yexception() << "Unable to parse input string"; + } + std::string path = uri.PrintS(TField::FlagPath).substr(1); // start from 1 to remove first '/' + if (path.length() < 1) { + ythrow yexception() << "Invalid path length"; + } + int kind; + if (!TryFromString(path, kind)) { + ythrow yexception() << "Unable to cast \"kind\" field: " << path; + } + + if (!Proto.EKind_IsValid(kind)) { + ythrow yexception() << "Invalid operation kind: " << kind; + } + + Proto.set_kind(static_cast(kind)); + + std::string query = uri.PrintS(TField::FlagQuery); + + if (!query.empty()) { + TCgiParameters params(query.substr(1)); // start from 1 to remove first '?' + for (auto it : params) { + auto data = Proto.add_data(); + data->set_key(it.first); + data->set_value(it.second); +#ifdef YDB_SDK_USE_STD_STRING + Index[it.first].push_back(&data->value()); +#else + Index[it.first].push_back(&data->value().ConstRef()); +#endif + } + } + } + + TImpl(const TImpl& other) { + Proto.CopyFrom(other.Proto); + BuildIndex(); + } + + TImpl(TImpl&&) = default; + + TImpl& operator=(const TImpl&) = delete; + TImpl& operator=(TImpl&&) = delete; + + ~TImpl() = default; + + Ydb::TOperationId& GetProto() { + return Proto; + } + + const Ydb::TOperationId& GetProto() const { + return Proto; + } + + const std::vector& GetValue(const std::string& key) const { + auto it = Index.find(key); + if (it != Index.end()) { + return it->second; + } + ythrow yexception() << "Unable to find key: " << key; + } + + std::string GetSubKind() const { + auto it = Index.find("kind"); + if (it == Index.end()) { + return std::string(); + } + + if (it->second.size() != 1) { + ythrow yexception() << "Unable to retreive sub-kind"; + } + + return *it->second.at(0); + } + +private: + void BuildIndex() { + for (const auto& data : Proto.data()) { +#ifdef YDB_SDK_USE_STD_STRING + Index[data.key()].push_back(&data.value()); +#else + Index[data.key()].push_back(&data.value().ConstRef()); +#endif + } + } + + Ydb::TOperationId Proto; + std::unordered_map> Index; +}; + +TOperationId::TOperationId() { + Impl = std::make_unique(); +} + +TOperationId::TOperationId(const std::string &string, bool allowEmpty) { + Impl = std::make_unique(string, allowEmpty); +} + +TOperationId::TOperationId(const TOperationId& other) { + Impl = std::make_unique(*other.Impl); +} + +TOperationId::TOperationId(TOperationId&& other) { + Impl = std::make_unique(std::move(*other.Impl)); +} + +TOperationId& TOperationId::operator=(const TOperationId& other) { + Impl = std::make_unique(*other.Impl); + return *this; +} + +TOperationId& TOperationId::operator=(TOperationId&& other) { + Impl = std::make_unique(std::move(*other.Impl)); + return *this; +} + +TOperationId::~TOperationId() { + Impl.reset(); +} + +TOperationId::EKind TOperationId::GetKind() const { + return static_cast(Impl->GetProto().kind()); +} + +void TOperationId::SetKind(const EKind& kind) { + Impl->GetProto().set_kind(static_cast(kind)); +} + +std::vector TOperationId::GetData() const { + std::vector result; + for (auto data : Impl->GetProto().data()) { + result.push_back({data.key(), data.value()}); + } + return result; +} + +void TOperationId::AddOptionalValue(const std::string& key, const std::string& value) { + NKikimr::NOperationId::AddOptionalValue(Impl->GetProto(), key, value); +} + +const std::vector& TOperationId::GetValue(const std::string& key) const { + return Impl->GetValue(key); +} + +std::string TOperationId::GetSubKind() const { + return Impl->GetSubKind(); +} + +std::string TOperationId::ToString() const { + return ProtoToString(Impl->GetProto()); +} + +const Ydb::TOperationId& TOperationId::GetProto() const { + return Impl->GetProto(); +} + +void AddOptionalValue(Ydb::TOperationId& proto, const std::string& key, const std::string& value) { + auto data = proto.add_data(); + data->set_key(NYdb::TStringType{key}); + data->set_value(NYdb::TStringType{value}); +} + +TOperationId::EKind ParseKind(const std::string_view value) { + if (value.starts_with("ss/backgrounds")) { + return TOperationId::SS_BG_TASKS; + } + + if (value.starts_with("export")) { + return TOperationId::EXPORT; + } + + if (value.starts_with("import")) { + return TOperationId::IMPORT; + } + + if (value.starts_with("buildindex")) { + return TOperationId::BUILD_INDEX; + } + + if (value.starts_with("scriptexec")) { + return TOperationId::SCRIPT_EXECUTION; + } + + return TOperationId::UNUSED; +} + +} // namespace NOperationId +} // namespace NKikimr diff --git a/ydb/public/sdk/cpp/src/library/operation_id/protos/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/operation_id/protos/CMakeLists.txt new file mode 100644 index 000000000000..95e4bcf0bdde --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/operation_id/protos/CMakeLists.txt @@ -0,0 +1,13 @@ +file(GLOB_RECURSE LIB_OPERATION_ID_PROTOS + *.proto +) + +_ydb_sdk_add_proto_library(lib-operation_id-protos + SOURCES ${LIB_OPERATION_ID_PROTOS} +) + +target_include_directories(lib-operation_id-protos PRIVATE + ${YDB_SDK_BINARY_DIR}/ydb-cpp-sdk +) + +_ydb_sdk_install_targets(TARGETS lib-operation_id-protos) diff --git a/ydb/public/lib/operation_id/protos/operation_id.proto b/ydb/public/sdk/cpp/src/library/operation_id/protos/operation_id.proto similarity index 100% rename from ydb/public/lib/operation_id/protos/operation_id.proto rename to ydb/public/sdk/cpp/src/library/operation_id/protos/operation_id.proto diff --git a/ydb/public/lib/operation_id/protos/ya.make b/ydb/public/sdk/cpp/src/library/operation_id/protos/ya.make similarity index 100% rename from ydb/public/lib/operation_id/protos/ya.make rename to ydb/public/sdk/cpp/src/library/operation_id/protos/ya.make diff --git a/ydb/public/sdk/cpp/src/library/operation_id/ya.make b/ydb/public/sdk/cpp/src/library/operation_id/ya.make new file mode 100644 index 000000000000..e52d53af8567 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/operation_id/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + operation_id.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/operation_id/protos + library/cpp/cgiparam + library/cpp/uri +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/CMakeLists.txt new file mode 100644 index 000000000000..c3540fa32bae --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(persqueue-obfuscate) + +target_link_libraries(persqueue-obfuscate PUBLIC + yutil +) + +target_sources(persqueue-obfuscate PRIVATE + obfuscate.cpp +) + +_ydb_sdk_install_targets(TARGETS persqueue-obfuscate) diff --git a/ydb/library/persqueue/obfuscate/obfuscate.cpp b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.cpp similarity index 75% rename from ydb/library/persqueue/obfuscate/obfuscate.cpp rename to ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.cpp index c1a9b0b1a7b7..0c260a743b31 100644 --- a/ydb/library/persqueue/obfuscate/obfuscate.cpp +++ b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.cpp @@ -1,10 +1,10 @@ #include "obfuscate.h" -#include +#include namespace NPersQueue { -TString ObfuscateString(TString str) { +std::string ObfuscateString(std::string str) { ui32 publicPartSize = Min(4, str.size() / 4); for (ui32 i = publicPartSize; i < str.size() - publicPartSize; ++i) { str[i] = '*'; diff --git a/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.h b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.h new file mode 100644 index 000000000000..bd30949f2b9e --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/obfuscate.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NPersQueue { + +std::string ObfuscateString(std::string str); + +} // namespace NPersQueue diff --git a/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/ya.make b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/ya.make new file mode 100644 index 000000000000..f3c0614ba78d --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/obfuscate/ya.make @@ -0,0 +1,10 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + obfuscate.h + obfuscate.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/CMakeLists.txt new file mode 100644 index 000000000000..f343adef3ae0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(persqueue-topic_parser_public) + +target_link_libraries(persqueue-topic_parser_public PUBLIC + yutil +) + +target_sources(persqueue-topic_parser_public PRIVATE + topic_parser.cpp +) + +_ydb_sdk_install_targets(TARGETS persqueue-topic_parser_public) diff --git a/ydb/library/persqueue/topic_parser_public/topic_parser.cpp b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.cpp similarity index 64% rename from ydb/library/persqueue/topic_parser_public/topic_parser.cpp rename to ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.cpp index 720a2a754c2b..a5e760b91400 100644 --- a/ydb/library/persqueue/topic_parser_public/topic_parser.cpp +++ b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.cpp @@ -1,65 +1,67 @@ #include "topic_parser.h" +#include + #include namespace NPersQueue { -bool CorrectName(const TString& topic) { - if (!topic.StartsWith("rt3.")) +bool CorrectName(const std::string& topic) { + if (!std::string_view{topic}.starts_with("rt3.")) return false; auto pos = topic.find("--"); - if (pos == TString::npos || pos == 4) //dc is empty + if (pos == std::string::npos || pos == 4) //dc is empty return false; pos += 2; //skip "--" if (pos == topic.size()) // no real topic return false; auto pos2 = topic.find("--", pos); - if (pos2 == TString::npos) + if (pos2 == std::string::npos) return true; if (pos2 == pos || pos2 + 2 == topic.size()) //producer or topic is empty return false; return true; } -TString GetDC(const TString& topic) { +std::string GetDC(const std::string& topic) { if (!CorrectName(topic)) return "unknown"; auto pos = topic.find("--"); - Y_ABORT_UNLESS(pos != TString::npos); + Y_ABORT_UNLESS(pos != std::string::npos); Y_ABORT_UNLESS(pos > 4); //length of "rt3." auto res = topic.substr(4, pos - 4); return res; } -TString GetRealTopic(const TString& topic) { +std::string GetRealTopic(const std::string& topic) { if (!CorrectName(topic)) return topic; auto pos = topic.find("--"); - Y_ABORT_UNLESS(pos != TString::npos); + Y_ABORT_UNLESS(pos != std::string::npos); Y_ABORT_UNLESS(topic.size() > pos + 2); return topic.substr(pos + 2); } -TString GetTopicPath(const TString& topic) { +std::string GetTopicPath(const std::string& topic) { return ConvertOldTopicName(GetRealTopic(topic)); } -TString GetAccount(const TString& topic) { +std::string GetAccount(const std::string& topic) { auto res = GetTopicPath(topic); return res.substr(0, res.find("/")); } -TString GetProducer(const TString& topic) { +std::string GetProducer(const std::string& topic) { if (!CorrectName(topic)) return "unknown"; auto res = GetRealTopic(topic); return res.substr(0, res.find("--")); } -TString ConvertNewTopicName(const TString& topic) { - TString t = NormalizePath(topic); +std::string ConvertNewTopicName(const std::string& topic) { + std::string t = NormalizePath(topic); auto pos = t.rfind("/"); - if (pos == TString::npos) + if (pos == std::string::npos) return t; TStringBuilder res; for (ui32 i = 0; i < pos; ++i) { @@ -72,9 +74,9 @@ TString ConvertNewTopicName(const TString& topic) { } -TString ConvertOldTopicName(const TString& topic) { +std::string ConvertOldTopicName(const std::string& topic) { auto pos = topic.rfind("--"); - if (pos == TString::npos) + if (pos == std::string::npos) return topic; TStringBuilder res; for (ui32 i = 0; i < pos; ++i) { @@ -86,11 +88,11 @@ TString ConvertOldTopicName(const TString& topic) { return res; } -TString BuildFullTopicName(const TString& topicPath, const TString& topicDC) { +std::string BuildFullTopicName(const std::string& topicPath, const std::string& topicDC) { return "rt3." + topicDC + "--" + ConvertNewTopicName(topicPath); } -TString ConvertOldProducerName(const TString& producer) { +std::string ConvertOldProducerName(const std::string& producer) { TStringBuilder res; for (ui32 i = 0; i < producer.size(); ++i) { if (producer[i] == '@') res << "/"; @@ -100,20 +102,20 @@ TString ConvertOldProducerName(const TString& producer) { } -TString NormalizePath(const TString& path) { +std::string NormalizePath(const std::string& path) { size_t st = 0; size_t end = path.size(); - if (path.StartsWith("/")) st = 1; - if (path.EndsWith("/") && end > st) end--; + if (std::string_view{path}.starts_with("/")) st = 1; + if (std::string_view{path}.ends_with("/") && end > st) end--; return path.substr(st, end - st); } -TString ConvertNewConsumerName(const TString& consumer) { +std::string ConvertNewConsumerName(const std::string& consumer) { TStringBuilder res; ui32 pos = 0; - TString c = NormalizePath(consumer); - if (c.StartsWith("shared/")) + std::string c = NormalizePath(consumer); + if (std::string_view{c}.starts_with("shared/")) pos = 7; for (ui32 i = pos; i < c.size(); ++i) { if (c[i] == '/') res << "@"; @@ -122,7 +124,7 @@ TString ConvertNewConsumerName(const TString& consumer) { return res; } -TString ConvertNewProducerName(const TString& producer) { +std::string ConvertNewProducerName(const std::string& producer) { TStringBuilder res; for (ui32 i = 0; i < producer.size(); ++i) { if (producer[i] == '/') res << "@"; @@ -132,7 +134,7 @@ TString ConvertNewProducerName(const TString& producer) { } -TString ConvertOldConsumerName(const TString& consumer) { +std::string ConvertOldConsumerName(const std::string& consumer) { TStringBuilder res; bool shared = true; for (ui32 i = 0; i < consumer.size(); ++i) { diff --git a/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.h b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.h new file mode 100644 index 000000000000..1d8122c0851e --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/topic_parser.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace NPersQueue { + +std::string GetDC(const std::string& topic); + +std::string GetRealTopic(const std::string& topic); + +std::string BuildFullTopicName(const std::string& topicPath, const std::string& topicDC); + +std::string GetProducer(const std::string& topic); +std::string GetAccount(const std::string& topic); +std::string GetTopicPath(const std::string& topic); + +std::string NormalizePath(const std::string& path); + +bool CorrectName(const std::string& topic); + +std::string ConvertNewTopicName(const std::string& topic); + +std::string ConvertNewConsumerName(const std::string& consumer); +std::string ConvertNewProducerName(const std::string& consumer); + + +std::string ConvertOldTopicName(const std::string& topic); +std::string ConvertOldProducerName(const std::string& producer); +std::string ConvertOldConsumerName(const std::string& consumer); + + +} // namespace NPersQueue diff --git a/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/ya.make b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/ya.make new file mode 100644 index 000000000000..051193a750fa --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/persqueue/topic_parser_public/ya.make @@ -0,0 +1,10 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + topic_parser.h + topic_parser.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/proto_output/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/proto_output/CMakeLists.txt new file mode 100644 index 000000000000..70bd7b0017d1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/proto_output/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(proto_output) + +target_link_libraries(proto_output PUBLIC + yutil + api-protos + protobuf::libprotobuf +) + +target_sources(proto_output PRIVATE + proto_output.cpp +) + +_ydb_sdk_install_targets(TARGETS proto_output) + diff --git a/ydb/public/sdk/cpp/src/library/proto_output/proto_output.cpp b/ydb/public/sdk/cpp/src/library/proto_output/proto_output.cpp new file mode 100644 index 000000000000..86eda0c2eccd --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/proto_output/proto_output.cpp @@ -0,0 +1,56 @@ +#include + +#include +#include +#include + +#include + +#include + +namespace { + std::string ShortUtf8DebugString(const google::protobuf::Message& msg) { + google::protobuf::TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + + std::string str; + printer.PrintToString(msg, &str); + + // Copied from text_format.h + // Single line mode currently might have an extra space at the end. + if (str.size() > 0 && str[str.size() - 1] == ' ') { + str.resize(str.size() - 1); + } + return str; + } +} + +Y_DECLARE_OUT_SPEC(, Ydb::Issue::IssueMessage, stream, value) { + google::protobuf::TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + + NYdb::TStringType str; + printer.PrintToString(value, &str); + + // Copied from text_format.h + // Single line mode currently might have an extra space at the end. + if (str.size() > 0 && str[str.size() - 1] == ' ') { + str.resize(str.size() - 1); + } + + stream << "{ " << str << " }"; +} + +Y_DECLARE_OUT_SPEC(, Ydb::VariantType, stream, value) { + stream << "{ " << ShortUtf8DebugString(value) << " }"; +} + +Y_DECLARE_OUT_SPEC(, Ydb::Topic::StreamReadMessage_ReadResponse, stream, value) { + stream << "{ " << ShortUtf8DebugString(value) << " }"; +} + +Y_DECLARE_OUT_SPEC(, Ydb::Topic::StreamReadMessage_CommitOffsetResponse, stream, value) { + stream << "{ " << ShortUtf8DebugString(value) << " }"; +} diff --git a/ydb/public/sdk/cpp/src/library/retry/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/retry/CMakeLists.txt new file mode 100644 index 000000000000..b2a166488081 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/CMakeLists.txt @@ -0,0 +1,20 @@ +file(GLOB_RECURSE LIBRARY_RETRY_PROTOS + protos/*.proto +) +_ydb_sdk_add_proto_library(retry-protos + SOURCES ${LIBRARY_RETRY_PROTOS} +) +_ydb_sdk_install_targets(TARGETS retry-protos) + +_ydb_sdk_add_library(retry) +target_link_libraries(retry + PUBLIC + yutil + retry-protos +) +target_sources(retry + PRIVATE + retry.cpp + utils.cpp +) +_ydb_sdk_install_targets(TARGETS retry) diff --git a/ydb/public/sdk/cpp/src/library/retry/protos/retry_options.proto b/ydb/public/sdk/cpp/src/library/retry/protos/retry_options.proto new file mode 100644 index 000000000000..063fff01412d --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/protos/retry_options.proto @@ -0,0 +1,9 @@ +package NRetry; + +message TRetryOptionsPB { + optional uint32 MaxTries = 1 [default = 5]; + optional uint32 InitialSleepMs = 2 [default = 0]; + optional uint32 SleepIncrementMs = 3 [default = 0]; + optional uint32 RandomDeltaMs = 4 [default = 10]; + optional uint32 ExponentalMultiplierMs = 5 [default = 100]; +} diff --git a/ydb/public/sdk/cpp/src/library/retry/retry.cpp b/ydb/public/sdk/cpp/src/library/retry/retry.cpp new file mode 100644 index 000000000000..1ed6b9ac7dc2 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/retry.cpp @@ -0,0 +1,51 @@ +#include "retry.h" + +#include + +namespace { +class TRetryOptionsWithRetCodePolicy : public IRetryPolicy { +public: + explicit TRetryOptionsWithRetCodePolicy(const TRetryOptions& opts) + : Opts(opts) + { + } + + class TRetryState : public IRetryState { + public: + explicit TRetryState(const TRetryOptions& opts) + : Opts(opts) + { + } + + std::optional GetNextRetryDelay(bool ret) override { + if (ret || Attempt == Opts.RetryCount) { + return std::nullopt; + } + return Opts.GetTimeToSleep(Attempt++); + } + + private: + const TRetryOptions Opts; + size_t Attempt = 0; + }; + + IRetryState::TPtr CreateRetryState() const override { + return std::make_unique(Opts); + } + +private: + const TRetryOptions Opts; +}; +} // namespace + +bool DoWithRetryOnRetCode(std::function func, TRetryOptions retryOptions) { + return DoWithRetryOnRetCode(std::move(func), std::make_shared(retryOptions), retryOptions.SleepFunction); +} + +TRetryOptions MakeRetryOptions(const NRetry::TRetryOptionsPB& retryOptions) { + return TRetryOptions(retryOptions.maxtries(), + TDuration::MilliSeconds(retryOptions.initialsleepms()), + TDuration::MilliSeconds(retryOptions.randomdeltams()), + TDuration::MilliSeconds(retryOptions.sleepincrementms()), + TDuration::MilliSeconds(retryOptions.exponentalmultiplierms())); +} diff --git a/ydb/public/sdk/cpp/src/library/retry/retry.h b/ydb/public/sdk/cpp/src/library/retry/retry.h new file mode 100644 index 000000000000..24b0440a1d1a --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/retry.h @@ -0,0 +1,223 @@ +#pragma once + +#include +#include "utils.h" + +#include + +#include +#include +#include + +#include + +struct TRetryOptions { + ui32 RetryCount; + + // TotalDuration = SleepDuration +/- SleepRandomDelta + (attempt * SleepIncrement) + (2**attempt * SleepExponentialMultiplier) + TDuration SleepDuration; + TDuration SleepRandomDelta; + TDuration SleepIncrement; + TDuration SleepExponentialMultiplier; + + std::function SleepFunction; + + TRetryOptions(ui32 retryCount = 3, TDuration sleepDuration = TDuration::Seconds(1), TDuration sleepRandomDelta = TDuration::Zero(), + TDuration sleepIncrement = TDuration::Zero(), TDuration sleepExponentialMultiplier = TDuration::Zero(), + std::function sleepFunction = [](TDuration d) { Sleep(d); }) // can't use Sleep itself due to Win compilation error + : RetryCount(retryCount) + , SleepDuration(sleepDuration) + , SleepRandomDelta(sleepRandomDelta) + , SleepIncrement(sleepIncrement) + , SleepExponentialMultiplier(sleepExponentialMultiplier) + , SleepFunction(sleepFunction) + { + } + + TRetryOptions& WithCount(ui32 retryCount) { + RetryCount = retryCount; + return *this; + } + + TRetryOptions& WithSleep(TDuration sleepDuration) { + SleepDuration = sleepDuration; + return *this; + } + + TRetryOptions& WithRandomDelta(TDuration sleepRandomDelta) { + SleepRandomDelta = sleepRandomDelta; + return *this; + } + + TRetryOptions& WithIncrement(TDuration sleepIncrement) { + SleepIncrement = sleepIncrement; + return *this; + } + + TRetryOptions& WithExponentialMultiplier(TDuration sleepExponentialMultiplier) { + SleepExponentialMultiplier = sleepExponentialMultiplier; + return *this; + } + + TRetryOptions& WithSleepFunction(std::function sleepFunction) { + SleepFunction = sleepFunction; + return *this; + } + + // for compatibility attempt == 0 by default + TDuration GetTimeToSleep(ui32 attempt = 0) const { + return SleepDuration + NRetryPrivate::AddRandomDelta(SleepRandomDelta) + NRetryPrivate::AddIncrement(attempt, SleepIncrement) + NRetryPrivate::AddExponentialMultiplier(attempt, SleepExponentialMultiplier); + } + + static TRetryOptions Count(ui32 retryCount) { + return TRetryOptions(retryCount); + } + + static TRetryOptions Default() { + return TRetryOptions(); + } + + static TRetryOptions NoRetry() { + return TRetryOptions(0); + } +}; + +TRetryOptions MakeRetryOptions(const NRetry::TRetryOptionsPB& retryOptions); + + +namespace NRetryDetails { + +template +class TRetryOptionsPolicy : public IRetryPolicy { +public: + explicit TRetryOptionsPolicy(const TRetryOptions& opts) + : Opts(opts) + { + } + + using IRetryState = typename IRetryPolicy::IRetryState; + + class TRetryState : public IRetryState { + public: + explicit TRetryState(const TRetryOptions& opts) + : Opts(opts) + { + } + + std::optional GetNextRetryDelay(const TException&) override { + if (Attempt == Opts.RetryCount) { + return std::nullopt; + } + return Opts.GetTimeToSleep(Attempt++); + } + + private: + const TRetryOptions Opts; + size_t Attempt = 0; + }; + + typename IRetryState::TPtr CreateRetryState() const override { + return std::make_unique(Opts); + } + +private: + const TRetryOptions Opts; +}; + +} // namespace NRetryDetails + +template +typename IRetryPolicy::TPtr MakeRetryPolicy(const TRetryOptions& opts) { + return std::make_shared>(opts); +} + +template +typename IRetryPolicy::TPtr MakeRetryPolicy(const NRetry::TRetryOptionsPB& opts) { + return MakeRetryPolicy(MakeRetryOptions(opts)); +} + +template +std::optional DoWithRetry(std::function func, const typename IRetryPolicy::TPtr& retryPolicy, bool throwLast = true, std::function onFail = {}, std::function sleepFunction = {}) { + typename IRetryPolicy::IRetryState::TPtr retryState; + while (true) { + try { + return func(); + } catch (const TException& ex) { + if (onFail) { + onFail(ex); + } + + if (!retryState) { + retryState = retryPolicy->CreateRetryState(); + } + + if (const std::optional delay = retryState->GetNextRetryDelay(ex)) { + if (*delay) { + if (sleepFunction) { + sleepFunction(*delay); + } else { + Sleep(*delay); + } + } + } else { + if (throwLast) { + throw; + } + break; + } + } + } + return std::nullopt; +} + +template +std::optional DoWithRetry(std::function func, std::function onFail, TRetryOptions retryOptions, bool throwLast = true) { + return DoWithRetry(std::move(func), MakeRetryPolicy(retryOptions), throwLast, std::move(onFail), retryOptions.SleepFunction); +} + +template +std::optional DoWithRetry(std::function func, TRetryOptions retryOptions, bool throwLast = true) { + return DoWithRetry(std::move(func), MakeRetryPolicy(retryOptions), throwLast, {}, retryOptions.SleepFunction); +} + +template +bool DoWithRetry(std::function func, const typename IRetryPolicy::TPtr& retryPolicy, bool throwLast = true, std::function onFail = {}, std::function sleepFunction = {}) { + auto f = [&]() { + func(); + return nullptr; + }; + return DoWithRetry(f, retryPolicy, throwLast, std::move(onFail), std::move(sleepFunction)).Defined(); +} + +template +bool DoWithRetry(std::function func, std::function onFail, TRetryOptions retryOptions, bool throwLast) { + return DoWithRetry(std::move(func), MakeRetryPolicy(retryOptions), throwLast, onFail, retryOptions.SleepFunction); +} + +template +bool DoWithRetry(std::function func, TRetryOptions retryOptions, bool throwLast = true) { + return DoWithRetry(std::move(func), MakeRetryPolicy(retryOptions), throwLast, {}, retryOptions.SleepFunction); +} + +template +TRetCode DoWithRetryOnRetCode(std::function func, const typename IRetryPolicy::TPtr& retryPolicy, std::function sleepFunction = {}) { + auto retryState = retryPolicy->CreateRetryState(); + while (true) { + TRetCode code = func(); + if (const std::optional delay = retryState->GetNextRetryDelay(code)) { + if (*delay) { + if (sleepFunction) { + sleepFunction(*delay); + } else { + Sleep(*delay); + } + } + } else { + return code; + } + } +} + +bool DoWithRetryOnRetCode(std::function func, TRetryOptions retryOptions); + +Y_DECLARE_PODTYPE(TRetryOptions); diff --git a/ydb/public/sdk/cpp/src/library/retry/retry_policy_ut.cpp b/ydb/public/sdk/cpp/src/library/retry/retry_policy_ut.cpp new file mode 100644 index 000000000000..32a5896e0829 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/retry_policy_ut.cpp @@ -0,0 +1,108 @@ +#include "retry_policy.h" + +#include + +Y_UNIT_TEST_SUITE(RetryPolicy) { + Y_UNIT_TEST(NoRetryPolicy) { + auto policy = IRetryPolicy::GetNoRetryPolicy(); + UNIT_ASSERT(!policy->CreateRetryState()->GetNextRetryDelay(42)); + } + + using ITestPolicy = IRetryPolicy; + + ERetryErrorClass ErrorClassFunction(ERetryErrorClass err) { + return err; + } + +#define ASSERT_INTERVAL(from, to, val) { \ + auto v = val; \ + UNIT_ASSERT(v); \ + UNIT_ASSERT_GE_C(*v, from, *v); \ + UNIT_ASSERT_LE_C(*v, to, *v); \ + } + + Y_UNIT_TEST(FixedIntervalPolicy) { + auto policy = ITestPolicy::GetFixedIntervalPolicy(ErrorClassFunction, TDuration::MilliSeconds(100), TDuration::Seconds(100)); + auto state = policy->CreateRetryState(); + for (int i = 0; i < 5; ++i) { + ASSERT_INTERVAL(TDuration::MilliSeconds(50), TDuration::MilliSeconds(100), state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + ASSERT_INTERVAL(TDuration::Seconds(50), TDuration::Seconds(100), state->GetNextRetryDelay(ERetryErrorClass::LongRetry)); + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::NoRetry)); + } + } + + Y_UNIT_TEST(ExponentialBackoffPolicy) { + auto policy = ITestPolicy::GetExponentialBackoffPolicy(ErrorClassFunction, TDuration::MilliSeconds(100), TDuration::Seconds(100), TDuration::Seconds(500)); + auto state = policy->CreateRetryState(); + + // Step 1 + ASSERT_INTERVAL(TDuration::MilliSeconds(50), TDuration::MilliSeconds(100), state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + + // Step 2 + ASSERT_INTERVAL(TDuration::Seconds(50), TDuration::Seconds(100), state->GetNextRetryDelay(ERetryErrorClass::LongRetry)); + + // Step 3 + ASSERT_INTERVAL(TDuration::Seconds(100), TDuration::Seconds(200), state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + + // Step 4 + ASSERT_INTERVAL(TDuration::Seconds(200), TDuration::Seconds(400), state->GetNextRetryDelay(ERetryErrorClass::LongRetry)); + + // Step 5. Max delay + ASSERT_INTERVAL(TDuration::Seconds(250), TDuration::Seconds(500), state->GetNextRetryDelay(ERetryErrorClass::LongRetry)); + ASSERT_INTERVAL(TDuration::Seconds(250), TDuration::Seconds(500), state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + + // No retry + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::NoRetry)); + } + + void TestMaxRetries(bool exponentialBackoff) { + ITestPolicy::TPtr policy; + if (exponentialBackoff) { + policy = ITestPolicy::GetExponentialBackoffPolicy(ErrorClassFunction, TDuration::MilliSeconds(10), TDuration::MilliSeconds(200), TDuration::Seconds(30), 3); + } else { + policy = ITestPolicy::GetFixedIntervalPolicy(ErrorClassFunction, TDuration::MilliSeconds(100), TDuration::MilliSeconds(300), 3); + } + auto state = policy->CreateRetryState(); + UNIT_ASSERT(state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + UNIT_ASSERT(state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + UNIT_ASSERT(state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + } + + void TestMaxTime(bool exponentialBackoff) { + ITestPolicy::TPtr policy; + const TDuration maxDelay = TDuration::Seconds(2); + if (exponentialBackoff) { + policy = ITestPolicy::GetExponentialBackoffPolicy(ErrorClassFunction, TDuration::MilliSeconds(10), TDuration::MilliSeconds(200), TDuration::Seconds(30), 100500, maxDelay); + } else { + policy = ITestPolicy::GetFixedIntervalPolicy(ErrorClassFunction, TDuration::MilliSeconds(100), TDuration::MilliSeconds(300), 100500, maxDelay); + } + const TInstant start = TInstant::Now(); + auto state = policy->CreateRetryState(); + for (int i = 0; i < 3; ++i) { + auto delay = state->GetNextRetryDelay(ERetryErrorClass::ShortRetry); + const TInstant now = TInstant::Now(); + UNIT_ASSERT(delay || now - start >= maxDelay); + } + Sleep(maxDelay); + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + UNIT_ASSERT(!state->GetNextRetryDelay(ERetryErrorClass::ShortRetry)); + } + + Y_UNIT_TEST(MaxRetriesExponentialBackoff) { + TestMaxRetries(true); + } + + Y_UNIT_TEST(MaxRetriesFixedInterval) { + TestMaxRetries(false); + } + + Y_UNIT_TEST(MaxTimeExponentialBackoff) { + TestMaxTime(true); + } + + Y_UNIT_TEST(MaxTimeFixedInterval) { + TestMaxTime(false); + } +} diff --git a/ydb/public/sdk/cpp/src/library/retry/retry_ut.cpp b/ydb/public/sdk/cpp/src/library/retry/retry_ut.cpp new file mode 100644 index 000000000000..92153e987ebc --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/retry_ut.cpp @@ -0,0 +1,117 @@ +#include "retry.h" + +#include + +namespace { + class TDoOnSecondOrThrow { + public: + ui32 operator()() { + if (attempt++ != 1) { + throw yexception(); + } + return 42; + } + + private: + ui32 attempt = 0; + }; + + class TDoOnSecondOrFail { + public: + bool operator()() { + return (attempt++ == 1); + } + + private: + ui32 attempt = 0; + }; +} + +Y_UNIT_TEST_SUITE(Retry) { + Y_UNIT_TEST(RetryOnExceptionSuccess) { + UNIT_ASSERT_NO_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, TRetryOptions(1, TDuration::Zero()))); + } + Y_UNIT_TEST(RetryOnExceptionSuccessWithOnFail) { + ui32 value = 0; + std::function cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT_NO_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, cb, TRetryOptions(1, TDuration::Zero()), true)); + UNIT_ASSERT_EQUAL(value, 1); + } + Y_UNIT_TEST(RetryOnExceptionFail) { + UNIT_ASSERT_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, TRetryOptions(0, TDuration::Zero())), yexception); + } + Y_UNIT_TEST(RetryOnExceptionFailWithOnFail) { + ui32 value = 0; + std::function cb = [&value](const yexception&) { value += 1; }; + UNIT_ASSERT_EXCEPTION(DoWithRetry(TDoOnSecondOrThrow{}, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); + UNIT_ASSERT_EQUAL(value, 1); + } + + Y_UNIT_TEST(RetryOnExceptionSuccessWithValue) { + std::function f = TDoOnSecondOrThrow{}; + UNIT_ASSERT(42 == *DoWithRetry(f, TRetryOptions(1, TDuration::Zero()), false)); + } + Y_UNIT_TEST(RetryOnExceptionSuccessWithValueWithOnFail) { + ui32 value = 0; + std::function f = TDoOnSecondOrThrow{}; + std::function cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(42 == *DoWithRetry(f, cb, TRetryOptions(1, TDuration::Zero()), false)); + UNIT_ASSERT_EQUAL(value, 1); + } + Y_UNIT_TEST(RetryOnExceptionFailWithValue) { + std::function f = TDoOnSecondOrThrow{}; + UNIT_ASSERT(!DoWithRetry(f, TRetryOptions(0, TDuration::Zero()), false).Defined()); + } + Y_UNIT_TEST(RetryOnExceptionFailWithValueWithOnFail) { + ui32 value = 0; + std::function f = TDoOnSecondOrThrow{}; + std::function cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(!DoWithRetry(f, cb, TRetryOptions(0, TDuration::Zero()), false).Defined()); + UNIT_ASSERT_EQUAL(value, 1); + } + + Y_UNIT_TEST(RetryOnExceptionSuccessWithValueAndRethrow) { + std::function f = TDoOnSecondOrThrow{}; + UNIT_ASSERT(42 == *DoWithRetry(f, TRetryOptions(1, TDuration::Zero()), true)); + } + Y_UNIT_TEST(RetryOnExceptionSuccessWithValueAndRethrowWithOnFail) { + ui32 value = 0; + std::function f = TDoOnSecondOrThrow{}; + std::function cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT(42 == *DoWithRetry(f, cb, TRetryOptions(1, TDuration::Zero()), true)); + UNIT_ASSERT_EQUAL(value, 1); + } + Y_UNIT_TEST(RetryOnExceptionFailWithValueAndRethrow) { + std::function f = TDoOnSecondOrThrow{}; + UNIT_ASSERT_EXCEPTION(DoWithRetry(f, TRetryOptions(0, TDuration::Zero()), true), yexception); + } + Y_UNIT_TEST(RetryOnExceptionFailWithValueAndRethrowWithOnFail) { + ui32 value = 0; + std::function f = TDoOnSecondOrThrow{}; + std::function cb = [&value](const yexception&){ value += 1; }; + UNIT_ASSERT_EXCEPTION(42 == *DoWithRetry(f, cb, TRetryOptions(0, TDuration::Zero()), true), yexception); + UNIT_ASSERT_EQUAL(value, 1); + } + + Y_UNIT_TEST(RetryOnRetCodeSuccess) { + UNIT_ASSERT(true == DoWithRetryOnRetCode(TDoOnSecondOrFail{}, TRetryOptions(1, TDuration::Zero()))); + } + Y_UNIT_TEST(RetryOnRetCodeFail) { + UNIT_ASSERT(false == DoWithRetryOnRetCode(TDoOnSecondOrFail{}, TRetryOptions(0, TDuration::Zero()))); + } + Y_UNIT_TEST(MakeRetryOptionsFromProto) { + NRetry::TRetryOptionsPB protoOptions; + protoOptions.SetMaxTries(1); + protoOptions.SetInitialSleepMs(2); + protoOptions.SetSleepIncrementMs(3); + protoOptions.SetRandomDeltaMs(4); + protoOptions.SetExponentalMultiplierMs(5); + + const TRetryOptions options = MakeRetryOptions(protoOptions); + UNIT_ASSERT_EQUAL(options.RetryCount, 1); + UNIT_ASSERT_EQUAL(options.SleepDuration, TDuration::MilliSeconds(2)); + UNIT_ASSERT_EQUAL(options.SleepIncrement, TDuration::MilliSeconds(3)); + UNIT_ASSERT_EQUAL(options.SleepRandomDelta, TDuration::MilliSeconds(4)); + UNIT_ASSERT_EQUAL(options.SleepExponentialMultiplier, TDuration::MilliSeconds(5)); + } +} diff --git a/ydb/public/sdk/cpp/src/library/retry/utils.cpp b/ydb/public/sdk/cpp/src/library/retry/utils.cpp new file mode 100644 index 000000000000..24d943f5185b --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/utils.cpp @@ -0,0 +1,20 @@ +#include "utils.h" + +#include + +TDuration NRetryPrivate::AddRandomDelta(TDuration maxDelta) { + if (maxDelta == TDuration::Zero()) { + return TDuration::Zero(); + } + + const TDuration delta = TDuration::MicroSeconds(RandomNumber(2 * maxDelta.MicroSeconds())); + return delta - maxDelta; +} + +TDuration NRetryPrivate::AddIncrement(ui32 attempt, TDuration increment) { + return TDuration::MicroSeconds(attempt * increment.MicroSeconds()); +} + +TDuration NRetryPrivate::AddExponentialMultiplier(ui32 attempt, TDuration exponentialMultiplier) { + return TDuration::MicroSeconds((1ull << Min(63u, attempt)) * exponentialMultiplier.MicroSeconds()); +} diff --git a/ydb/public/sdk/cpp/src/library/retry/utils.h b/ydb/public/sdk/cpp/src/library/retry/utils.h new file mode 100644 index 000000000000..a8fd3d1a8988 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/retry/utils.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace NRetryPrivate { + TDuration AddRandomDelta(TDuration delta); + TDuration AddIncrement(ui32 attempt, TDuration increment); + TDuration AddExponentialMultiplier(ui32 attempt, TDuration exponentialMultiplier); + +} diff --git a/ydb/public/sdk/cpp/src/library/string_utils/base64/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/string_utils/base64/CMakeLists.txt new file mode 100644 index 000000000000..372d7c49ce5c --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/base64/CMakeLists.txt @@ -0,0 +1,12 @@ +_ydb_sdk_add_library(string_utils-base64) + +target_link_libraries(string_utils-base64 PUBLIC + yutil + aklomp::base64 +) + +target_sources(string_utils-base64 PRIVATE + ${YDB_SDK_SOURCE_DIR}/src/library/string_utils/base64/base64.cpp +) + +_ydb_sdk_install_targets(TARGETS string_utils-base64) diff --git a/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.cpp b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.cpp new file mode 100644 index 000000000000..1b18220b496b --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.cpp @@ -0,0 +1,224 @@ +#include "base64.h" + +#include +#include +#include +#include + +#include + +static const char base64_etab_std[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const char base64_bkw[] = { + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', // 0..15 + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', // 16..31 + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\76', '\0', '\76', '\0', '\77', // 32.47 + '\64', '\65', '\66', '\67', '\70', '\71', '\72', '\73', '\74', '\75', '\0', '\0', '\0', '\0', '\0', '\0', // 48..63 + '\0', '\0', '\1', '\2', '\3', '\4', '\5', '\6', '\7', '\10', '\11', '\12', '\13', '\14', '\15', '\16', // 64..79 + '\17', '\20', '\21', '\22', '\23', '\24', '\25', '\26', '\27', '\30', '\31', '\0', '\0', '\0', '\0', '\77', // 80..95 + '\0', '\32', '\33', '\34', '\35', '\36', '\37', '\40', '\41', '\42', '\43', '\44', '\45', '\46', '\47', '\50', // 96..111 + '\51', '\52', '\53', '\54', '\55', '\56', '\57', '\60', '\61', '\62', '\63', '\0', '\0', '\0', '\0', '\0', // 112..127 + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', // 128..143 + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0'}; + +static_assert(Y_ARRAY_SIZE(base64_bkw) == 256, "wrong size"); + +// Base64 for url encoding, RFC3548 +static const char base64_etab_url[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +static inline unsigned char GetBase64EncodedIndex0(unsigned char octet0) { + return (octet0 >> 2); +} + +static inline unsigned char GetBase64EncodedIndex1(unsigned char octet0, unsigned char octet1) { + return (((octet0 << 4) & 0x30) | ((octet1 >> 4) & 0x0f)); +} + +static inline unsigned char GetBase64EncodedIndex2(unsigned char octet1, unsigned char octet2) { + return (((octet1 << 2) & 0x3c) | ((octet2 >> 6) & 0x03)); +} + +static inline unsigned char GetBase64EncodedIndex3(unsigned char octet2) { + return (octet2 & 0x3f); +} + +template +static inline char* Base64EncodeImpl(char* outstr, const unsigned char* instr, size_t len) { + const char* const base64_etab = (urlVersion ? base64_etab_url : base64_etab_std); + const char pad = (urlVersion ? ',' : '='); + + size_t idx = 0; + + while (idx + 2 < len) { + *outstr++ = base64_etab[GetBase64EncodedIndex0(instr[idx])]; + *outstr++ = base64_etab[GetBase64EncodedIndex1(instr[idx], instr[idx + 1])]; + *outstr++ = base64_etab[GetBase64EncodedIndex2(instr[idx + 1], instr[idx + 2])]; + *outstr++ = base64_etab[GetBase64EncodedIndex3(instr[idx + 2])]; + idx += 3; + } + if (idx < len) { + *outstr++ = base64_etab[GetBase64EncodedIndex0(instr[idx])]; + if (idx + 1 < len) { + *outstr++ = base64_etab[GetBase64EncodedIndex1(instr[idx], instr[idx + 1])]; + *outstr++ = base64_etab[GetBase64EncodedIndex2(instr[idx + 1], '\0')]; + } else { + *outstr++ = base64_etab[GetBase64EncodedIndex1(instr[idx], '\0')]; + if (usePadding) { + *outstr++ = pad; + } + } + if (usePadding) { + *outstr++ = pad; + } + } + *outstr = 0; + + return outstr; +} + +static char* Base64EncodePlain(char* outstr, const unsigned char* instr, size_t len) { + return Base64EncodeImpl(outstr, instr, len); +} + +char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len) { + return Base64EncodeImpl(outstr, instr, len); +} + +char* Base64EncodeUrlNoPadding(char* outstr, const unsigned char* instr, size_t len) { + return Base64EncodeImpl(outstr, instr, len); +} + +inline void uudecode_1(char* dst, unsigned char* src) { + dst[0] = char((base64_bkw[src[0]] << 2) | (base64_bkw[src[1]] >> 4)); + dst[1] = char((base64_bkw[src[1]] << 4) | (base64_bkw[src[2]] >> 2)); + dst[2] = char((base64_bkw[src[2]] << 6) | base64_bkw[src[3]]); +} + +static size_t Base64DecodePlain(void* dst, const char* b, const char* e) { + size_t n = 0; + while (b < e) { + uudecode_1((char*)dst + n, (unsigned char*)b); + + b += 4; + n += 3; + } + + if (n > 0) { + if (b[-1] == ',' || b[-1] == '=') { + n--; + + if (b[-2] == ',' || b[-2] == '=') { + n--; + } + } + } + + return n; +} + +// Table for Base64StrictDecode +static const char base64_bkw_strict[] = + "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100" + "\100\100\100\100\100\100\100\100\100\100\100\76\101\76\100\77\64\65\66\67\70\71\72\73\74\75\100\100\100\101\100\100" + "\100\0\1\2\3\4\5\6\7\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\100\100\100\100\77" + "\100\32\33\34\35\36\37\40\41\42\43\44\45\46\47\50\51\52\53\54\55\56\57\60\61\62\63\100\100\100\100\100" + "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100" + "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100" + "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100" + "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100"; + +size_t Base64StrictDecode(void* out, const char* b, const char* e) { + char* dst = (char*)out; + const unsigned char* src = (unsigned char*)b; + const unsigned char* const end = (unsigned char*)e; + + Y_ENSURE(!((e - b) % 4), "incorrect input length for base64 decode"); + + while (src < end) { + const char zeroth = base64_bkw_strict[src[0]]; + const char first = base64_bkw_strict[src[1]]; + const char second = base64_bkw_strict[src[2]]; + const char third = base64_bkw_strict[src[3]]; + + constexpr char invalid = 64; + constexpr char padding = 65; + if (Y_UNLIKELY(zeroth == invalid || first == invalid || + second == invalid || third == invalid || + zeroth == padding || first == padding)) + { + ythrow yexception() << "invalid character in input"; + } + + dst[0] = char((zeroth << 2) | (first >> 4)); + dst[1] = char((first << 4) | (second >> 2)); + dst[2] = char((second << 6) | third); + + src += 4; + dst += 3; + + if (src[-1] == ',' || src[-1] == '=') { + --dst; + + if (src[-2] == ',' || src[-2] == '=') { + --dst; + } + } else if (Y_UNLIKELY(src[-2] == ',' || src[-2] == '=')) { + ythrow yexception() << "incorrect padding"; + } + } + + return dst - (char*)out; +} + +size_t Base64Decode(void* dst, const char* b, const char* e) { + const auto size = e - b; + Y_ENSURE(!(size % 4), "incorrect input length for base64 decode"); + if (Y_LIKELY(size < 8)) { + return Base64DecodePlain(dst, b, e); + } + + size_t outLen; + base64_decode(b, size, (char*)dst, &outLen, 0); + + return outLen; +} + +size_t Base64DecodeUneven(void* dst, const std::string_view s) { + const size_t tailSize = s.length() % 4; + if (tailSize == 0) { + return Base64Decode(dst, s.begin(), s.end()); + } + + // divide s into even part and tail and decode in two step, to avoid memory allocation + char tail[4] = {'=', '=', '=', '='}; + memcpy(tail, s.end() - tailSize, tailSize); + size_t decodedEven = s.length() > 4 ? Base64Decode(dst, s.begin(), s.end() - tailSize) : 0; + // there should not be tail of size 1 it's incorrect for 8-bit bytes + size_t decodedTail = tailSize != 1 ? Base64Decode(static_cast(dst) + decodedEven, tail, tail + 4) : 0; + return decodedEven + decodedTail; +} + +std::string Base64DecodeUneven(const std::string_view s) { + std::string ret; + ret.resize(Base64DecodeBufSize(s.size())); + size_t size = Base64DecodeUneven(const_cast(ret.data()), s); + ret.resize(size); + return ret; +} + +char* Base64Encode(char* outstr, const unsigned char* instr, size_t len) { + if (Y_LIKELY(len < 8)) { + return Base64EncodePlain(outstr, instr, len); + } + + size_t outLen; + base64_encode((char*)instr, len, outstr, &outLen, 0); + + *(outstr + outLen) = '\0'; + return outstr + outLen; +} diff --git a/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.h b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.h new file mode 100644 index 000000000000..9fe787af8ede --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64.h @@ -0,0 +1,154 @@ +#pragma once + +#include + +/* @return Size of the buffer required to decode Base64 encoded data of size `len`. + */ +constexpr size_t Base64DecodeBufSize(const size_t len) noexcept { + return (len + 3) / 4 * 3; +} + +/* Decode Base64 encoded data. Can decode both regular Base64 and Base64URL encoded data. Can decode + * only valid Base64[URL] data, behaviour for invalid data is unspecified. + * + * @throws Throws exception in case of incorrect padding. + * + * @param dst memory for writing output. + * @param b pointer to the beginning of base64 encoded string. + * @param a pointer to the end of base64 encoded string + * + * @return Return number of bytes decoded. + */ +size_t Base64Decode(void* dst, const char* b, const char* e); + +inline std::string_view Base64Decode(const std::string_view src, void* dst) { + return std::string_view(static_cast(dst), Base64Decode(dst, src.begin(), src.end())); +} + +inline void Base64Decode(const std::string_view src, std::string& dst) { + dst.resize(Base64DecodeBufSize(src.size())); + dst.resize(Base64Decode(src, dst.data()).size()); +} + +//WARNING: can process not whole input silently, use Base64StrictDecode instead of this function +inline std::string Base64Decode(const std::string_view s) { + std::string ret; + Base64Decode(s, ret); + return ret; +} + +/// +/// @brief Decodes Base64 string with strict verification +/// of invalid symbols, also tries to decode Base64 string with padding +/// inside. +// +/// @throws Throws exceptions on inputs which contain invalid symbols +/// or incorrect padding. +/// @{ +/// +/// @param b a pointer to the beginning of base64 encoded string. +/// @param e a pointer to the end of base64 encoded string. +/// @param dst memory for writing output. +/// +/// @return Returns number of bytes decoded. +/// +size_t Base64StrictDecode(void* dst, const char* b, const char* e); + +/// +/// @param src a base64 encoded string. +/// @param dst an pointer to allocated memory +/// for writing result. +/// +/// @return Returns dst wrapped into std::string_view. +/// +inline std::string_view Base64StrictDecode(const std::string_view src, void* dst) { + return std::string_view(static_cast(dst), Base64StrictDecode(dst, src.begin(), src.end())); +} + +/// +/// @param src a base64 encoded string. +/// @param dst a decoded string. +/// +inline void Base64StrictDecode(const std::string_view src, std::string& dst) { + dst.resize(Base64DecodeBufSize(src.size())); + dst.resize(Base64StrictDecode(src, dst.data()).size()); +} + +/// +/// @param src a base64 encoded string. +/// +/// @returns a decoded string. +/// +inline std::string Base64StrictDecode(const std::string_view src) { + std::string ret; + Base64StrictDecode(src, ret); + return ret; +} +/// @} + +/// Works with strings which length is not divisible by 4. +std::string Base64DecodeUneven(const std::string_view s); +size_t Base64DecodeUneven(void* dst, const std::string_view s); + +//encode +constexpr size_t Base64EncodeBufSize(const size_t len) noexcept { + return (len + 2) / 3 * 4 + 1; +} + +char* Base64Encode(char* outstr, const unsigned char* instr, size_t len); +char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len); + +/// Make base64 string which stay unchaged after applying 'urlencode' function +/// as it doesn't contain character, which cannot be used in urls +/// @param outstr a pointer to allocated memory for writing result. +/// @param instr a to buffer to encode +/// @param len size of instr buffer +/// +/// @return Returns pointer to last symbol in outstr buffer. +/// +char* Base64EncodeUrlNoPadding(char* outstr, const unsigned char* instr, size_t len); + +inline std::string_view Base64Encode(const std::string_view src, void* output) { + return std::string_view(static_cast(output), Base64Encode(static_cast(output), reinterpret_cast(src.data()), src.size())); +} + +inline std::string_view Base64EncodeUrl(const std::string_view src, void* output) { + return std::string_view(static_cast(output), Base64EncodeUrl(static_cast(output), reinterpret_cast(src.data()), src.size())); +} + +inline std::string_view Base64EncodeUrlNoPadding(const std::string_view src, void* output) { + return std::string_view(static_cast(output), Base64EncodeUrlNoPadding(static_cast(output), reinterpret_cast(src.data()), src.size())); +} + +inline void Base64Encode(const std::string_view src, std::string& dst) { + dst.resize(Base64EncodeBufSize(src.size())); + dst.resize(Base64Encode(src, dst.data()).size()); +} + +inline void Base64EncodeUrl(const std::string_view src, std::string& dst) { + dst.resize(Base64EncodeBufSize(src.size())); + dst.resize(Base64EncodeUrl(src, dst.data()).size()); +} + +inline void Base64EncodeUrlNoPadding(const std::string_view src, std::string& dst) { + dst.resize(Base64EncodeBufSize(src.size())); + dst.resize(Base64EncodeUrlNoPadding(src, dst.data()).size()); +} + +inline std::string Base64Encode(const std::string_view s) { + std::string ret; + Base64Encode(s, ret); + return ret; +} + +inline std::string Base64EncodeUrl(const std::string_view s) { + std::string ret; + Base64EncodeUrl(s, ret); + return ret; +} + +inline std::string Base64EncodeUrlNoPadding(const std::string_view s) { + std::string ret; + Base64EncodeUrlNoPadding(s, ret); + return ret; +} diff --git a/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_decode_uneven_ut.cpp b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_decode_uneven_ut.cpp new file mode 100644 index 000000000000..0c9e453008f9 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_decode_uneven_ut.cpp @@ -0,0 +1,46 @@ +#include + +#include + +Y_UNIT_TEST_SUITE(TBase64DecodeUneven) { + Y_UNIT_TEST(Base64DecodeUneven) { + const std::string wikipedia_slogan = + "Man is distinguished, not only by his reason, " + "but by this singular passion from other animals, which is a lust of the " + "mind, that by a perseverance of delight in the continued and " + "indefatigable generation of knowledge, exceeds the short " + "vehemence of any carnal pleasure."; + const std::string encoded = + "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0" + "aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1" + "c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0" + "aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdl" + "LCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4="; + + UNIT_ASSERT_VALUES_EQUAL(encoded, Base64Encode(wikipedia_slogan)); + UNIT_ASSERT_VALUES_EQUAL(wikipedia_slogan, Base64DecodeUneven(encoded)); + + const std::string encoded_url1 = + "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0" + "aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1" + "c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0" + "aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdl" + "LCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4,"; + const std::string encoded_url2 = + "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0" + "aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1" + "c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0" + "aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdl" + "LCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4"; + UNIT_ASSERT_VALUES_EQUAL(wikipedia_slogan, Base64DecodeUneven(encoded_url1)); + UNIT_ASSERT_VALUES_EQUAL(wikipedia_slogan, Base64DecodeUneven(encoded_url2)); + + const std::string lp = "Linkin Park"; + UNIT_ASSERT_VALUES_EQUAL(lp, Base64DecodeUneven(Base64Encode(lp))); + UNIT_ASSERT_VALUES_EQUAL(lp, Base64DecodeUneven(Base64EncodeUrl(lp))); + + const std::string dp = "ADP GmbH\nAnalyse Design & Programmierung\nGesellschaft mit beschränkter Haftung"; + UNIT_ASSERT_VALUES_EQUAL(dp, Base64DecodeUneven(Base64Encode(dp))); + UNIT_ASSERT_VALUES_EQUAL(dp, Base64DecodeUneven(Base64EncodeUrl(dp))); + } +} diff --git a/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_ut.cpp b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_ut.cpp new file mode 100644 index 000000000000..811066918a82 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/base64/base64_ut.cpp @@ -0,0 +1,539 @@ +#include "base64.h" + +#include +#include +#include +#include +#include +#include + +#include + + +#include +#include +#include + +#include + +using namespace std::string_view_literals; + +#define BASE64_UT_DECLARE_BASE64_IMPL(prefix, encFunction, decFunction) \ + Y_DECLARE_UNUSED \ + static size_t prefix##Base64Decode(void* dst, const char* b, const char* e) { \ + const auto size = e - b; \ + Y_ENSURE(!(size % 4), "incorrect input length for base64 decode"); \ + \ + size_t outLen; \ + decFunction(b, size, (char*)dst, &outLen); \ + return outLen; \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline std::string_view prefix##Base64Decode(const std::string_view& src, void* dst) { \ + return std::string_view((const char*)dst, ::NB64Etalon::prefix##Base64Decode(dst, src.begin(), src.end())); \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline void prefix##Base64Decode(const std::string_view& src, std::string& dst) { \ + dst.resize(Base64DecodeBufSize(src.size())); \ + dst.resize(::NB64Etalon::prefix##Base64Decode(src, dst.begin()).size()); \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline std::string prefix##Base64Decode(const std::string_view& s) { \ + std::string ret; \ + prefix##Base64Decode(s, ret); \ + return ret; \ + } \ + \ + Y_DECLARE_UNUSED \ + static char* prefix##Base64Encode(char* outstr, const unsigned char* instr, size_t len) { \ + size_t outLen; \ + encFunction((char*)instr, len, outstr, &outLen); \ + *(outstr + outLen) = '\0'; \ + return outstr + outLen; \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline std::string_view prefix##Base64Encode(const std::string_view& src, void* tmp) { \ + return std::string_view((const char*)tmp, ::NB64Etalon::prefix##Base64Encode((char*)tmp, (const unsigned char*)src.data(), src.size())); \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline void prefix##Base64Encode(const std::string_view& src, std::string& dst) { \ + dst.resize(Base64EncodeBufSize(src.size())); \ + dst.resize(::NB64Etalon::prefix##Base64Encode(src, dst.begin()).size()); \ + } \ + \ + Y_DECLARE_UNUSED \ + static inline std::string prefix##Base64Encode(const std::string_view& s) { \ + std::string ret; \ + prefix##Base64Encode(s, ret); \ + return ret; \ + } + +namespace NB64Etalon { + BASE64_UT_DECLARE_BASE64_IMPL(PLAIN32, plain32_base64_encode, plain32_base64_decode) + BASE64_UT_DECLARE_BASE64_IMPL(PLAIN64, plain64_base64_encode, plain64_base64_decode) + BASE64_UT_DECLARE_BASE64_IMPL(NEON32, neon32_base64_encode, neon32_base64_decode) + BASE64_UT_DECLARE_BASE64_IMPL(NEON64, neon64_base64_encode, neon64_base64_decode) + BASE64_UT_DECLARE_BASE64_IMPL(AVX2, avx2_base64_encode, avx2_base64_decode) + BASE64_UT_DECLARE_BASE64_IMPL(SSSE3, ssse3_base64_encode, ssse3_base64_decode) + +#undef BASE64_UT_DECLARE_BASE64_IMPL + + struct TImpls { + enum EImpl : size_t { + PLAIN32_IMPL, + PLAIN64_IMPL, + NEON32_IMPL, + NEON64_IMPL, + AVX2_IMPL, + SSSE3_IMPL, + MAX_IMPL + }; + + using TEncodeF = void (*)(const std::string_view&, std::string&); + using TDecodeF = void (*)(const std::string_view&, std::string&); + + struct TImpl { + TEncodeF Encode = nullptr; + TDecodeF Decode = nullptr; + }; + + std::array Impl; + + TImpls() { + Impl[PLAIN32_IMPL].Encode = PLAIN32Base64Encode; + Impl[PLAIN32_IMPL].Decode = PLAIN32Base64Decode; + Impl[PLAIN64_IMPL].Encode = PLAIN64Base64Encode; + Impl[PLAIN64_IMPL].Decode = PLAIN64Base64Decode; +#if defined(_arm32_) + Impl[NEON32_IMPL].Encode = NEON32Base64Encode; + Impl[NEON32_IMPL].Decode = NEON32Base64Decode; +#elif defined(_arm64_) + Impl[NEON64_IMPL].Encode = NEON64Base64Encode; + Impl[NEON64_IMPL].Decode = NEON64Base64Decode; +#elif defined(_x86_64_) + if (NX86::HaveSSSE3()) { + Impl[SSSE3_IMPL].Encode = SSSE3Base64Encode; + Impl[SSSE3_IMPL].Decode = SSSE3Base64Decode; + } + + if (NX86::HaveAVX2()) { + Impl[AVX2_IMPL].Encode = AVX2Base64Encode; + Impl[AVX2_IMPL].Decode = AVX2Base64Decode; + } +#else + ythrow yexception() << "Failed to identify the platform"; +#endif + } + }; + + TImpls GetImpls() { + static const TImpls IMPLS; + return IMPLS; + } +} + +template <> +void Out(IOutputStream& o, typename TTypeTraits::TFuncParam v) { + switch (v) { + case NB64Etalon::TImpls::PLAIN32_IMPL: + o << std::string_view{"PLAIN32"}; + return; + case NB64Etalon::TImpls::PLAIN64_IMPL: + o << std::string_view{"PLAIN64"}; + return; + case NB64Etalon::TImpls::NEON64_IMPL: + o << std::string_view{"NEON64"}; + return; + case NB64Etalon::TImpls::NEON32_IMPL: + o << std::string_view{"NEON32"}; + return; + case NB64Etalon::TImpls::SSSE3_IMPL: + o << std::string_view{"SSSE3"}; + return; + case NB64Etalon::TImpls::AVX2_IMPL: + o << std::string_view{"AVX2"}; + return; + default: + ythrow yexception() << "invalid"; + } +} + +static void TestEncodeDecodeIntoString(const std::string& plain, const std::string& encoded, const std::string& encodedUrl, const std::string& encodedUrlNoPadding) { + std::string a, b; + + Base64Encode(plain, a); + UNIT_ASSERT_VALUES_EQUAL(a, encoded); + + Base64Decode(a, b); + UNIT_ASSERT_VALUES_EQUAL(b, plain); + + Base64EncodeUrl(plain, a); + UNIT_ASSERT_VALUES_EQUAL(a, encodedUrl); + + Base64Decode(a, b); + UNIT_ASSERT_VALUES_EQUAL(b, plain); + + Base64EncodeUrlNoPadding(plain, a); + UNIT_ASSERT_VALUES_EQUAL(a, encodedUrlNoPadding); + + std::string c = Base64DecodeUneven(a); + UNIT_ASSERT_VALUES_EQUAL(c, plain); +} + +static void TestEncodeStrictDecodeIntoString(const std::string& plain, const std::string& encoded, const std::string& encodedUrl) { + std::string a, b; + + Base64Encode(plain, a); + UNIT_ASSERT_VALUES_EQUAL(a, encoded); + + Base64StrictDecode(a, b); + UNIT_ASSERT_VALUES_EQUAL(b, plain); + + Base64EncodeUrl(plain, a); + UNIT_ASSERT_VALUES_EQUAL(a, encodedUrl); + + Base64StrictDecode(a, b); + UNIT_ASSERT_VALUES_EQUAL(b, plain); +} + +Y_UNIT_TEST_SUITE(TBase64) { + Y_UNIT_TEST(TestEncode) { + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12z"), "MTJ6"); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("123"), "MTIz"); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12"), "MTI="); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("1"), "MQ=="); + } + + Y_UNIT_TEST(TestIntoString) { + { + std::string str; + for (size_t i = 0; i < 256; ++i) + str += char(i); + + const std::string base64 = + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJy" + "gpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9Q" + "UVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eH" + "l6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6Ch" + "oqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIyc" + "rLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy" + "8/T19vf4+fr7/P3+/w=="; + const std::string base64Url = + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJy" + "gpKissLS4vMDEyMzQ1Njc4OTo7PD0-P0BBQkNERUZHSElKS0xNTk9Q" + "UVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eH" + "l6e3x9fn-AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6Ch" + "oqOkpaanqKmqq6ytrq-wsbKztLW2t7i5uru8vb6_wMHCw8TFxsfIyc" + "rLzM3Oz9DR0tPU1dbX2Nna29zd3t_g4eLj5OXm5-jp6uvs7e7v8PHy" + "8_T19vf4-fr7_P3-_w,,"; + const std::string base64UrlWithoutPadding = + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJy" + "gpKissLS4vMDEyMzQ1Njc4OTo7PD0-P0BBQkNERUZHSElKS0xNTk9Q" + "UVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eH" + "l6e3x9fn-AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6Ch" + "oqOkpaanqKmqq6ytrq-wsbKztLW2t7i5uru8vb6_wMHCw8TFxsfIyc" + "rLzM3Oz9DR0tPU1dbX2Nna29zd3t_g4eLj5OXm5-jp6uvs7e7v8PHy" + "8_T19vf4-fr7_P3-_w"; + + TestEncodeDecodeIntoString(str, base64, base64Url, base64UrlWithoutPadding); + TestEncodeStrictDecodeIntoString(str, base64, base64Url); + } + + { + const std::string str = "http://yandex.ru:1234/request?param=value&lll=fff#fragment"; + + const std::string base64 = "aHR0cDovL3lhbmRleC5ydToxMjM0L3JlcXVlc3Q/cGFyYW09dmFsdWUmbGxsPWZmZiNmcmFnbWVudA=="; + const std::string base64Url = "aHR0cDovL3lhbmRleC5ydToxMjM0L3JlcXVlc3Q_cGFyYW09dmFsdWUmbGxsPWZmZiNmcmFnbWVudA,,"; + const std::string base64UrlWithoutPadding = "aHR0cDovL3lhbmRleC5ydToxMjM0L3JlcXVlc3Q_cGFyYW09dmFsdWUmbGxsPWZmZiNmcmFnbWVudA"; + + TestEncodeDecodeIntoString(str, base64, base64Url, base64UrlWithoutPadding); + TestEncodeStrictDecodeIntoString(str, base64, base64Url); + } + } + + Y_UNIT_TEST(TestDecode) { + UNIT_ASSERT_EXCEPTION(Base64Decode("a"), yexception); + UNIT_ASSERT_EXCEPTION(Base64StrictDecode("a"), yexception); + + UNIT_ASSERT_VALUES_EQUAL(Base64Decode(""), ""); + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode(""), ""); + + UNIT_ASSERT_VALUES_EQUAL(Base64Decode("MTI="), "12"); + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode("MTI="), "12"); + + UNIT_ASSERT_VALUES_EQUAL(Base64Decode("QQ=="), "A"); + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode("QQ=="), "A"); + + UNIT_ASSERT_EXCEPTION(Base64StrictDecode("M=I="), yexception); + + UNIT_ASSERT_VALUES_EQUAL(Base64Decode("dnluZHg="), "vyndx"); + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode("dnluZHg="), "vyndx"); + + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode("dnluZHg=dmlkZW8="), "vyndxvideo"); + + UNIT_ASSERT_EXCEPTION(Base64StrictDecode("aHR0cDovL2ltZy5tZWdhLXBvcm5vLnJ1Lw=a"), yexception); + + UNIT_ASSERT_EXCEPTION(Base64StrictDecode("aHh=="), yexception); + UNIT_ASSERT_EXCEPTION(Base64StrictDecode("\1\1\1\2"), yexception); + } + + Y_UNIT_TEST(TestDecodeUneven) { + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven(""), ""); + + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("YWFh"), "aaa"); + + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("MTI="), "12"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("MTI,"), "12"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("MTI"), "12"); + + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("QQ=="), "A"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("QQ,,"), "A"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("QQ"), "A"); + + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("dnluZHg="), "vyndx"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("dnluZHg,"), "vyndx"); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven("dnluZHg"), "vyndx"); + } + + Y_UNIT_TEST(TestDecodeRandom) { + std::string input; + constexpr size_t testSize = 240000; + for (size_t i = 0; i < testSize; ++i) { + input.push_back(rand() % 256); + } + std::string output; + std::string encoded = Base64Encode(input); + std::string encodedUrl = std::string::Uninitialized(Base64EncodeBufSize(input.length())); + Base64EncodeUrlNoPadding(input, encodedUrl); + UNIT_ASSERT_VALUES_EQUAL(Base64Decode(encoded), input); + UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode(encoded), input); + UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven(encodedUrl), input); + } + + Y_UNIT_TEST(TestAllPossibleOctets) { + const std::string x("\0\x01\x02\x03\x04\x05\x06\x07\b\t\n\x0B\f\r\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7F"sv); + const std::string xEnc = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn8="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestTwoPaddingCharacters) { + const std::string x("a"); + const std::string xEnc = "YQ=="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestOnePaddingCharacter) { + const std::string x("aa"); + const std::string xEnc = "YWE="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestNoPaddingCharacters) { + const std::string x("aaa"); + const std::string xEnc = "YWFh"; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestTrailingZero) { + const std::string x("foo\0"sv); + const std::string xEnc = "Zm9vAA=="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestTwoTrailingZeroes) { + const std::string x("foo\0\0"sv); + const std::string xEnc = "Zm9vAAA="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestZero) { + const std::string x("\0"sv); + const std::string xEnc = "AA=="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestSymbolsAfterZero) { + const std::string x("\0a"sv); + const std::string xEnc = "AGE="; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestEmptyString) { + const std::string x = ""; + const std::string xEnc = ""; + const std::string y = Base64Decode(xEnc); + const std::string yEnc = Base64Encode(x); + UNIT_ASSERT_VALUES_EQUAL(x, y); + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + } + + Y_UNIT_TEST(TestBackendsConsistencyOnRandomData) { + constexpr size_t TEST_CASES_COUNT = 1000; + constexpr size_t MAX_DATA_SIZE = 1000; + TFastRng prng{42}; + std::vector xs{TEST_CASES_COUNT}; + std::string xEnc; + std::string xDec; + std::string yEnc; + std::string yDec; + + for (auto& x : xs) { + const size_t size = prng() % MAX_DATA_SIZE; + for (size_t j = 0; j < size; ++j) { + x += static_cast(prng() % 256); + } + } + + static const auto IMPLS = NB64Etalon::GetImpls(); + for (size_t i = 0; i < static_cast(NB64Etalon::TImpls::MAX_IMPL); ++i) { + for (size_t j = 0; j < static_cast(NB64Etalon::TImpls::MAX_IMPL); ++j) { + const auto ei = static_cast(i); + const auto ej = static_cast(j); + const auto impl = IMPLS.Impl[i]; + const auto otherImpl = IMPLS.Impl[j]; + if (!impl.Encode && !impl.Decode || !otherImpl.Encode && !otherImpl.Decode) { + continue; + } + + for (const auto& x : xs) { + impl.Encode(x, xEnc); + impl.Decode(xEnc, xDec); + Y_ENSURE(x == xDec, "something is wrong with " << ei << " implementation"); + + otherImpl.Encode(x, yEnc); + otherImpl.Decode(xEnc, yDec); + Y_ENSURE(x == yDec, "something is wrong with " << ej << " implementation"); + + UNIT_ASSERT_VALUES_EQUAL(xEnc, yEnc); + UNIT_ASSERT_VALUES_EQUAL(xDec, yDec); + } + } + } + } + + Y_UNIT_TEST(TestIfEncodedDataIsZeroTerminatedOnRandomData) { + constexpr size_t TEST_CASES_COUNT = 1000; + constexpr size_t MAX_DATA_SIZE = 1000; + TFastRng prng{42}; + std::string x; + std::vector buf; + for (size_t i = 0; i < TEST_CASES_COUNT; ++i) { + const size_t size = prng() % MAX_DATA_SIZE; + x.clear(); + for (size_t j = 0; j < size; ++j) { + x += static_cast(prng() % 256); + } + + buf.assign(Base64EncodeBufSize(x.size()), Max()); + const auto* const xEncEnd = Base64Encode(buf.data(), (const unsigned char*)x.data(), x.size()); + UNIT_ASSERT_VALUES_EQUAL(*xEncEnd, '\0'); + } + } + + Y_UNIT_TEST(TestDecodeURLEncodedNoPadding) { + const auto x = "123"; + const auto xDec = Base64Decode("MTIz"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedOnePadding) { + const auto x = "12"; + const auto xDec = Base64Decode("MTI,"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedTwoPadding) { + const auto x = "1"; + const auto xDec = Base64Decode("MQ,,"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedWithoutPadding) { + const auto x = "1"; + const auto xDec = Base64DecodeUneven("MQ"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeNoPaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?a"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9h"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeOnePaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz8="); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeTwoPaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?aa"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9hYQ=="); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedNoPaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?a"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9h"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedOnePaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz8,"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeURLEncodedTwoPaddingLongString) { + const auto x = "How do I convert between big-endian and little-endian values in C++?aa"; + const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9hYQ,,"); + UNIT_ASSERT_VALUES_EQUAL(x, xDec); + } + + Y_UNIT_TEST(TestDecodeUnevenDst) { + const auto x = "How do I convert between big-endian and little-endian values in C++?aa"; + std::string b64 = "SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9hYQ"; + std::vector buf(Base64DecodeBufSize(b64.Size()), '\0'); + Base64DecodeUneven(buf.begin(), b64); + std::string res(buf.data()); + UNIT_ASSERT_VALUES_EQUAL(x, res); + } + + Y_UNIT_TEST(TestDecodeUnevenDst2) { + const auto x = "How do I convert between big-endian and little-endian values in C++?"; + std::string b64 = "SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz8"; + std::vector buf(Base64DecodeBufSize(b64.Size()), '\0'); + Base64DecodeUneven(buf.begin(), b64); + std::string res(buf.data()); + UNIT_ASSERT_VALUES_EQUAL(x, res); + } +} diff --git a/ydb/public/sdk/cpp/src/library/string_utils/helpers/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/string_utils/helpers/CMakeLists.txt new file mode 100644 index 000000000000..ae96855924c3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/helpers/CMakeLists.txt @@ -0,0 +1,8 @@ +_ydb_sdk_add_library(string_utils-helpers) + +target_sources(string_utils-helpers + PRIVATE + helpers.cpp +) + +_ydb_sdk_install_targets(TARGETS string_utils-helpers) diff --git a/ydb/public/sdk/cpp/src/library/string_utils/helpers/helpers.cpp b/ydb/public/sdk/cpp/src/library/string_utils/helpers/helpers.cpp new file mode 100644 index 000000000000..a7d626fd1877 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/helpers/helpers.cpp @@ -0,0 +1,245 @@ +#include + +#include +#include +#include + +namespace NUtils { + namespace { + void DoSplit(std::string_view src, std::string_view& l, std::string_view& r, size_t pos, size_t len) { + const auto right = src.substr(pos + len); // in case if (&l == &src) + l = src.substr(0, pos); + r = right; + } + } // namespace + +char* ToLower(char* str) noexcept(noexcept(std::tolower(0))) { + for (char* dst = str; *dst != '\0'; ++dst) { + *dst = static_cast(std::tolower(static_cast(*dst))); + } + return str; +} + +void ToLower(std::string& str) { + for (char& c: str) { + c = std::tolower(c); + } +} + +std::string ToLower(const std::string& str) { + std::string ret(str); + ToLower(ret); + return ret; +} + +std::string ToTitle(const std::string& s) { + std::string res = ToLower(s); + if (!res.empty()) { + res[0] = std::toupper(res[0]); + } + return res; +} + +void RemoveAll(std::string& str, char ch) { + const auto pos = str.find(ch); // 'find' to avoid cloning of string in 'string.begin()' + if (pos == std::string::npos) { + return; + } + + auto begin = str.begin(); + auto end = begin + str.size(); + auto it = std::remove(begin + pos, end, ch); + str.erase(it, end); +} + +bool TrySplitOn(std::string_view src, std::string_view& l, std::string_view& r, size_t pos, size_t len) { + if (pos == std::string_view::npos) { + return false; + } + DoSplit(src, l, r, pos, len); + return true; +} + +bool TrySplit(std::string_view src, std::string_view& l, std::string_view& r, char delim) { + return TrySplitOn(src, l, r, src.find(delim), 1); +} + +bool TrySplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim) { + return TrySplitOn(src, l, r, src.find(delim), delim.size()); +} + +bool TryRSplit(std::string_view src, std::string_view& l, std::string_view& r, char delim) { + return TrySplitOn(src, l, r, src.rfind(delim), 1); +} + +bool TryRSplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim) { + return TrySplitOn(src, l, r, src.rfind(delim), delim.size()); +} + +void Split(std::string_view src, std::string_view& l, std::string_view& r, char delim) { + if (!TrySplit(src, l, r, delim)) { + l = src; + r = {}; + } +} + +void Split(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim) { + if (!TrySplit(src, l, r, delim)) { + l = src; + r = {}; + } +} + +void RSplit(std::string_view src, std::string_view& l, std::string_view& r, char delim) { + if (!TryRSplit(src, l, r, delim)) { + r = src; + l = {}; + } +} + +void RSplit(std::string_view src, std::string_view& l, std::string_view& r, std::string_view delim) { + if (!TryRSplit(src, l, r, delim)) { + r = src; + l = {}; + } +} + +std::string_view NextTok(std::string_view& src, char delim) { + std::string_view tok; + Split(src, tok, src, delim); + return tok; +} + +std::string_view NextTok(std::string_view& src, std::string_view delim) { + std::string_view tok; + Split(src, tok, src, delim); + return tok; +} + +bool NextTok(std::string_view& src, std::string_view& tok, char delim) { + if (!src.empty()) { + Split(src, tok, src, delim); + return true; + } + return false; +} + +bool NextTok(std::string_view& src, std::string_view& tok, std::string_view delim) { + if (!src.empty()) { + Split(src, tok, src, delim); + return true; + } + return false; +} + +std::string_view RNextTok(std::string_view& src, char delim) { + std::string_view tok; + RSplit(src, src, tok, delim); + return tok; +} + +std::string_view RNextTok(std::string_view& src, std::string_view delim) { + std::string_view tok; + RSplit(src, src, tok, delim); + return tok; +} + +std::string_view After(std::string_view src, char c) { + std::string_view l, r; + return TrySplit(src, l, r, c) ? r : src; +} + +std::string_view Before(std::string_view src, char c) { + std::string_view l, r; + return TrySplit(src, l, r, c) ? l : src; +} + +size_t SumLength() noexcept { + return 0; +} + +size_t Strlcat(char* dst, const char* src, size_t dsize) noexcept { + const char* odst = dst; + const char* osrc = src; + size_t n = dsize; + size_t dlen; + + // Find the end of dst and adjust bytes left but don't go past end. + while (n-- != 0 && *dst != '\0') { + dst++; + } + dlen = dst - odst; + n = dsize - dlen; + + if (n-- == 0) { + return dlen + std::strlen(src); + } + for (; *src != '\0'; ++src) { + if (n != 0) { + *dst++ = *src; + --n; + } + } + *dst = '\0'; + + return dlen + (src - osrc); // count doesn't include NUL +} + +size_t Strlcpy(char* dst, const char* src, size_t dsize) noexcept { + const char* osrc = src; + size_t nleft = dsize; + + // Copy as many bytes as will fit. + if (nleft != 0) { + while (--nleft != 0) { + if ((*dst++ = *src++) == '\0') { + break; + } + } + } + + // Not enough room in dst, add NUL and traverse rest of src. + if (nleft == 0) { + if (dsize != 0) { + *dst = '\0'; // NUL-terminate dst + } + while (*src++ != '\0') { + // pass + } + } + + return src - osrc - 1; // count does not include NUL +} + +void CopyAll(char*) noexcept { +} + +template <> +std::u16string FromAscii(const std::string_view& s) { + std::u16string res; + res.resize(s.size()); + + auto dst = res.begin(); + + for (const char* src = s.data(); dst != res.end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return res; +} + +template <> +std::u32string FromAscii(const std::string_view& s) { + std::u32string res; + res.resize(s.size()); + + auto dst = res.begin(); + + for (const char* src = s.data(); dst != res.end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return res; +} + +} // namespace NUtils diff --git a/ydb/public/sdk/cpp/src/library/string_utils/helpers/ya.make b/ydb/public/sdk/cpp/src/library/string_utils/helpers/ya.make new file mode 100644 index 000000000000..081fdffccced --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/helpers/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + helpers.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/string_utils/misc/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/string_utils/misc/CMakeLists.txt new file mode 100644 index 000000000000..780a43a35c24 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/string_utils/misc/CMakeLists.txt @@ -0,0 +1,8 @@ +_ydb_sdk_add_library(string_utils-misc INTERFACE) + +target_link_libraries(string_utils-misc INTERFACE + yutil + string_utils-helpers +) + +_ydb_sdk_install_targets(TARGETS string_utils-misc) diff --git a/ydb/public/sdk/cpp/src/library/uuid/CMakeLists.txt b/ydb/public/sdk/cpp/src/library/uuid/CMakeLists.txt new file mode 100644 index 000000000000..c0c8240987d1 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/uuid/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(library-uuid) + +target_link_libraries(library-uuid PUBLIC + yutil +) + +target_sources(library-uuid PRIVATE + uuid.cpp +) + +_ydb_sdk_install_targets(TARGETS library-uuid) diff --git a/ydb/public/sdk/cpp/src/library/uuid/uuid.cpp b/ydb/public/sdk/cpp/src/library/uuid/uuid.cpp new file mode 100644 index 000000000000..af69eee6485c --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/uuid/uuid.cpp @@ -0,0 +1,82 @@ +#include "uuid.h" + +#include + +namespace NYdb { +namespace NUuid { + +static void WriteHexDigit(ui8 digit, IOutputStream& out) { + if (digit <= 9) { + out << char('0' + digit); + } + else { + out << char('a' + digit - 10); + } +} + +static void WriteHex(ui16 bytes, IOutputStream& out, bool reverseBytes = false) { + if (reverseBytes) { + WriteHexDigit((bytes >> 4) & 0x0f, out); + WriteHexDigit(bytes & 0x0f, out); + WriteHexDigit((bytes >> 12) & 0x0f, out); + WriteHexDigit((bytes >> 8) & 0x0f, out); + } else { + WriteHexDigit((bytes >> 12) & 0x0f, out); + WriteHexDigit((bytes >> 8) & 0x0f, out); + WriteHexDigit((bytes >> 4) & 0x0f, out); + WriteHexDigit(bytes & 0x0f, out); + } +} + +void UuidToString(ui16 dw[8], IOutputStream& out) { + WriteHex(dw[1], out); + WriteHex(dw[0], out); + out << '-'; + WriteHex(dw[2], out); + out << '-'; + WriteHex(dw[3], out); + out << '-'; + WriteHex(dw[4], out, true); + out << '-'; + WriteHex(dw[5], out, true); + WriteHex(dw[6], out, true); + WriteHex(dw[7], out, true); +} + +std::string UuidBytesToString(const std::string& in) { + TStringStream ss; + + UuidBytesToString(TString(in), ss); + + return std::string(ss.Str()); +} + +void UuidBytesToString(const std::string& in, IOutputStream& out) { + ui16 dw[8]; + std::memcpy(dw, in.data(), sizeof(dw)); + NUuid::UuidToString(dw, out); +} + +void UuidHalfsToString(ui64 low, ui64 hi, IOutputStream& out) { + union { + ui16 dw[8]; + ui64 half[2]; + } buf; + buf.half[0] = low; + buf.half[1] = hi; + NUuid::UuidToString(buf.dw, out); +} + +void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out) { + union { + char bytes[16]; + ui64 half[2]; + } buf; + buf.half[0] = low; + buf.half[1] = hi; + out.Write(buf.bytes, 16); +} + +} +} + diff --git a/ydb/public/sdk/cpp/src/library/uuid/uuid.h b/ydb/public/sdk/cpp/src/library/uuid/uuid.h new file mode 100644 index 000000000000..f1b23dfcc809 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/uuid/uuid.h @@ -0,0 +1,125 @@ +#pragma once +#include +#include + +#include +#include +#include + +class IOutputStream; + +namespace NYdb { +namespace NUuid { + +static constexpr ui32 UUID_LEN = 16; + +std::string UuidBytesToString(const std::string& in); +void UuidBytesToString(const std::string& in, IOutputStream& out); +void UuidHalfsToString(ui64 low, ui64 hi, IOutputStream& out); +void UuidToString(ui16 dw[8], IOutputStream& out); +void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out); + +inline bool GetDigit(char c, ui32& digit) { + digit = 0; + if ('0' <= c && c <= '9') { + digit = c - '0'; + } + else if ('a' <= c && c <= 'f') { + digit = c - 'a' + 10; + } + else if ('A' <= c && c <= 'F') { + digit = c - 'A' + 10; + } + else { + return false; // non-hex character + } + return true; +} + +template +inline bool IsValidUuid(const T& buf) { + if (buf.Size() != 36) { + return false; + } + + for (size_t i = 0; i < buf.size(); ++i) { + const char c = buf.data()[i]; + + if (c == '-') { + if (i != 8 && i != 13 && i != 18 && i != 23) { + return false; + } + } else if (!std::isxdigit(c)) { + return false; + } + } + + return true; +} + +template +bool ParseUuidToArray(const T& buf, ui16* dw, bool shortForm) { + if (buf.size() != (shortForm ? 32 : 36)) { + return false; + } + + size_t partId = 0; + ui64 partValue = 0; + size_t digitCount = 0; + + for (size_t i = 0; i < buf.size(); ++i) { + const char c = buf.data()[i]; + + if (!shortForm && (i == 8 || i == 13 || i == 18 || i == 23)) { + if (c == '-') { + continue; + } else { + return false; + } + } + + ui32 digit = 0; + if (!GetDigit(c, digit)) { + return false; + } + + partValue = partValue * 16 + digit; + + if (++digitCount == 4) { + dw[partId++] = partValue; + digitCount = 0; + } + } + + std::swap(dw[0], dw[1]); + for (ui32 i = 4; i < 8; ++i) { + dw[i] = ((dw[i] >> 8) & 0xff) | ((dw[i] & 0xff) << 8); + } + + return true; +} + +inline void UuidHalfsToBytes(char *dst, size_t dstSize, ui64 hi, ui64 low) { + union { + char bytes[UUID_LEN]; + ui64 half[2]; + } buf; + Y_ABORT_UNLESS(UUID_LEN == dstSize); + buf.half[0] = low; + buf.half[1] = hi; + memcpy(dst, buf.bytes, sizeof(buf)); +} + +inline void UuidBytesToHalfs(const char *str, size_t sz, ui64 &high, ui64 &low) { + union { + char bytes[UUID_LEN]; + ui64 half[2]; + } buf; + Y_ABORT_UNLESS(UUID_LEN == sz); + memcpy(buf.bytes, str, sizeof(buf)); + low = buf.half[0]; + high = buf.half[1]; +} + +} +} diff --git a/ydb/public/sdk/cpp/src/library/uuid/ya.make b/ydb/public/sdk/cpp/src/library/uuid/ya.make new file mode 100644 index 000000000000..dc000d20ebc0 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/uuid/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + uuid.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/src/library/ya.make b/ydb/public/sdk/cpp/src/library/ya.make new file mode 100644 index 000000000000..66b749452d79 --- /dev/null +++ b/ydb/public/sdk/cpp/src/library/ya.make @@ -0,0 +1,11 @@ +RECURSE( + decimal + grpc/client + issue + jwt + operation_id + persqueue/obfuscate + persqueue/topic_parser_public + string_utils/helpers + uuid +) diff --git a/ydb/public/sdk/cpp/src/ya.make b/ydb/public/sdk/cpp/src/ya.make new file mode 100644 index 000000000000..736afe96ef74 --- /dev/null +++ b/ydb/public/sdk/cpp/src/ya.make @@ -0,0 +1,4 @@ +RECURSE( + client + library +) diff --git a/ydb/public/sdk/cpp/tests/CMakeLists.txt b/ydb/public/sdk/cpp/tests/CMakeLists.txt new file mode 100644 index 000000000000..729c6ee07782 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(integration) +add_subdirectory(unit) diff --git a/ydb/public/sdk/cpp/tests/integration/CMakeLists.txt b/ydb/public/sdk/cpp/tests/integration/CMakeLists.txt new file mode 100644 index 000000000000..510ce5332f16 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(basic_example_it) +add_subdirectory(bulk_upsert_simple_it) diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/CMakeLists.txt b/ydb/public/sdk/cpp/tests/integration/basic_example_it/CMakeLists.txt new file mode 100644 index 000000000000..b884f9560aad --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/CMakeLists.txt @@ -0,0 +1,15 @@ +add_ydb_test(NAME basic-example_it + SOURCES + main.cpp + basic_example_data.cpp + basic_example.cpp + LINK_LIBRARIES + yutil + api-protos + YDB-CPP-SDK::Driver + YDB-CPP-SDK::Proto + YDB-CPP-SDK::Table + GTest::gtest_main + LABELS + integration +) diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.cpp b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.cpp new file mode 100644 index 000000000000..9928d4a8d509 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.cpp @@ -0,0 +1,516 @@ +#include "basic_example.h" + +#include + +#include +#include + +using namespace NYdb::NTable; + +static std::string JoinPath(const std::string& basePath, const std::string& path) { + if (basePath.empty()) { + return path; + } + + std::filesystem::path prefixPathSplit(basePath); + prefixPathSplit /= path; + + return prefixPathSplit; +} + +TRunArgs GetRunArgs() { + std::string database = std::getenv("YDB_DATABASE"); + std::string endpoint = std::getenv("YDB_ENDPOINT"); + + auto driverConfig = NYdb::TDriverConfig() + .SetEndpoint(endpoint) + .SetDatabase(database) + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); + + NYdb::TDriver driver(driverConfig); + return {driver, JoinPath(database, "basic")}; +} + +/////////////////////////////////////////////////////////////////////////////// + +//! Creates sample tables with CrateTable API. +void CreateTables(TTableClient client, const std::string& path) { + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + auto seriesDesc = TTableBuilder() + .AddNullableColumn("series_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("title", NYdb::EPrimitiveType::Utf8) + .AddNullableColumn("series_info", NYdb::EPrimitiveType::Utf8) + .AddNullableColumn("release_date", NYdb::EPrimitiveType::Uint64) + .SetPrimaryKeyColumn("series_id") + .Build(); + + return session.CreateTable(JoinPath(path, "series"), std::move(seriesDesc)).GetValueSync(); + })); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + auto seasonsDesc = TTableBuilder() + .AddNullableColumn("series_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("season_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("title", NYdb::EPrimitiveType::Utf8) + .AddNullableColumn("first_aired", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("last_aired", NYdb::EPrimitiveType::Uint64) + .SetPrimaryKeyColumns({"series_id", "season_id"}) + .Build(); + + return session.CreateTable(JoinPath(path, "seasons"), std::move(seasonsDesc)).GetValueSync(); + })); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + auto episodesDesc = TTableBuilder() + .AddNullableColumn("series_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("season_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("episode_id", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("title", NYdb::EPrimitiveType::Utf8) + .AddNullableColumn("air_date", NYdb::EPrimitiveType::Uint64) + .SetPrimaryKeyColumns({"series_id", "season_id", "episode_id"}) + .Build(); + + return session.CreateTable(JoinPath(path, "episodes"), + std::move(episodesDesc)).GetValueSync(); + })); +} + +/////////////////////////////////////////////////////////////////////////////// + +//! Fills sample tables with data in single parameterized data query. +NYdb::TStatus FillTableDataTransaction(TSession session, const std::string& path) { + auto query = std::format(R"( + PRAGMA TablePathPrefix("{}"); + + DECLARE $seriesData AS List>; + + DECLARE $seasonsData AS List>; + + DECLARE $episodesData AS List>; + + REPLACE INTO series + SELECT + series_id, + title, + series_info, + CAST(release_date AS Uint16) AS release_date + FROM AS_TABLE($seriesData); + + REPLACE INTO seasons + SELECT + series_id, + season_id, + title, + CAST(first_aired AS Uint16) AS first_aired, + CAST(last_aired AS Uint16) AS last_aired + FROM AS_TABLE($seasonsData); + + REPLACE INTO episodes + SELECT + series_id, + season_id, + episode_id, + title, + CAST(air_date AS Uint16) AS air_date + FROM AS_TABLE($episodesData); + )", path); + + auto params = GetTablesDataParams(); + + return session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + params).GetValueSync(); +} + +//! Shows basic usage of YDB data queries and transactions. +static NYdb::TStatus SelectSimpleTransaction(TSession session, const std::string& path, + std::optional& resultSet) +{ + auto query = std::format(R"( + PRAGMA TablePathPrefix("{}"); + + SELECT series_id, title, CAST(CAST(release_date AS Date) AS String) AS release_date + FROM series + WHERE series_id = 1 + ORDER BY series_id; + )", path); + + auto txControl = + // Begin new transaction with SerializableRW mode + TTxControl::BeginTx(TTxSettings::SerializableRW()) + // Commit transaction at the end of the query + .CommitTx(); + + // Executes data query with specified transaction control settings. + auto result = session.ExecuteDataQuery(query, txControl).GetValueSync(); + + if (result.IsSuccess()) { + // Index of result set corresponds to its order in YQL query + resultSet = result.GetResultSet(0); + } + + return result; +} + +//! Shows basic usage of mutating operations. +static NYdb::TStatus UpsertSimpleTransaction(TSession session, const std::string& path) { + auto query = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + UPSERT INTO episodes (series_id, season_id, episode_id, title) VALUES + (2, 6, 1, "TBD"); + )", path); + + return session.ExecuteDataQuery(query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); +} + +//! Shows usage of parameters in data queries. +static NYdb::TStatus SelectWithParamsTransaction(TSession session, const std::string& path, + uint64_t seriesId, uint64_t seasonId, std::optional& resultSet) +{ + auto query = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $seriesId AS Uint64; + DECLARE $seasonId AS Uint64; + + SELECT sa.title AS season_title, sr.title AS series_title + FROM seasons AS sa + INNER JOIN series AS sr + ON sa.series_id = sr.series_id + WHERE sa.series_id = $seriesId AND sa.season_id = $seasonId + ORDER BY season_title, series_title; + )", path); + + // Type of parameter values should be exactly the same as in DECLARE statements. + auto params = session.GetParamsBuilder() + .AddParam("$seriesId") + .Uint64(seriesId) + .Build() + .AddParam("$seasonId") + .Uint64(seasonId) + .Build() + .Build(); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + params).GetValueSync(); + + if (result.IsSuccess()) { + resultSet = result.GetResultSet(0); + } + + return result; +} + +//! Shows usage of prepared queries. +static NYdb::TStatus PreparedSelectTransaction(TSession session, const std::string& path, + uint64_t seriesId, uint64_t seasonId, uint64_t episodeId, std::optional& resultSet) +{ + // Once prepared, query data is stored in the session and identified by QueryId. + // Local query cache is used to keep track of queries, prepared in current session. + auto query = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $seriesId AS Uint64; + DECLARE $seasonId AS Uint64; + DECLARE $episodeId AS Uint64; + + SELECT * + FROM episodes + WHERE series_id = $seriesId AND season_id = $seasonId AND episode_id = $episodeId + ORDER BY season_id, episode_id; + )", path); + + // Prepare query or get result from query cache + auto prepareResult = session.PrepareDataQuery(query).GetValueSync(); + if (!prepareResult.IsSuccess()) { + return prepareResult; + } + + auto dataQuery = prepareResult.GetQuery(); + + auto params = dataQuery.GetParamsBuilder() + .AddParam("$seriesId") + .Uint64(seriesId) + .Build() + .AddParam("$seasonId") + .Uint64(seasonId) + .Build() + .AddParam("$episodeId") + .Uint64(episodeId) + .Build() + .Build(); + + auto result = dataQuery.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + params).GetValueSync(); + + if (result.IsSuccess()) { + resultSet = result.GetResultSet(0); + } + + return result; +} + +//! Shows usage of transactions consisting of multiple data queries with client logic between them. +static NYdb::TStatus MultiStepTransaction(TSession session, const std::string& path, uint64_t seriesId, uint64_t seasonId, + std::optional& resultSet) +{ + auto query1 = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $seriesId AS Uint64; + DECLARE $seasonId AS Uint64; + + SELECT first_aired AS from_date FROM seasons + WHERE series_id = $seriesId AND season_id = $seasonId + ORDER BY from_date; + )", path); + + auto params1 = session.GetParamsBuilder() + .AddParam("$seriesId") + .Uint64(seriesId) + .Build() + .AddParam("$seasonId") + .Uint64(seasonId) + .Build() + .Build(); + + // Execute first query to get the required values to the client. + // Transaction control settings don't set CommitTx flag to keep transaction active + // after query execution. + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()), + params1).GetValueSync(); + + if (!result.IsSuccess()) { + return result; + } + + // Get active transaction id + auto tx = result.GetTransaction(); + + NYdb::TResultSetParser parser(result.GetResultSet(0)); + parser.TryNextRow(); + auto date = parser.ColumnParser("from_date").GetOptionalUint64(); + + // Perform some client logic on returned values + auto userFunc = [] (const TInstant fromDate) { + return fromDate + TDuration::Days(15); + }; + + TInstant fromDate = TInstant::Days(*date); + TInstant toDate = userFunc(fromDate); + + // Construct next query based on the results of client logic + auto query2 = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $seriesId AS Uint64; + DECLARE $fromDate AS Uint64; + DECLARE $toDate AS Uint64; + + SELECT season_id, episode_id, title, air_date FROM episodes + WHERE series_id = $seriesId AND air_date >= $fromDate AND air_date <= $toDate + ORDER BY season_id, episode_id; + )", path); + + auto params2 = session.GetParamsBuilder() + .AddParam("$seriesId") + .Uint64(seriesId) + .Build() + .AddParam("$fromDate") + .Uint64(fromDate.Days()) + .Build() + .AddParam("$toDate") + .Uint64(toDate.Days()) + .Build() + .Build(); + + // Execute second query. + // Transaction control settings continues active transaction (tx) and + // commits it at the end of second query execution. + result = session.ExecuteDataQuery( + query2, + TTxControl::Tx(*tx).CommitTx(), + params2).GetValueSync(); + + if (result.IsSuccess()) { + resultSet = result.GetResultSet(0); + } + + return result; +} + +// Show usage of explicit Begin/Commit transaction control calls. +// In most cases it's better to use transaction control settings in ExecuteDataQuery calls instead +// to avoid additional hops to YDB cluster and allow more efficient execution of queries. +static NYdb::TStatus ExplicitTclTransaction(TSession session, const std::string& path, const TInstant& airDate) { + auto beginResult = session.BeginTransaction(TTxSettings::SerializableRW()).GetValueSync(); + if (!beginResult.IsSuccess()) { + return beginResult; + } + + // Get newly created transaction id + auto tx = beginResult.GetTransaction(); + + auto query = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $airDate AS Date; + + UPDATE episodes SET air_date = CAST($airDate AS Uint16) WHERE title = "TBD"; + )", path); + + auto params = session.GetParamsBuilder() + .AddParam("$airDate") + .Date(airDate) + .Build() + .Build(); + + // Execute data query. + // Transaction control settings continues active transaction (tx) + auto updateResult = session.ExecuteDataQuery(query, + TTxControl::Tx(tx), + params).GetValueSync(); + + if (!updateResult.IsSuccess()) { + return updateResult; + } + + // Commit active transaction (tx) + return tx.Commit().GetValueSync(); +} + +static NYdb::TStatus ScanQuerySelect(TTableClient client, const std::string& path, std::vector& vectorResultSet) { + vectorResultSet.clear(); + auto query = std::format(R"( + --!syntax_v1 + PRAGMA TablePathPrefix("{}"); + + DECLARE $series AS List; + + SELECT series_id, season_id, title, CAST(CAST(first_aired AS Date) AS String) AS first_aired + FROM seasons + WHERE series_id IN $series + ORDER BY season_id; + )", path); + + auto parameters = NYdb::TParamsBuilder() + .AddParam("$series") + .BeginList() + .AddListItem().Uint64(1) + .AddListItem().Uint64(10) + .EndList().Build() + .Build(); + + // Executes scan query + auto resultScanQuery = client.StreamExecuteScanQuery(query, parameters).GetValueSync(); + + if (!resultScanQuery.IsSuccess()) { + return resultScanQuery; + } + + bool eos = false; + + while (!eos) { + auto streamPart = resultScanQuery.ReadNext().ExtractValueSync(); + + if (!streamPart.IsSuccess()) { + eos = true; + if (!streamPart.EOS()) { + return streamPart; + } + continue; + } + + if (streamPart.HasResultSet()) { + auto rs = streamPart.ExtractResultSet(); + vectorResultSet.push_back(rs); + } + } + return NYdb::TStatus(NYdb::EStatus::SUCCESS, NYdb::NIssue::TIssues()); +} + +/////////////////////////////////////////////////////////////////////////////// + +NYdb::TResultSet SelectSimple(TTableClient client, const std::string& path) { + std::optional resultSet; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, &resultSet](TSession session) { + return SelectSimpleTransaction(session, path, resultSet); + })); + + return resultSet.value(); +} + +void UpsertSimple(TTableClient client, const std::string& path) { + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + return UpsertSimpleTransaction(session, path); + })); +} + +NYdb::TResultSet SelectWithParams(TTableClient client, const std::string& path) { + std::optional resultSet; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, &resultSet](TSession session) { + return SelectWithParamsTransaction(session, path, 2, 3, resultSet); + })); + + return resultSet.value(); +} + +NYdb::TResultSet PreparedSelect(TTableClient client, const std::string& path, ui32 seriesId, ui32 seasonId, ui32 episodeId) { + std::optional resultSet; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, seriesId, seasonId, episodeId, &resultSet](TSession session) { + return PreparedSelectTransaction(session, path, seriesId, seasonId, episodeId, resultSet); + })); + + return resultSet.value(); +} + +NYdb::TResultSet MultiStep(TTableClient client, const std::string& path) { + std::optional resultSet; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, &resultSet](TSession session) { + return MultiStepTransaction(session, path, 2, 5, resultSet); + })); + + return resultSet.value(); +} + +void ExplicitTcl(TTableClient client, const std::string& path) { + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + return ExplicitTclTransaction(session, path, TInstant()); + })); +} + +std::vector ScanQuerySelect(TTableClient client, const std::string& path) { + std::vector resultSets; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, &resultSets](TTableClient& client) { + return ScanQuerySelect(client, path, resultSets); + })); + + return resultSets; +} diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.h b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.h new file mode 100644 index 000000000000..e35fba699432 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include + +#include + +using namespace NYdb::NTable; + +struct TRunArgs { + NYdb::TDriver driver; + std::string path; +}; + +NYdb::TParams GetTablesDataParams(); + +void CreateTables(TTableClient client, const std::string& path); +TRunArgs GetRunArgs(); +NYdb::TStatus FillTableDataTransaction(TSession session, const std::string& path); +NYdb::TResultSet SelectSimple(TTableClient client, const std::string& path); +void UpsertSimple(TTableClient client, const std::string& path); +NYdb::TResultSet SelectWithParams(TTableClient client, const std::string& path); +NYdb::TResultSet PreparedSelect(TTableClient client, const std::string& path, ui32 seriesId, ui32 seasonId, ui32 episodeId); +NYdb::TResultSet MultiStep(TTableClient client, const std::string& path); +void ExplicitTcl(TTableClient client, const std::string& path); +NYdb::TResultSet PreparedSelect(TTableClient client, const std::string& path, ui32 seriesId, ui32 seasonId, ui32 episodeId); +std::vector ScanQuerySelect(TTableClient client, const std::string& path); diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example_data.cpp b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example_data.cpp new file mode 100644 index 000000000000..fd0693011b7e --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/basic_example_data.cpp @@ -0,0 +1,202 @@ +#include "basic_example.h" + +struct TSeries { + ui64 SeriesId; + std::string Title; + TInstant ReleaseDate; + std::string SeriesInfo; + + TSeries(ui64 seriesId, const std::string& title, const TInstant& releaseDate, const std::string& seriesInfo) + : SeriesId(seriesId) + , Title(title) + , ReleaseDate(releaseDate) + , SeriesInfo(seriesInfo) {} +}; + +struct TSeason { + ui64 SeriesId; + ui64 SeasonId; + std::string Title; + TInstant FirstAired; + TInstant LastAired; + + TSeason(ui64 seriesId, ui64 seasonId, const std::string& title, const TInstant& firstAired, const TInstant& lastAired) + : SeriesId(seriesId) + , SeasonId(seasonId) + , Title(title) + , FirstAired(firstAired) + , LastAired(lastAired) {} +}; + +struct TEpisode { + ui64 SeriesId; + ui64 SeasonId; + ui64 EpisodeId; + std::string Title; + TInstant AirDate; + + TEpisode(ui64 seriesId, ui64 seasonId, ui64 episodeId, const std::string& title, const TInstant& airDate) + : SeriesId(seriesId) + , SeasonId(seasonId) + , EpisodeId(episodeId) + , Title(title) + , AirDate(airDate) {} +}; + +NYdb::TParams GetTablesDataParams() { + std::vector seriesData = { + TSeries(1, "IT Crowd", TInstant::ParseIso8601("2006-02-03"), + "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by " + "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry."), + TSeries(2, "Silicon Valley", TInstant::ParseIso8601("2014-04-06"), + "Silicon Valley is an American comedy television series created by Mike Judge, John Altschuler and " + "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.") + }; + + std::vector seasonsData = { + TSeason(1, 1, "Season 1", TInstant::ParseIso8601("2006-02-03"), TInstant::ParseIso8601("2006-03-03")), + TSeason(1, 2, "Season 2", TInstant::ParseIso8601("2007-08-24"), TInstant::ParseIso8601("2007-09-28")), + TSeason(1, 3, "Season 3", TInstant::ParseIso8601("2008-11-21"), TInstant::ParseIso8601("2008-12-26")), + TSeason(1, 4, "Season 4", TInstant::ParseIso8601("2010-06-25"), TInstant::ParseIso8601("2010-07-30")), + TSeason(2, 1, "Season 1", TInstant::ParseIso8601("2014-04-06"), TInstant::ParseIso8601("2014-06-01")), + TSeason(2, 2, "Season 2", TInstant::ParseIso8601("2015-04-12"), TInstant::ParseIso8601("2015-06-14")), + TSeason(2, 3, "Season 3", TInstant::ParseIso8601("2016-04-24"), TInstant::ParseIso8601("2016-06-26")), + TSeason(2, 4, "Season 4", TInstant::ParseIso8601("2017-04-23"), TInstant::ParseIso8601("2017-06-25")), + TSeason(2, 5, "Season 5", TInstant::ParseIso8601("2018-03-25"), TInstant::ParseIso8601("2018-05-13")) + }; + + std::vector episodesData = { + TEpisode(1, 1, 1, "Yesterday's Jam", TInstant::ParseIso8601("2006-02-03")), + TEpisode(1, 1, 2, "Calamity Jen", TInstant::ParseIso8601("2006-02-03")), + TEpisode(1, 1, 3, "Fifty-Fifty", TInstant::ParseIso8601("2006-02-10")), + TEpisode(1, 1, 4, "The Red Door", TInstant::ParseIso8601("2006-02-17")), + TEpisode(1, 1, 5, "The Haunting of Bill Crouse", TInstant::ParseIso8601("2006-02-24")), + TEpisode(1, 1, 6, "Aunt Irma Visits", TInstant::ParseIso8601("2006-03-03")), + TEpisode(1, 2, 1, "The Work Outing", TInstant::ParseIso8601("2006-08-24")), + TEpisode(1, 2, 2, "Return of the Golden Child", TInstant::ParseIso8601("2007-08-31")), + TEpisode(1, 2, 3, "Moss and the German", TInstant::ParseIso8601("2007-09-07")), + TEpisode(1, 2, 4, "The Dinner Party", TInstant::ParseIso8601("2007-09-14")), + TEpisode(1, 2, 5, "Smoke and Mirrors", TInstant::ParseIso8601("2007-09-21")), + TEpisode(1, 2, 6, "Men Without Women", TInstant::ParseIso8601("2007-09-28")), + TEpisode(1, 3, 1, "From Hell", TInstant::ParseIso8601("2008-11-21")), + TEpisode(1, 3, 2, "Are We Not Men?", TInstant::ParseIso8601("2008-11-28")), + TEpisode(1, 3, 3, "Tramps Like Us", TInstant::ParseIso8601("2008-12-05")), + TEpisode(1, 3, 4, "The Speech", TInstant::ParseIso8601("2008-12-12")), + TEpisode(1, 3, 5, "Friendface", TInstant::ParseIso8601("2008-12-19")), + TEpisode(1, 3, 6, "Calendar Geeks", TInstant::ParseIso8601("2008-12-26")), + TEpisode(1, 4, 1, "Jen The Fredo", TInstant::ParseIso8601("2010-06-25")), + TEpisode(1, 4, 2, "The Final Countdown", TInstant::ParseIso8601("2010-07-02")), + TEpisode(1, 4, 3, "Something Happened", TInstant::ParseIso8601("2010-07-09")), + TEpisode(1, 4, 4, "Italian For Beginners", TInstant::ParseIso8601("2010-07-16")), + TEpisode(1, 4, 5, "Bad Boys", TInstant::ParseIso8601("2010-07-23")), + TEpisode(1, 4, 6, "Reynholm vs Reynholm", TInstant::ParseIso8601("2010-07-30")), + TEpisode(2, 1, 1, "Minimum Viable Product", TInstant::ParseIso8601("2014-04-06")), + TEpisode(2, 1, 2, "The Cap Table", TInstant::ParseIso8601("2014-04-13")), + TEpisode(2, 1, 3, "Articles of Incorporation", TInstant::ParseIso8601("2014-04-20")), + TEpisode(2, 1, 4, "Fiduciary Duties", TInstant::ParseIso8601("2014-04-27")), + TEpisode(2, 1, 5, "Signaling Risk", TInstant::ParseIso8601("2014-05-04")), + TEpisode(2, 1, 6, "Third Party Insourcing", TInstant::ParseIso8601("2014-05-11")), + TEpisode(2, 1, 7, "Proof of Concept", TInstant::ParseIso8601("2014-05-18")), + TEpisode(2, 1, 8, "Optimal Tip-to-Tip Efficiency", TInstant::ParseIso8601("2014-06-01")), + TEpisode(2, 2, 1, "Sand Hill Shuffle", TInstant::ParseIso8601("2015-04-12")), + TEpisode(2, 2, 2, "Runaway Devaluation", TInstant::ParseIso8601("2015-04-19")), + TEpisode(2, 2, 3, "Bad Money", TInstant::ParseIso8601("2015-04-26")), + TEpisode(2, 2, 4, "The Lady", TInstant::ParseIso8601("2015-05-03")), + TEpisode(2, 2, 5, "Server Space", TInstant::ParseIso8601("2015-05-10")), + TEpisode(2, 2, 6, "Homicide", TInstant::ParseIso8601("2015-05-17")), + TEpisode(2, 2, 7, "Adult Content", TInstant::ParseIso8601("2015-05-24")), + TEpisode(2, 2, 8, "White Hat/Black Hat", TInstant::ParseIso8601("2015-05-31")), + TEpisode(2, 2, 9, "Binding Arbitration", TInstant::ParseIso8601("2015-06-07")), + TEpisode(2, 2, 10, "Two Days of the Condor", TInstant::ParseIso8601("2015-06-14")), + TEpisode(2, 3, 1, "Founder Friendly", TInstant::ParseIso8601("2016-04-24")), + TEpisode(2, 3, 2, "Two in the Box", TInstant::ParseIso8601("2016-05-01")), + TEpisode(2, 3, 3, "Meinertzhagen's Haversack", TInstant::ParseIso8601("2016-05-08")), + TEpisode(2, 3, 4, "Maleant Data Systems Solutions", TInstant::ParseIso8601("2016-05-15")), + TEpisode(2, 3, 5, "The Empty Chair", TInstant::ParseIso8601("2016-05-22")), + TEpisode(2, 3, 6, "Bachmanity Insanity", TInstant::ParseIso8601("2016-05-29")), + TEpisode(2, 3, 7, "To Build a Better Beta", TInstant::ParseIso8601("2016-06-05")), + TEpisode(2, 3, 8, "Bachman's Earnings Over-Ride", TInstant::ParseIso8601("2016-06-12")), + TEpisode(2, 3, 9, "Daily Active Users", TInstant::ParseIso8601("2016-06-19")), + TEpisode(2, 3, 10, "The Uptick", TInstant::ParseIso8601("2016-06-26")), + TEpisode(2, 4, 1, "Success Failure", TInstant::ParseIso8601("2017-04-23")), + TEpisode(2, 4, 2, "Terms of Service", TInstant::ParseIso8601("2017-04-30")), + TEpisode(2, 4, 3, "Intellectual Property", TInstant::ParseIso8601("2017-05-07")), + TEpisode(2, 4, 4, "Teambuilding Exercise", TInstant::ParseIso8601("2017-05-14")), + TEpisode(2, 4, 5, "The Blood Boy", TInstant::ParseIso8601("2017-05-21")), + TEpisode(2, 4, 6, "Customer Service", TInstant::ParseIso8601("2017-05-28")), + TEpisode(2, 4, 7, "The Patent Troll", TInstant::ParseIso8601("2017-06-04")), + TEpisode(2, 4, 8, "The Keenan Vortex", TInstant::ParseIso8601("2017-06-11")), + TEpisode(2, 4, 9, "Hooli-Con", TInstant::ParseIso8601("2017-06-18")), + TEpisode(2, 4, 10, "Server Error", TInstant::ParseIso8601("2017-06-25")), + TEpisode(2, 5, 1, "Grow Fast or Die Slow", TInstant::ParseIso8601("2018-03-25")), + TEpisode(2, 5, 2, "Reorientation", TInstant::ParseIso8601("2018-04-01")), + TEpisode(2, 5, 3, "Chief Operating Officer", TInstant::ParseIso8601("2018-04-08")), + TEpisode(2, 5, 4, "Tech Evangelist", TInstant::ParseIso8601("2018-04-15")), + TEpisode(2, 5, 5, "Facial Recognition", TInstant::ParseIso8601("2018-04-22")), + TEpisode(2, 5, 6, "Artificial Emotional Intelligence", TInstant::ParseIso8601("2018-04-29")), + TEpisode(2, 5, 7, "Initial Coin Offering", TInstant::ParseIso8601("2018-05-06")), + TEpisode(2, 5, 8, "Fifty-One Percent", TInstant::ParseIso8601("2018-05-13")) + }; + + NYdb::TParamsBuilder paramsBuilder; + + auto& seriesParam = paramsBuilder.AddParam("$seriesData"); + seriesParam.BeginList(); + for (auto& series : seriesData) { + seriesParam.AddListItem() + .BeginStruct() + .AddMember("series_id") + .Uint64(series.SeriesId) + .AddMember("title") + .Utf8(series.Title) + .AddMember("release_date") + .Date(series.ReleaseDate) + .AddMember("series_info") + .Utf8(series.SeriesInfo) + .EndStruct(); + } + seriesParam.EndList(); + seriesParam.Build(); + + auto& seasonsParam = paramsBuilder.AddParam("$seasonsData"); + seasonsParam.BeginList(); + for (auto& season : seasonsData) { + seasonsParam.AddListItem() + .BeginStruct() + .AddMember("series_id") + .Uint64(season.SeriesId) + .AddMember("season_id") + .Uint64(season.SeasonId) + .AddMember("title") + .Utf8(season.Title) + .AddMember("first_aired") + .Date(season.FirstAired) + .AddMember("last_aired") + .Date(season.LastAired) + .EndStruct(); + } + seasonsParam.EndList(); + seasonsParam.Build(); + + auto& episodesParam = paramsBuilder.AddParam("$episodesData"); + episodesParam.BeginList(); + for (auto& episode : episodesData) { + episodesParam.AddListItem() + .BeginStruct() + .AddMember("series_id") + .Uint64(episode.SeriesId) + .AddMember("season_id") + .Uint64(episode.SeasonId) + .AddMember("episode_id") + .Uint64(episode.EpisodeId) + .AddMember("title") + .Utf8(episode.Title) + .AddMember("air_date") + .Date(episode.AirDate) + .EndStruct(); + } + episodesParam.EndList(); + episodesParam.Build(); + + return paramsBuilder.Build(); +} diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/main.cpp b/ydb/public/sdk/cpp/tests/integration/basic_example_it/main.cpp new file mode 100644 index 000000000000..62aaba5bf7c8 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/main.cpp @@ -0,0 +1,222 @@ +#include "basic_example.h" + +#include + +#include + +#include + +#include + +static void ValidateResultSet(const std::vector& columns, + const std::vector>& values, + const NYdb::TResultSet& rs) { + Ydb::ResultSet protoResultSet; + for (const auto& column : columns) { + auto* protoColumn = protoResultSet.add_columns(); + protoColumn->set_name(column.Name); + *protoColumn->mutable_type() = column.Type.GetProto(); + } + + for (const auto& row : values) { + auto* protoRow = protoResultSet.add_rows(); + for (const auto& value : row) { + *protoRow->add_items() = value.GetProto(); + } + } + + NYdb::TStringType expected, result; + google::protobuf::TextFormat::PrintToString(protoResultSet, &expected); + google::protobuf::TextFormat::PrintToString(NYdb::TProtoAccessor::GetProto(rs), &result); + + ASSERT_EQ(expected, result); +} + + +static NYdb::TType MakeOptionalType(NYdb::EPrimitiveType type) { + return NYdb::TTypeBuilder().BeginOptional().Primitive(type).EndOptional().Build(); +} + + +TEST(Integration, BasicExample) { + auto [driver, path] = GetRunArgs(); + + NYdb::NTable::TTableClient client(driver); + + try { + CreateTables(client, path); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](NYdb::NTable::TSession session) { + return FillTableDataTransaction(session, path); + })); + + { + std::vector columns = { + {"series_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + {"release_date", MakeOptionalType(NYdb::EPrimitiveType::String)}, + }; + + std::vector> values = {{ + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Utf8("IT Crowd").Build(), + NYdb::TValueBuilder().String("2006-02-03").Build(), + }}; + + ValidateResultSet(columns, values, SelectSimple(client, path)); + } + + UpsertSimple(client, path); + + { + std::vector columns = { + {"season_title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + {"series_title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + }; + + std::vector> values = {{ + NYdb::TValueBuilder().Utf8("Season 3").Build(), + NYdb::TValueBuilder().Utf8("Silicon Valley").Build(), + }}; + + ValidateResultSet(columns, values, SelectWithParams(client, path)); + } + + { + std::vector columns = { + {"air_date", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"episode_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"season_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"series_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + }; + + std::vector> values = {{ + NYdb::TValueBuilder().Uint64(16957).Build(), + NYdb::TValueBuilder().Uint64(7).Build(), + NYdb::TValueBuilder().Uint64(3).Build(), + NYdb::TValueBuilder().Uint64(2).Build(), + NYdb::TValueBuilder().Utf8("To Build a Better Beta").Build(), + }}; + + ValidateResultSet(columns, values, PreparedSelect(client, path, 2, 3, 7)); + } + + { + std::vector columns = { + {"air_date", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"episode_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"season_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"series_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + }; + + std::vector> values = {{ + NYdb::TValueBuilder().Uint64(16964).Build(), + NYdb::TValueBuilder().Uint64(8).Build(), + NYdb::TValueBuilder().Uint64(3).Build(), + NYdb::TValueBuilder().Uint64(2).Build(), + NYdb::TValueBuilder().Utf8("Bachman's Earnings Over-Ride").Build(), + }}; + + ValidateResultSet(columns, values, PreparedSelect(client, path, 2, 3, 8)); + } + + { + std::vector columns = { + {"season_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"episode_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + {"air_date", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + }; + + std::vector> values = { + { + NYdb::TValueBuilder().Uint64(5).Build(), + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Utf8("Grow Fast or Die Slow").Build(), + NYdb::TValueBuilder().Uint64(17615).Build(), + }, + { + NYdb::TValueBuilder().Uint64(5).Build(), + NYdb::TValueBuilder().Uint64(2).Build(), + NYdb::TValueBuilder().Utf8("Reorientation").Build(), + NYdb::TValueBuilder().Uint64(17622).Build(), + }, + { + NYdb::TValueBuilder().Uint64(5).Build(), + NYdb::TValueBuilder().Uint64(3).Build(), + NYdb::TValueBuilder().Utf8("Chief Operating Officer").Build(), + NYdb::TValueBuilder().Uint64(17629).Build(), + }, + }; + + ValidateResultSet(columns, values, MultiStep(client, path)); + } + + ExplicitTcl(client, path); + + { + std::vector columns = { + {"air_date", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"episode_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"season_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"series_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + }; + + std::vector> values = {{ + NYdb::TValueBuilder().Uint64(0).Build(), + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Uint64(6).Build(), + NYdb::TValueBuilder().Uint64(2).Build(), + NYdb::TValueBuilder().Utf8("TBD").Build(), + }}; + + ValidateResultSet(columns, values, PreparedSelect(client, path, 2, 6, 1)); + } + + { + std::vector columns = { + {"series_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"season_id", MakeOptionalType(NYdb::EPrimitiveType::Uint64)}, + {"title", MakeOptionalType(NYdb::EPrimitiveType::Utf8)}, + {"first_aired", MakeOptionalType(NYdb::EPrimitiveType::String)}, + }; + + std::vector> values = { + { + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Utf8("Season 1").Build(), + NYdb::TValueBuilder().String("2006-02-03").Build(), + }, + { + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Uint64(2).Build(), + NYdb::TValueBuilder().Utf8("Season 2").Build(), + NYdb::TValueBuilder().String("2007-08-24").Build(), + }, + { + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Uint64(3).Build(), + NYdb::TValueBuilder().Utf8("Season 3").Build(), + NYdb::TValueBuilder().String("2008-11-21").Build(), + }, + { + NYdb::TValueBuilder().Uint64(1).Build(), + NYdb::TValueBuilder().Uint64(4).Build(), + NYdb::TValueBuilder().Utf8("Season 4").Build(), + NYdb::TValueBuilder().String("2010-06-25").Build(), + }, + }; + + std::string expected = "{\"series_id\":1,\"season_id\":1,\"title\":\"Season 1\",\"first_aired\":\"2006-02-03\"}\n{\"series_id\":1,\"season_id\":2,\"title\":\"Season 2\",\"first_aired\":\"2007-08-24\"}\n{\"series_id\":1,\"season_id\":3,\"title\":\"Season 3\",\"first_aired\":\"2008-11-21\"}\n{\"series_id\":1,\"season_id\":4,\"title\":\"Season 4\",\"first_aired\":\"2010-06-25\"}\n"; + auto result = ScanQuerySelect(client, path); + ASSERT_EQ(result.size(), size_t(1)); + ValidateResultSet(columns, values, result[0]); + } + } catch (const NYdb::NStatusHelpers::TYdbErrorException& e) { + FAIL() << "Execution failed due to fatal error:\n" << e.what() << std::endl; + } +} diff --git a/ydb/public/sdk/cpp/tests/integration/basic_example_it/ya.make b/ydb/public/sdk/cpp/tests/integration/basic_example_it/ya.make new file mode 100644 index 000000000000..91f3fae069a4 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/basic_example_it/ya.make @@ -0,0 +1,28 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc) + +IF (SANITIZER_TYPE == "thread") + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/public/api/protos + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/table +) + +SRCS( + basic_example_data.cpp + basic_example.cpp + main.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/CMakeLists.txt b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/CMakeLists.txt new file mode 100644 index 000000000000..bc7ffbb35b6b --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/CMakeLists.txt @@ -0,0 +1,12 @@ +add_ydb_test(NAME bulk_upsert_simple_it + SOURCES + main.cpp + bulk_upsert.cpp + bulk_upsert.h + LINK_LIBRARIES + yutil + YDB-CPP-SDK::Table + GTest::gtest_main + LABELS + integration +) diff --git a/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp new file mode 100644 index 000000000000..ce74b1cc16f9 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp @@ -0,0 +1,158 @@ +#include "bulk_upsert.h" + +#include +#include + +static constexpr size_t BATCH_SIZE = 1000; + +static std::string JoinPath(const std::string& basePath, const std::string& path) { + if (basePath.empty()) { + return path; + } + + std::filesystem::path prefixPathSplit(basePath); + prefixPathSplit /= path; + + return prefixPathSplit; +} + +TRunArgs GetRunArgs() { + + std::string database = std::getenv("YDB_DATABASE"); + std::string endpoint = std::getenv("YDB_ENDPOINT"); + + auto driverConfig = TDriverConfig() + .SetEndpoint(endpoint) + .SetDatabase(database) + .SetAuthToken(std::getenv("YDB_TOKEN") ? std::getenv("YDB_TOKEN") : ""); + + TDriver driver(driverConfig); + return {driver, JoinPath(database, "bulk")}; +} + +TStatus CreateTable(TTableClient& client, const std::string& table) { + TRetryOperationSettings settings; + auto status = client.RetryOperationSync([&table](TSession session) { + auto tableDesc = TTableBuilder() + .AddNullableColumn("App", EPrimitiveType::Utf8) + .AddNullableColumn("Timestamp", EPrimitiveType::Timestamp) + .AddNullableColumn("Host", EPrimitiveType::Utf8) + .AddNonNullableColumn("Id", EPrimitiveType::Uint64) + .AddNullableColumn("HttpCode", EPrimitiveType::Uint32) + .AddNullableColumn("Message", EPrimitiveType::Utf8) + .SetPrimaryKeyColumns({"App", "Timestamp", "Host", "Id"}) + .Build(); + + return session.CreateTable(table, std::move(tableDesc)).GetValueSync(); + }, settings); + + return status; +} + +TStatistic GetLogBatch(uint64_t logOffset, std::vector& logBatch, uint32_t lastNumber) { + logBatch.clear(); + uint32_t correctSumApp = 0; + uint32_t correctSumHost = 0; + uint32_t correctRowCount = 0; + + for (size_t i = 0; i < BATCH_SIZE; ++i) { + TLogMessage message; + message.Pk.Id = correctRowCount + lastNumber; + message.Pk.App = "App_" + std::to_string(logOffset % 10); + message.Pk.Host = "192.168.0." + std::to_string(logOffset % 11); + message.Pk.Timestamp = TInstant::Now() + TDuration::MilliSeconds(i % 1000); + message.HttpCode = 200; + message.Message = i % 2 ? "GET / HTTP/1.1" : "GET /images/logo.png HTTP/1.1"; + logBatch.emplace_back(message); + + correctSumApp += logOffset % 10; + correctSumHost += logOffset % 11; + ++correctRowCount; + + } + return {correctSumApp, correctSumHost, correctRowCount}; +} + +TStatus WriteLogBatch(TTableClient& tableClient, const std::string& table, const std::vector& logBatch, + const TRetryOperationSettings& retrySettings) { + TValueBuilder rows; + rows.BeginList(); + for (const auto& message : logBatch) { + rows.AddListItem() + .BeginStruct() + .AddMember("Id").Uint64(message.Pk.Id) + .AddMember("App").Utf8(message.Pk.App) + .AddMember("Host").Utf8(message.Pk.Host) + .AddMember("Timestamp").Timestamp(message.Pk.Timestamp) + .AddMember("HttpCode").Uint32(message.HttpCode) + .AddMember("Message").Utf8(message.Message) + .EndStruct(); + } + rows.EndList(); + auto bulkUpsertOperation = [table, rowsValue = rows.Build()](TTableClient& tableClient) { + TValue r = rowsValue; + auto status = tableClient.BulkUpsert(table, std::move(r)); + return status.GetValueSync(); + }; + + auto status = tableClient.RetryOperationSync(bulkUpsertOperation, retrySettings); + return status; +} + +static TStatus SelectTransaction(TSession session, const std::string& path, + std::optional& resultSet) { + std::filesystem::path filesystemPath(path); + auto query = std::format(R"( + PRAGMA TablePathPrefix("{}"); + + SELECT + SUM(CAST(SUBSTRING(CAST(App as string), 4) as Int32)), + SUM(CAST(SUBSTRING(CAST(Host as string), 10) as Int32)), + COUNT(*) + FROM {} + )", filesystemPath.parent_path().string(), filesystemPath.filename().string()); + + auto txControl = + TTxControl::BeginTx(TTxSettings::SerializableRW()) + .CommitTx(); + + auto result = session.ExecuteDataQuery(query, txControl).GetValueSync(); + + if (result.IsSuccess()) { + resultSet = result.GetResultSet(0); + } + + return result; +} + +TStatistic Select(TTableClient& client, const std::string& path) { + std::optional resultSet; + NStatusHelpers::ThrowOnError(client.RetryOperationSync([path, &resultSet](TSession session) { + return SelectTransaction(session, path, resultSet); + })); + + TResultSetParser parser(*resultSet); + + uint64_t sumApp = 0; + uint64_t sumHost = 0; + uint64_t rowCount = 0; + + if (parser.ColumnsCount() != 3 || parser.RowsCount() != 1) { + throw NStatusHelpers::TYdbErrorException(TStatus(EStatus::GENERIC_ERROR, + {NYdb::NIssue::TIssue("The number of columns should be: 3.\nThe number of rows should be: 1")})); + } + + if (parser.TryNextRow()) { + sumApp = *parser.ColumnParser("column0").GetOptionalInt64(); + sumHost = *parser.ColumnParser("column1").GetOptionalInt64(); + rowCount = parser.ColumnParser("column2").GetUint64(); + } + + return {sumApp, sumHost, rowCount}; +} + +void DropTable(TTableClient& client, const std::string& path) { + NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + return session.DropTable(path).ExtractValueSync(); + })); +} diff --git a/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.h b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.h new file mode 100644 index 000000000000..e6d13a5d7f71 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/bulk_upsert.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +using namespace NYdb; +using namespace NYdb::NTable; + +struct TRunArgs { + TDriver Driver; + std::string Path; +}; + +struct TLogMessage { + struct TPrimaryKeyLogMessage { + std::string App; + std::string Host; + TInstant Timestamp; + uint64_t Id; + }; + + TPrimaryKeyLogMessage Pk; + uint32_t HttpCode; + std::string Message; +}; + +struct TStatistic { + uint64_t SumApp; + uint64_t SumHost; + uint64_t RowCount; +}; + +TRunArgs GetRunArgs(); +TStatus CreateTable(TTableClient& client, const std::string& table); +TStatistic GetLogBatch(uint64_t logOffset, std::vector& logBatch, uint32_t lastNumber); +TStatus WriteLogBatch(TTableClient& tableClient, const std::string& table, const std::vector& logBatch, + const TRetryOperationSettings& retrySettings); +TStatistic Select(TTableClient& client, const std::string& path); +void DropTable(TTableClient& client, const std::string& path); diff --git a/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/main.cpp b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/main.cpp new file mode 100644 index 000000000000..589f0ff3d99e --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/main.cpp @@ -0,0 +1,54 @@ +#include "bulk_upsert.h" + +#include + +#include + + +TEST(Integration, BulkUpsert) { + + uint32_t correctSumApp = 0; + uint32_t correctSumHost = 0; + uint32_t correctRowCount = 0; + + auto [driver, path] = GetRunArgs(); + + TTableClient client(driver); + uint32_t count = 1000; + TStatus statusCreate = CreateTable(client, path); + if (!statusCreate.IsSuccess()) { + FAIL() << "Create table failed with status: " << ToString(statusCreate) << std::endl; + } + + TRetryOperationSettings writeRetrySettings; + writeRetrySettings + .Idempotent(true) + .MaxRetries(20); + + std::vector logBatch; + for (uint32_t offset = 0; offset < count; ++offset) { + + auto [batchSumApp, batchSumHost, batchRowCount] = GetLogBatch(offset, logBatch, correctRowCount); + correctSumApp += batchSumApp; + correctSumHost += batchSumHost; + correctRowCount += batchRowCount; + + TStatus statusWrite = WriteLogBatch(client, path, logBatch, writeRetrySettings); + if (!statusWrite.IsSuccess()) { + FAIL() << "Write failed with status: " << ToString(statusWrite) << std::endl; + } + } + + try { + auto [sumApp, sumHost, rowCount] = Select(client, path); + EXPECT_EQ(rowCount, correctRowCount); + EXPECT_EQ(sumApp, correctSumApp); + EXPECT_EQ(sumHost, correctSumHost); + } catch (const NYdb::NStatusHelpers::TYdbErrorException& e) { + driver.Stop(true); + FAIL() << "Execution failed due to fatal error:\n" << e.what() << std::endl; + } + + DropTable(client, path); + driver.Stop(true); +} diff --git a/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/ya.make b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/ya.make new file mode 100644 index 000000000000..9e09c3705fdc --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/bulk_upsert_simple_it/ya.make @@ -0,0 +1,25 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc) + +IF (SANITIZER_TYPE == "thread") + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table +) + +SRCS( + bulk_upsert.cpp + main.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/integration/ya.make b/ydb/public/sdk/cpp/tests/integration/ya.make new file mode 100644 index 000000000000..bf1a4d52ca4a --- /dev/null +++ b/ydb/public/sdk/cpp/tests/integration/ya.make @@ -0,0 +1,4 @@ +RECURSE( + basic_example_it + bulk_upsert_simple_it +) diff --git a/ydb/public/sdk/cpp/tests/unit/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..1f0962419e46 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(client) +add_subdirectory(library) diff --git a/ydb/public/sdk/cpp/tests/unit/client/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/client/CMakeLists.txt new file mode 100644 index 000000000000..b9fea2c44398 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/CMakeLists.txt @@ -0,0 +1,119 @@ +add_subdirectory(draft/helpers) + +add_ydb_test(NAME client-ydb_coordination_ut + SOURCES + coordination/coordination_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + YDB-CPP-SDK::Coordination + api-grpc + LABELS + unit +) + +add_ydb_test(NAME client-extensions-discovery_mutator_ut + SOURCES + discovery_mutator/discovery_mutator_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + YDB-CPP-SDK::DiscoveryMutator + YDB-CPP-SDK::Table + LABELS + unit +) + +add_ydb_test(NAME client-draft_ut + SOURCES + draft/ydb_scripting_response_headers_ut.cpp + draft/ydb_view_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + YDB-CPP-SDK::Draft + LABELS + unit +) +target_link_libraries(client-draft_ut + PRIVATE + client-draft_ut_helpers +) + +add_ydb_test(NAME client-ydb_driver_ut + SOURCES + driver/driver_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + YDB-CPP-SDK::Driver + YDB-CPP-SDK::Table + LABELS + unit +) + +add_ydb_test(NAME client-impl-ydb_endpoints_ut + INCLUDE_DIRS + ${YDB_SDK_SOURCE_DIR}/src/client/impl/ydb_endpoints + SOURCES + endpoints/endpoints_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + client-impl-ydb_endpoints + LABELS + unit +) + +add_ydb_test(NAME client-oauth2_ut + SOURCES + oauth2_token_exchange/credentials_ut.cpp + oauth2_token_exchange/jwt_token_source_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + http-server + json + string_utils-base64 + client-ydb_types-credentials-oauth2 + LABELS + unit +) + +# add_ydb_test(NAME client-ydb_params_ut +# SOURCES +# params/params_ut.cpp +# LINK_LIBRARIES +# yutil +# cpp-testing-unittest_main +# YDB-CPP-SDK::Params +# YDB-CPP-SDK::YsonValue +# LABELS +# unit +# ) + +add_ydb_test(NAME client-ydb_result_ut + SOURCES + result/result_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + YDB-CPP-SDK::Result + YDB-CPP-SDK::Params + LABELS + unit +) + +# add_ydb_test(NAME client-ydb_value_ut +# SOURCES +# value/value_ut.cpp +# LINK_LIBRARIES +# yutil +# cpp-testing-unittest_main +# YDB-CPP-SDK::Value +# YDB-CPP-SDK::JsonValue +# YDB-CPP-SDK::YsonValue +# YDB-CPP-SDK::Params +# LABELS +# unit +# ) diff --git a/ydb/public/sdk/cpp/tests/unit/client/coordination/coordination_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/coordination/coordination_ut.cpp new file mode 100644 index 000000000000..6867d679838c --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/coordination/coordination_ut.cpp @@ -0,0 +1,298 @@ +#include + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +using namespace NYdb; +using namespace NYdb::NCoordination; + +namespace { + + class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { + public: + grpc::Status ListEndpoints( + grpc::ServerContext* context, + const Ydb::Discovery::ListEndpointsRequest* request, + Ydb::Discovery::ListEndpointsResponse* response) override + { + Y_UNUSED(context); + + std::cerr << "ListEndpoints: " << request->ShortDebugString() << std::endl; + + const auto* result = MapFindPtr(MockResults, request->database()); + Y_ABORT_UNLESS(result, "Mock service doesn't have a result for database '%s'", request->database().c_str()); + + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result()->PackFrom(*result); + return grpc::Status::OK; + } + + // From database name to result + std::unordered_map MockResults; + }; + + class TMockCoordinationService : public Ydb::Coordination::V1::CoordinationService::Service { + public: + grpc::Status Session( + grpc::ServerContext* context, + grpc::ServerReaderWriter< + Ydb::Coordination::SessionResponse, + Ydb::Coordination::SessionRequest>* stream) override + { + Y_UNUSED(context); + + std::cerr << "Session stream started" << std::endl; + + Ydb::Coordination::SessionRequest request; + + // Process session start + { + if (!stream->Read(&request)) { + // Disconnected before the request was sent + return grpc::Status::OK; + } + std::cerr << "Session request: " << request.ShortDebugString() << std::endl; + Y_ABORT_UNLESS(request.has_session_start(), "Expected session start"); + auto& start = request.session_start(); + uint64_t sessionId = start.session_id(); + if (!sessionId) { + sessionId = ++LastSessionId; + } + Ydb::Coordination::SessionResponse response; + auto* started = response.mutable_session_started(); + started->set_session_id(sessionId); + started->set_timeout_millis(start.timeout_millis()); + stream->Write(response); + request.Clear(); + } + + size_t pings_received = 0; + while (stream->Read(&request)) { + std::cerr << "Session request: " << request.ShortDebugString() << std::endl; + Y_ABORT_UNLESS(request.has_ping(), "Only ping requests are supported"); + if (++pings_received <= 2) { + // Only reply to the first 2 ping requests + Ydb::Coordination::SessionResponse response; + auto* pong = response.mutable_pong(); + pong->set_opaque(request.ping().opaque()); + stream->Write(response); + } + request.Clear(); + } + + return grpc::Status::OK; + } + + private: + std::atomic LastSessionId{ 0 }; + }; + + template + std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + return builder.BuildAndStart(); + } + +} // namespace + +Y_UNIT_TEST_SUITE(Coordination) { + + Y_UNIT_TEST(SessionStartTimeout) { + TPortManager pm; + + // Start a fake endpoint on a random port + ui16 fakeEndpointPort = pm.GetPort(); + TInet6StreamSocket fakeEndpointSocket; + { + TSockAddrInet6 addr("::", fakeEndpointPort); + SetReuseAddressAndPort(fakeEndpointSocket); + Y_ABORT_UNLESS(fakeEndpointSocket.Bind(&addr) == 0, + "Failed to bind to port %" PRIu16, fakeEndpointPort); + Y_ABORT_UNLESS(fakeEndpointSocket.Listen(1) == 0, + "Failed to listen on port %" PRIu16, fakeEndpointPort); + } + + // Fill mock discovery service with our fake database + TMockDiscoveryService discoveryService; + { + auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("localhost"); + endpoint->set_port(fakeEndpointPort); + } + + // Start our mock discovery service + ui16 discoveryPort = pm.GetPort(); + std::string discoveryAddr = TStringBuilder() << "0.0.0.0:" << discoveryPort; + auto discoveryServer = StartGrpcServer(discoveryAddr, discoveryService); + + auto config = TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB"); + TDriver driver(config); + TClient client(driver); + + auto settings = TSessionSettings() + .Timeout(TDuration::MilliSeconds(500)); + + // We expect either connection or session start timeout + auto startTimestamp = TInstant::Now(); + auto res = client.StartSession("/Some/Path", settings).ExtractValueSync(); + auto endTimestamp = TInstant::Now(); + auto elapsed = endTimestamp - startTimestamp; + + std::cerr << "Got: " << ToString(res.GetStatus()) << ": " << res.GetIssues().ToString() << std::endl; + + // Both connection and session timeout return EStatus::TIMEOUT + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::TIMEOUT, res.GetIssues().ToString()); + + // Our timeout is 500ms, but allow up to 5 seconds of slack on very busy servers + UNIT_ASSERT_C(elapsed < TDuration::Seconds(5), "Timeout after too much time: " << elapsed); + } + + Y_UNIT_TEST(SessionPingTimeout) { + TPortManager pm; + + // Start a fake coordination service + TMockCoordinationService coordinationService; + ui16 coordinationPort = pm.GetPort(); + auto coordinationServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << coordinationPort, + coordinationService); + + // Fill a fake discovery service + TMockDiscoveryService discoveryService; + { + auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("localhost"); + endpoint->set_port(coordinationPort); + } + + // Start a fake discovery service + ui16 discoveryPort = pm.GetPort(); + auto discoveryServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << discoveryPort, + discoveryService); + + // Create a driver and a client + auto config = TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB"); + TDriver driver(config); + TClient client(driver); + + auto stoppedPromise = NThreading::NewPromise(); + auto stoppedFuture = stoppedPromise.GetFuture(); + auto settings = TSessionSettings() + .OnStateChanged([](auto state) { + std::cerr << "Session state: " << ToString(state) << std::endl; + }) + .OnStopped([stoppedPromise]() mutable { + stoppedPromise.SetValue(); + }) + .Timeout(TDuration::MilliSeconds(1000)); + + auto startTimestamp = TInstant::Now(); + auto res = client.StartSession("/Some/Path", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); + + auto session = res.ExtractResult(); + stoppedFuture.Wait(); + auto endTimestamp = TInstant::Now(); + auto elapsed = endTimestamp - startTimestamp; + + // We expect first few pings to succeed + UNIT_ASSERT_C(elapsed > TDuration::Seconds(1), "Elapsed time too short: " << elapsed); + + // We expect timeout to hit pretty soon afterwards + UNIT_ASSERT_C(elapsed < TDuration::Seconds(4), "Elapsed time too large: " << elapsed); + + // Check the last failure stored in a session + auto res2 = session.Close().ExtractValueSync(); + std::cerr << "Close: " << ToString(res2.GetStatus()) << ": " << res2.GetIssues().ToString() << std::endl; + UNIT_ASSERT_VALUES_EQUAL_C(res2.GetStatus(), EStatus::TIMEOUT, res2.GetIssues().ToString()); + } + + Y_UNIT_TEST(SessionCancelByDriver) { + TPortManager pm; + + // Start a fake coordination service + TMockCoordinationService coordinationService; + ui16 coordinationPort = pm.GetPort(); + auto coordinationServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << coordinationPort, + coordinationService); + + // Fill a fake discovery service + TMockDiscoveryService discoveryService; + { + auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("localhost"); + endpoint->set_port(coordinationPort); + } + + // Start a fake discovery service + ui16 discoveryPort = pm.GetPort(); + auto discoveryServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << discoveryPort, + discoveryService); + + // Create a driver and a client + auto config = TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB"); + std::optional driver(std::in_place, config); + std::optional client(std::in_place, *driver); + + auto stoppedPromise = NThreading::NewPromise(); + auto stoppedFuture = stoppedPromise.GetFuture(); + auto settings = TSessionSettings() + .OnStateChanged([](auto state) { + std::cerr << "Session state: " << ToString(state) << std::endl; + }) + .OnStopped([stoppedPromise]() mutable { + stoppedPromise.SetValue(); + }) + .Timeout(TDuration::MilliSeconds(1000)); + + auto startTimestamp = TInstant::Now(); + auto res = client->StartSession("/Some/Path", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); + + auto session = res.ExtractResult(); + + // Stop and destroy driver, forcing session to be cancelled + client.reset(); + driver->Stop(); + driver.reset(); + + stoppedFuture.Wait(); + auto endTimestamp = TInstant::Now(); + auto elapsed = endTimestamp - startTimestamp; + + // We expect session to be cancelled promptly + UNIT_ASSERT_C(elapsed < TDuration::Seconds(1), "Elapsed time too large: " << elapsed); + + // Check the last failure stored in a session + auto res2 = session.Close().ExtractValueSync(); + std::cerr << "Close: " << ToString(res2.GetStatus()) << ": " << res2.GetIssues().ToString() << std::endl; + UNIT_ASSERT_VALUES_EQUAL_C(res2.GetStatus(), EStatus::CLIENT_CANCELLED, res2.GetIssues().ToString()); + } + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/coordination/ya.make b/ydb/public/sdk/cpp/tests/unit/client/coordination/ya.make new file mode 100644 index 000000000000..a92953b8120b --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/coordination/ya.make @@ -0,0 +1,23 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + ydb/public/api/grpc + ydb/public/sdk/cpp/src/client/coordination +) + +SRCS( + coordination_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp new file mode 100644 index 000000000000..12061641d777 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp @@ -0,0 +1,40 @@ +#include + +#include +#include +#include + +#include +#include + +using namespace NYdb; +using namespace NDiscoveryMutator; + +Y_UNIT_TEST_SUITE(DiscoveryMutator) { + Y_UNIT_TEST(Simple) { + + std::unordered_set dbs; + std::string discoveryEndpont = "localhost:100"; + + auto mutator = [&](Ydb::Discovery::ListEndpointsResult* proto, TStatus status, const IDiscoveryMutatorApi::TAuxInfo& aux) { + UNIT_ASSERT_VALUES_EQUAL(discoveryEndpont, status.GetEndpoint()); + UNIT_ASSERT_VALUES_EQUAL(discoveryEndpont, aux.DiscoveryEndpoint); + dbs.insert(aux.Database); + Y_UNUSED(proto); + return status; + }; + auto driver = TDriver( + TDriverConfig() + .SetDatabase("db1") + .SetEndpoint(discoveryEndpont)); + + driver.AddExtension(TDiscoveryMutator::TParams(std::move(mutator))); + + auto clientSettings = NTable::TClientSettings(); + clientSettings.Database("db2"); + + // By default this is sync operation + auto client = NTable::TTableClient(driver, clientSettings); + UNIT_ASSERT(dbs.find("db2") != dbs.end()); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/ya.make b/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/ya.make new file mode 100644 index 000000000000..f1f7daf6818b --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/discovery_mutator/ya.make @@ -0,0 +1,23 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + ydb/public/sdk/cpp/src/client/extension_common + ydb/public/sdk/cpp/src/client/table +) + +SRCS( + discovery_mutator_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/CMakeLists.txt new file mode 100644 index 000000000000..28133bff0e88 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/CMakeLists.txt @@ -0,0 +1,13 @@ +_ydb_sdk_add_library(client-draft_ut_helpers) + +target_link_libraries(client-draft_ut_helpers + PRIVATE + api-grpc + api-grpc-draft +) + +target_sources(client-draft_ut_helpers + PRIVATE + grpc_services/scripting.cpp + grpc_services/view.cpp +) diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_server.h b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_server.h new file mode 100644 index 000000000000..af3a9df8fed5 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_server.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +#include + +namespace NYdb::inline V3 { + +template +std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + return builder.BuildAndStart(); +} + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.cpp b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.cpp new file mode 100644 index 000000000000..f798c0548082 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.cpp @@ -0,0 +1,21 @@ +#include "scripting.h" + +namespace NYdb::inline V3::NScripting { + +grpc::Status TMockSlyDbProxy::ExecuteYql( + grpc::ServerContext* context, + [[maybe_unused]] const Ydb::Scripting::ExecuteYqlRequest* request, + Ydb::Scripting::ExecuteYqlResponse* response +) { + context->AddInitialMetadata("key", "value"); + + // Just to make sdk core happy + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result(); + + return grpc::Status::OK; +} + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.h b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.h new file mode 100644 index 000000000000..fcf4232f8e92 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/scripting.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NScripting { + +class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service +{ +public: + grpc::Status ExecuteYql( + grpc::ServerContext* context, + const Ydb::Scripting::ExecuteYqlRequest* request, + Ydb::Scripting::ExecuteYqlResponse* response) override; +}; + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.cpp b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.cpp new file mode 100644 index 000000000000..82e7335a9d56 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.cpp @@ -0,0 +1,19 @@ +#include "view.h" + +namespace NYdb::inline V3::NView { + +grpc::Status TViewDummyService::DescribeView( + [[maybe_unused]] grpc::ServerContext* context, + [[maybe_unused]] const Ydb::View::DescribeViewRequest* request, + Ydb::View::DescribeViewResponse* response +) { + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + Ydb::View::DescribeViewResult describeResult; + describeResult.set_query_text(DummyQueryText); + op->mutable_result()->PackFrom(describeResult); + return grpc::Status::OK; +} + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.h b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.h new file mode 100644 index 000000000000..98498635367f --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/grpc_services/view.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace NYdb::inline V3::NView { + +constexpr const char* DummyQueryText = "select 42"; + +class TViewDummyService : public Ydb::View::V1::ViewService::Service +{ +public: + grpc::Status DescribeView( + grpc::ServerContext* context, + const Ydb::View::DescribeViewRequest* request, + Ydb::View::DescribeViewResponse* response) override; +}; + +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/ya.make b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/ya.make new file mode 100644 index 000000000000..b74a9875830f --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/helpers/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/grpc/draft +) + +SRCS( + grpc_services/scripting.cpp + grpc_services/view.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/ya.make b/ydb/public/sdk/cpp/tests/unit/client/draft/ya.make new file mode 100644 index 000000000000..fb0e05a0945a --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/ya.make @@ -0,0 +1,24 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/tests/unit/client/draft/helpers +) + +SRCS( + ydb_scripting_response_headers_ut.cpp + ydb_view_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_scripting_response_headers_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_scripting_response_headers_ut.cpp new file mode 100644 index 000000000000..342b726b6828 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_scripting_response_headers_ut.cpp @@ -0,0 +1,30 @@ +#include "helpers/grpc_server.h" +#include "helpers/grpc_services/scripting.h" + +#include + +#include + +using namespace NYdb; +using namespace NYdb::V3::NScripting; + +Y_UNIT_TEST_SUITE(ResponseHeaders) { + Y_UNIT_TEST(PassHeader) { + TMockSlyDbProxy slyDbProxy; + + std::string addr = "localhost:10000"; + + auto server = StartGrpcServer(addr, slyDbProxy); + + auto config = TDriverConfig() + .SetEndpoint(addr); + TDriver driver(config); + TScriptingClient client(driver); + + auto result = client.ExecuteYqlScript("SMTH").GetValueSync(); + auto metadata = result.GetResponseMetadata(); + + UNIT_ASSERT(metadata.find("key") != metadata.end()); + UNIT_ASSERT_VALUES_EQUAL(metadata.find("key")->second, "value"); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_view_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_view_ut.cpp new file mode 100644 index 000000000000..73ea9b90b61e --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/draft/ydb_view_ut.cpp @@ -0,0 +1,28 @@ +#include "helpers/grpc_server.h" +#include "helpers/grpc_services/view.h" + +#include + +#include + +using namespace NYdb; +using namespace NYdb::V3::NView; + +Y_UNIT_TEST_SUITE(ViewClient) { + Y_UNIT_TEST(Basic) { + TString addr = "localhost:2000"; + TViewDummyService viewService; + + auto server = StartGrpcServer(addr, viewService); + + auto config = TDriverConfig().SetEndpoint(addr); + TDriver driver(config); + TViewClient client(driver); + + auto result = client.DescribeView("any").ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto queryText = result.GetViewDescription().GetQueryText(); + UNIT_ASSERT_STRINGS_EQUAL(queryText, DummyQueryText); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp new file mode 100644 index 000000000000..803ed512e010 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp @@ -0,0 +1,186 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +#include + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace { + + class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { + public: + grpc::Status ListEndpoints( + grpc::ServerContext* context, + const Ydb::Discovery::ListEndpointsRequest* request, + Ydb::Discovery::ListEndpointsResponse* response) override + { + Y_UNUSED(context); + + std::cerr << "ListEndpoints: " << request->ShortDebugString() << std::endl; + + const auto* result = MapFindPtr(MockResults, request->database()); + Y_ABORT_UNLESS(result, "Mock service doesn't have a result for database '%s'", request->database().c_str()); + + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result()->PackFrom(*result); + return grpc::Status::OK; + } + + // From database name to result + std::unordered_map MockResults; + }; + + class TMockTableService : public Ydb::Table::V1::TableService::Service { + public: + grpc::Status CreateSession( + grpc::ServerContext* context, + const Ydb::Table::CreateSessionRequest* request, + Ydb::Table::CreateSessionResponse* response) override + { + Y_UNUSED(context); + + std::cerr << "CreateSession: " << request->ShortDebugString() << std::endl; + + Ydb::Table::CreateSessionResult result; + result.set_session_id("my-session-id"); + + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result()->PackFrom(result); + return grpc::Status::OK; + } + }; + + template + std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + return builder.BuildAndStart(); + } + +} // namespace + +Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { + Y_UNIT_TEST(ConnectWrongPort) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100")); + auto client = NTable::TTableClient(driver); + auto sessionFuture = client.CreateSession(); + + UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10))); + } + + Y_UNIT_TEST(ConnectWrongPortRetry) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100")); + auto client = NTable::TTableClient(driver); + + std::atomic_int counter = 0; + std::function handler = + [&handler, &counter, client] (const NTable::TAsyncCreateSessionResult& future) mutable { + UNIT_ASSERT_EQUAL(future.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); + UNIT_ASSERT_EXCEPTION(future.GetValue().GetSession(), NYdb::TContractViolation); + ++counter; + if (counter.load() > 4) { + return; + } + + auto f = client.CreateSession(); + + f.Apply(handler).GetValueSync(); + }; + + client.CreateSession().Apply(handler).GetValueSync(); + UNIT_ASSERT_EQUAL(counter, 5); + } + + Y_UNIT_TEST(TokenCharacters) { + auto checkToken = [](const std::string& token) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100") + .SetAuthToken(token)); + auto client = NTable::TTableClient(driver); + + auto result = client.CreateSession().GetValueSync(); + + return result.GetStatus(); + }; + + std::vector InvalidTokens = { + std::string("\t"), + std::string("\n"), + std::string("\r") + }; + for (auto& t : InvalidTokens) { + UNIT_ASSERT_EQUAL(checkToken(t), EStatus::CLIENT_UNAUTHENTICATED); + } + + std::vector ValidTokens = { + std::string("qwerty 1234 <>,.?/:;\"'\\|}{~`!@#$%^&*()_+=-"), + std::string() + }; + for (auto& t : ValidTokens) { + UNIT_ASSERT_EQUAL(checkToken(t), EStatus::TRANSPORT_UNAVAILABLE); + } + } + + Y_UNIT_TEST(UsingIpAddresses) { + TPortManager pm; + + // Start our mock table service + TMockTableService tableService; + ui16 tablePort = pm.GetPort(); + auto tableServer = StartGrpcServer( + TStringBuilder() << "127.0.0.1:" << tablePort, + tableService); + + // Start our mock discovery service + TMockDiscoveryService discoveryService; + { + auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("this.dns.name.is.not.reachable"); + endpoint->set_port(tablePort); + endpoint->add_ip_v4("127.0.0.1"); + } + ui16 discoveryPort = pm.GetPort(); + auto discoveryServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << discoveryPort, + discoveryService); + + auto driver = TDriver( + TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB")); + auto client = NTable::TTableClient(driver); + auto sessionFuture = client.CreateSession(); + + UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10))); + auto sessionResult = sessionFuture.ExtractValueSync(); + UNIT_ASSERT(sessionResult.IsSuccess()); + auto session = sessionResult.GetSession(); + UNIT_ASSERT_VALUES_EQUAL(session.GetId(), "my-session-id"); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/driver/ya.make b/ydb/public/sdk/cpp/tests/unit/client/driver/ya.make new file mode 100644 index 000000000000..6cc0a699a53e --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/driver/ya.make @@ -0,0 +1,23 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + ydb/public/sdk/cpp/src/client/driver + ydb/public/sdk/cpp/src/client/table +) + +SRCS( + driver_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/endpoints/endpoints_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/endpoints/endpoints_ut.cpp new file mode 100644 index 000000000000..da911f04bc80 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/endpoints/endpoints_ut.cpp @@ -0,0 +1,314 @@ +#include + +#include +#include +#include + +#include +#include + +#include + +using namespace NYdb; + +class TTestObj : public NYdb::TEndpointObj { +public: + TTestObj(int& counter) + : Counter_(&counter) + { } + + TTestObj() + : Counter_(nullptr) + { } + + ~TTestObj() { + if (ObjectRegistred() && Counter_) { + (*Counter_)++; + } + Unlink(); + } + + virtual void OnEndpointRemoved() { + HostRemoved_ = true; + } + + bool HostRemoved() const { + return HostRemoved_; + } + + bool HostRemoved_ = false; + + int* Counter_; +}; + +class TDiscoveryEmulator : public TThread { +public: + TDiscoveryEmulator(TEndpointElectorSafe& elector, ui64 maxEvents) + : TThread(&ThreadProc, this) + , Elector_(elector) + , MaxEvents_(maxEvents) + { + Finished_.store(false); + } + + static void* ThreadProc(void* _this) { + SetCurrentThreadName("TDiscoveryEmulator"); + static_cast(_this)->Exec(); + return nullptr; + } + + void Exec() { + for (ui64 i = 0; i < MaxEvents_; i++) { + ui8 mask = RandomNumber(16); + std::vector endpoints; + + for (size_t i = 0; i < Pool_.size(); i++) { + if (mask & (1 << i)) + endpoints.emplace_back(Pool_[i]); + } + + Elector_.SetNewState(std::move(endpoints)); + + if (i % 256 == 0) { + Sleep(TDuration::MilliSeconds(10)); + } + } + Finished_.store(true); + } + + const std::vector& GetPool() const { + return Pool_; + } + + bool Finished() const { + return Finished_.load(); + } + +private: + TEndpointElectorSafe& Elector_; + ui64 MaxEvents_; + std::atomic_bool Finished_; + + static const std::vector Pool_; +}; + +const std::vector TDiscoveryEmulator::Pool_ = std::vector{{"One", 1}, {"Two", 2}, {"Three", 3}, {"Four", 4}}; + +Y_UNIT_TEST_SUITE(CheckUtils) { + + Y_UNIT_TEST(NewPromiseInitialized) { + NThreading::TPromise promise = NThreading::NewPromise(); + UNIT_ASSERT(promise.Initialized()); + } + + Y_UNIT_TEST(PromiseDefaultCtorNotInitialized) { + NThreading::TPromise promise; + UNIT_ASSERT(!promise.Initialized()); + } +} + +Y_UNIT_TEST_SUITE(EndpointElector) { + + Y_UNIT_TEST(Empty) { + TEndpointElectorSafe elector; + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey()).Endpoint, ""); + } + + Y_UNIT_TEST(GetEndpoint) { + TEndpointElectorSafe elector; + elector.SetNewState(std::vector{{"Two", 0, "", 2}, {"One", 0, "", 1}}); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("One", 0), true).Endpoint, "One"); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("Two", 0), true).Endpoint, "Two"); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("", 1), true).NodeId, 1); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("", 2), true).NodeId, 2); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("Three", 0), true).Endpoint, ""); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("", 3), true).NodeId, 0); + } + + Y_UNIT_TEST(DiffOnRemove) { + TEndpointElectorSafe elector; + auto removed = elector.SetNewState(std::vector{{"Two", 2}, {"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(removed.size(), 0); + removed = elector.SetNewState(std::vector{{"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(removed.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(removed[0], std::string("Two")); + } + + Y_UNIT_TEST(Pessimization) { + TEndpointElectorSafe elector; + elector.SetNewState(std::vector{{"Two", 2}, {"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); + elector.PessimizeEndpoint("One"); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 50); + elector.SetNewState(std::vector{{"Two", 2}}); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); + } + + Y_UNIT_TEST(Election) { + TEndpointElectorSafe elector; + elector.SetNewState(std::vector{{"Two", 2}, {"One_A", 1}, {"Three", 3}, {"One_B", 1}}); + std::unordered_set endpoints; + // Just to make sure no possible to get more than expected + size_t extra_attempts = 1000; + while (endpoints.size() != 2 || --extra_attempts) { + endpoints.insert(elector.GetEndpoint(TEndpointKey()).Endpoint); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("Three", 0)).Endpoint, "Three"); + } + UNIT_ASSERT_VALUES_EQUAL(endpoints.size(), 2); + UNIT_ASSERT(endpoints.find("One_A") != endpoints.end()); + UNIT_ASSERT(endpoints.find("One_B") != endpoints.end()); + + elector.SetNewState(std::vector{{"One", 1}}); + // no preferred endpoint, expect avaliable + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey("Three", 0)).Endpoint, "One"); + UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint(TEndpointKey()).Endpoint, "One"); + } + + Y_UNIT_TEST(EndpointAssociationTwoThreadsNoRace) { + TEndpointElectorSafe elector; + + TDiscoveryEmulator emulator(elector, 10000); + + emulator.Start(); + + int counter1 = 0; + int counter2 = 0; + + std::vector> storage; + while (!emulator.Finished()) { + auto obj = std::make_unique(counter2); + if (elector.LinkObjToEndpoint(TEndpointKey("Two", 2), obj.get(), nullptr)) { + counter1++; + } + storage.emplace_back(std::move(obj)); + // collect some objects + if (counter1 % 10 == 0) + storage.clear(); + } + + emulator.Join(); + + storage.clear(); + UNIT_ASSERT_VALUES_EQUAL(counter1, counter2); + } + + Y_UNIT_TEST(EndpointAssiciationSingleThread) { + TEndpointElectorSafe elector; + elector.SetNewState(std::vector{{"Two", 2, "", 2}, {"One_A", 10, "", 10}, {"Three", 3, "", 3}, {"One_B", 4, "", 4}}); + + auto obj1 = std::make_unique(); + auto obj2 = std::make_unique(); + auto obj3 = std::make_unique(); + auto obj4 = std::make_unique(); + auto obj5 = std::make_unique(); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), false); + + UNIT_ASSERT(elector.LinkObjToEndpoint(TEndpointKey("Two", 2), obj1.get(), nullptr)); + // Registred of same object twice is not allowed + UNIT_ASSERT_VALUES_EQUAL(elector.LinkObjToEndpoint(TEndpointKey("Two", 2), obj1.get(), nullptr), false); + + UNIT_ASSERT(elector.LinkObjToEndpoint(TEndpointKey("Three", 3), obj2.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint(TEndpointKey("Three", 3), obj3.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint(TEndpointKey("One_B", 4), obj4.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint(TEndpointKey("One_B", 4), obj5.get(), (void*)1)); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), true); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); + + { + std::map sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](ui64 nodeId, const NYdb::IObjRegistryHandle& handle) { + sizes[nodeId] = handle.Size(); + i++; + }, 0, Max(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(2), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(10), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(4), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(3), 2); + } + + { + std::map sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](ui64 nodeId, const NYdb::IObjRegistryHandle& handle) { + sizes[nodeId] = handle.Size(); + i++; + }, 0, Max(), (void*)1); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(2), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(10), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(4), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(3), 0); + } + + UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), false); + + obj1.reset(); + + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); + + { + std::map sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](ui64 nodeId, const NYdb::IObjRegistryHandle& handle) { + sizes[nodeId] = handle.Size(); + i++; + }, 0, Max(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(2), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(10), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(4), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(3), 2); + } + + obj1.reset(new TTestObj()); + elector.LinkObjToEndpoint(TEndpointKey("Two", 2), obj1.get(), nullptr); + + elector.SetNewState(std::vector{{"Two", 2, "", 2}, {"One_A", 10, "", 10}, {"One_C", 1, "", 1}}); + + UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), true); + + { + std::map sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](ui64 nodeId, const NYdb::IObjRegistryHandle& handle) { + sizes[nodeId] = handle.Size(); + i++; + }, 0, Max(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(i, 3); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(2), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(1), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at(10), 0); + } + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); + + obj2.reset(); + + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 1); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/endpoints/ya.make b/ydb/public/sdk/cpp/tests/unit/client/endpoints/ya.make new file mode 100644 index 000000000000..8d37b6886fc2 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/endpoints/ya.make @@ -0,0 +1,22 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +SRCS( + endpoints_ut.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/impl/ydb_endpoints +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/CMakeLists.txt new file mode 100644 index 000000000000..9fef72f137c6 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/CMakeLists.txt @@ -0,0 +1,14 @@ +add_ydb_test(NAME client-oauth2_ut + SOURCES + credentials_ut.cpp + jwt_token_source_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + http-server + json + string_utils-base64 + client-ydb_types-credentials-oauth2 + LABELS + unit +) diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp new file mode 100644 index 000000000000..18fb90254fd5 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp @@ -0,0 +1,1241 @@ +#include +#include +#include "jwt_check_helper.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace NYdb; + +extern const std::string TestRSAPrivateKeyContent; +extern const std::string TestRSAPublicKeyContent; +extern const std::string TestECPrivateKeyContent; +extern const std::string TestECPublicKeyContent; +extern const std::string TestHMACSecretKeyBase64Content; + +class TTestTokenExchangeServer: public THttpServer::ICallBack { +public: + struct TCheck { + bool ExpectRequest = true; + HttpCodes StatusCode = HTTP_OK; + TCgiParameters ExpectedInputParams; + std::optional InputParams; + std::string Response; + std::string ExpectedErrorPart; + std::string Error; + std::optional SubjectJwtCheck; + std::optional ActorJwtCheck; + + void Check() { + UNIT_ASSERT_C(InputParams || !ExpectRequest, "Request error: " << Error); + if (InputParams) { + if (SubjectJwtCheck || ActorJwtCheck) { + TCgiParameters inputParamsCopy = *InputParams; + if (SubjectJwtCheck) { + std::string subjectJwt; + UNIT_ASSERT(inputParamsCopy.Has("subject_token")); + UNIT_ASSERT(inputParamsCopy.Has("subject_token_type", "urn:ietf:params:oauth:token-type:jwt")); + subjectJwt = inputParamsCopy.Get("subject_token"); + inputParamsCopy.Erase("subject_token"); + inputParamsCopy.Erase("subject_token_type"); + SubjectJwtCheck->Check(subjectJwt); + } + if (ActorJwtCheck) { + std::string actorJwt; + UNIT_ASSERT(inputParamsCopy.Has("actor_token")); + UNIT_ASSERT(inputParamsCopy.Has("actor_token_type", "urn:ietf:params:oauth:token-type:jwt")); + actorJwt = inputParamsCopy.Get("actor_token"); + inputParamsCopy.Erase("actor_token"); + inputParamsCopy.Erase("actor_token_type"); + ActorJwtCheck->Check(actorJwt); + } + UNIT_ASSERT_VALUES_EQUAL(ExpectedInputParams.Print(), inputParamsCopy.Print()); + } else { + UNIT_ASSERT_VALUES_EQUAL(ExpectedInputParams.Print(), InputParams->Print()); + } + } + + if (!ExpectedErrorPart.empty()) { + UNIT_ASSERT_STRING_CONTAINS(Error, ExpectedErrorPart); + } else { + UNIT_ASSERT(Error.empty()); + } + } + + void Reset() { + Error.clear(); + InputParams = std::nullopt; + } + }; + + class TRequest: public TRequestReplier { + public: + explicit TRequest(TTestTokenExchangeServer* server) + : Server(server) + { + } + + bool DoReply(const TReplyParams& params) override { + with_lock (Server->Lock) { + const TParsedHttpFull parsed(params.Input.FirstLine()); + UNIT_ASSERT_VALUES_EQUAL(parsed.Path, "/exchange/token"); + const std::string bodyStr = params.Input.ReadAll(); + + Server->Check.InputParams.emplace(bodyStr); + THttpResponse resp(Server->Check.StatusCode); + resp.SetContent(TString{Server->Check.Response}); + resp.OutTo(params.Output); + return true; + } + } + + public: + TTestTokenExchangeServer* Server = nullptr; + }; + + TTestTokenExchangeServer() + : HttpOptions(PortManager.GetPort()) + , HttpServer(this, HttpOptions) + { + HttpServer.Start(); + } + + TClientRequest* CreateClient() override { + return new TRequest(this); + } + + std::string GetEndpoint() const { + return TStringBuilder() << "http://localhost:" << HttpOptions.Port << "/exchange/token"; + } + + void Run(const std::function& f, bool checkExpectations = true) { + Check.Reset(); + try { + f(); + } catch (const std::exception& ex) { + Check.Error = ex.what(); + } + + if (checkExpectations) { + CheckExpectations(); + } + } + + void Run(const TOauth2TokenExchangeParams& params, const std::string& expectedToken = {}, bool checkExpectations = true) { + std::string token; + Run([&]() { + auto factory = CreateOauth2TokenExchangeCredentialsProviderFactory(params); + if (!expectedToken.empty()) { + token = factory->CreateProvider()->GetAuthInfo(); + } + }, + checkExpectations); + + if (!expectedToken.empty()) { + UNIT_ASSERT_VALUES_EQUAL(expectedToken, token); + } + } + + void RunFromConfig(const std::string& fileName, const std::string& expectedToken = {}, bool checkExpectations = true, const std::string& explicitTokenEndpoint = {}) { + std::string token; + Run([&]() { + auto factory = CreateOauth2TokenExchangeFileCredentialsProviderFactory(fileName, explicitTokenEndpoint); + if (!expectedToken.empty()) { + token = factory->CreateProvider()->GetAuthInfo(); + } + }, + checkExpectations); + + if (!expectedToken.empty()) { + UNIT_ASSERT_VALUES_EQUAL(expectedToken, token); + } + } + + void CheckExpectations() { + with_lock (Lock) { + Check.Check(); + } + } + + void WithLock(const std::function& f) { + with_lock (Lock) { + f(); + } + } + +public: + TAdaptiveLock Lock; + TPortManager PortManager; + THttpServer::TOptions HttpOptions; + THttpServer HttpServer; + TCheck Check; +}; + +template + struct TJsonFillerArray { + TJsonFillerArray(TParent* parent, NJson::TJsonWriter* writer) + : Parent(parent) + , Writer(writer) + { + Writer->OpenArray(); + } + + TJsonFillerArray& Value(const TStringBuf& value) { + Writer->Write(value); + return *this; + } + + TParent& Build() { + Writer->CloseArray(); + return *Parent; + } + + TParent* Parent = nullptr; + NJson::TJsonWriter* Writer = nullptr; + }; + + template + struct TJsonFiller { + TJsonFiller(NJson::TJsonWriter* writer) + : Writer(writer) + {} + + TDerived& Field(const TStringBuf& key, const TStringBuf& value) { + Writer->WriteKey(key); + Writer->Write(value); + return *static_cast(this); + } + + TJsonFillerArray Array(const TStringBuf& key) { + Writer->WriteKey(key); + return TJsonFillerArray(static_cast(this), Writer); + } + + NJson::TJsonWriter* Writer = nullptr; + }; + +struct TTestConfigFile : public TJsonFiller { + struct TSubMap : public TJsonFiller { + TSubMap(TTestConfigFile* parent, NJson::TJsonWriter* writer) + : TJsonFiller(writer) + , Parent(parent) + { + Writer->OpenMap(); + } + + TTestConfigFile& Build() { + Writer->CloseMap(); + return *Parent; + } + + TTestConfigFile* Parent = nullptr; + }; + + TTestConfigFile() + : TJsonFiller(&Writer) + , Writer(&Result.Out, true) + , TmpFile(MakeTempName(nullptr, "oauth2_cfg")) + { + Writer.OpenMap(); + } + + TSubMap SubMap(const TStringBuf& key) { + Writer.WriteKey(key); + return TSubMap(this, &Writer); + } + + TString Build() { + Writer.CloseMap(); + Writer.Flush(); + + TUnbufferedFileOutput(TmpFile.Name()).Write(Result); + + return TmpFile.Name(); + } + + TStringBuilder Result; + NJson::TJsonWriter Writer; + TTempFile TmpFile; +}; + +Y_UNIT_TEST_SUITE(TestTokenExchange) { + void Exchanges(bool fromConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.ExpectedInputParams.emplace("audience", "test_aud"); + server.Check.ExpectedInputParams.emplace("scope", "s1 s2"); + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bEareR", "expires_in": 42})"; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Field("aud", "test_aud") + .Array("scope") + .Value("s1") + .Value("s2") + .Build() + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Build(), + "Bearer hello_token" + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .Audience("test_aud") + .AppendScope("s1") + .AppendScope("s2") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")), + "Bearer hello_token" + ); + } + + server.Check.ExpectedInputParams.emplace("audience", "test_aud_2"); + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Array("aud") + .Value("test_aud") + .Value("test_aud_2") + .Build() + .Array("scope") + .Value("s1") + .Value("s2") + .Build() + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Build(), + "Bearer hello_token" + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendAudience("test_aud") + .AppendAudience("test_aud_2") + .AppendScope("s1") + .AppendScope("s2") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")), + "Bearer hello_token" + ); + } + + server.Check.ExpectedInputParams.erase("scope"); + server.Check.ExpectedInputParams.emplace("resource", "test_res"); + server.Check.ExpectedInputParams.emplace("actor_token", "act_token"); + server.Check.ExpectedInputParams.emplace("actor_token_type", "act_token_type"); + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Array("aud") + .Value("test_aud") + .Value("test_aud_2") + .Build() + .Field("res", "test_res") + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .SubMap("actor-credentials") + .Field("type", "Fixed") + .Field("token", "act_token") + .Field("token-type", "act_token_type") + .Build() + .Build(), + "Bearer hello_token" + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendAudience("test_aud") + .AppendAudience("test_aud_2") + .Resource("test_res") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) + .ActorTokenSource(CreateFixedTokenSource("act_token", "act_token_type")), + "Bearer hello_token" + ); + } + } + + Y_UNIT_TEST(Exchanges) { + Exchanges(false); + } + + Y_UNIT_TEST(ExchangesFromConfig) { + Exchanges(true); + } + + void BadParams(bool fromConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectRequest = false; + + server.Check.ExpectedErrorPart = "no token endpoint"; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + ); + } + + server.Check.ExpectedErrorPart = "empty audience"; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Array("aud") + .Value("a") + .Value("") + .Build() + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendAudience("a") + .AppendAudience("") + ); + } + + server.Check.ExpectedErrorPart = "empty scope"; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Array("scope") + .Value("s") + .Value("") + .Build() + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendScope("s") + .AppendScope("") + ); + } + + server.Check.ExpectedErrorPart = "failed to parse url"; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", "not an url") + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint("not an url") + ); + } + } + + Y_UNIT_TEST(BadParams) { + BadParams(false); + } + + Y_UNIT_TEST(BadParamsFromConfig) { + BadParams(true); + } + + void BadResponse(bool fromConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + + TOauth2TokenExchangeParams params; + TTestConfigFile cfg; + if (fromConfig) { + cfg + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Build(); + } else { + params + .TokenEndpoint(server.GetEndpoint()) + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")); + } + + auto run = [&]() { + if (fromConfig) { + server.RunFromConfig(cfg.TmpFile.Name()); + } else { + server.Run(params); + } + }; + + server.Check.Response = R"(})"; + server.Check.ExpectedErrorPart = "json parsing error"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type1": "bearer", "expires_in": 42})"; + server.Check.ExpectedErrorPart = "no field \"token_type\" in response"; + run(); + + server.Check.Response = R"({"access_token1": "hello_token", "token_type": "bearer", "expires_in": 42})"; + server.Check.ExpectedErrorPart = "no field \"access_token\" in response"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in1": 42})"; + server.Check.ExpectedErrorPart = "no field \"expires_in\" in response"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "abc", "expires_in": 42})"; + server.Check.ExpectedErrorPart = "unsupported token type: \"abc\""; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in": 0})"; + server.Check.ExpectedErrorPart = "incorrect expiration time: 0"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in": "hello"})"; + server.Check.ExpectedErrorPart = "incorrect expiration time: 0"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in": -1})"; + server.Check.ExpectedErrorPart = "incorrect expiration time: -1"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in": 1, "scope": "s"})"; + server.Check.ExpectedErrorPart = "different scope. Expected \"\", but got \"s\""; + run(); + + server.Check.Response = R"({"access_token": "", "token_type": "bearer", "expires_in": 1})"; + server.Check.ExpectedErrorPart = "got empty token"; + run(); + + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bearer", "expires_in": 1, "scope": "s a"})"; + server.Check.ExpectedErrorPart = "different scope. Expected \"a\", but got \"s a\""; + server.Check.ExpectedInputParams.emplace("scope", "a"); + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Field("scope", "a") + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) + .Scope("a") + ); + } + server.Check.ExpectedInputParams.erase("scope"); + + + server.Check.ExpectedErrorPart = "can not connect to"; + server.Check.ExpectRequest = false; + if (fromConfig) { + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", "https://localhost:42/aaa") + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Build() + ); + } else { + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint("https://localhost:42/aaa") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) + ); + } + server.Check.ExpectRequest = true; + + // parsing response + server.Check.StatusCode = HTTP_FORBIDDEN; + server.Check.Response = R"(not json)"; + server.Check.ExpectedErrorPart = "Exchange token error in Oauth 2 token exchange credentials provider: 403 Forbidden, could not parse response: "; + run(); + + server.Check.Response = R"({"error": "smth"})"; + server.Check.ExpectedErrorPart = "Exchange token error in Oauth 2 token exchange credentials provider: 403 Forbidden, error: smth"; + run(); + + server.Check.Response = R"({"error": "smth", "error_description": "something terrible happened"})"; + server.Check.ExpectedErrorPart = "Exchange token error in Oauth 2 token exchange credentials provider: 403 Forbidden, error: smth, description: something terrible happened"; + run(); + + server.Check.StatusCode = HTTP_BAD_REQUEST; + server.Check.Response = R"({"error_uri": "my_uri", "error_description": "something terrible happened"})"; + server.Check.ExpectedErrorPart = "Exchange token error in Oauth 2 token exchange credentials provider: 400 Bad request, description: something terrible happened, error_uri: my_uri"; + run(); + } + + Y_UNIT_TEST(BadResponse) { + BadResponse(false); + } + + Y_UNIT_TEST(BadResponseFromConfig) { + BadResponse(true); + } + + void UpdatesToken(bool fromConfig) { + TCredentialsProviderFactoryPtr factory; + + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 1})"; + server.Run( + [&]() { + if (fromConfig) { + factory = CreateOauth2TokenExchangeFileCredentialsProviderFactory( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Build() + .Build()); + } else { + factory = CreateOauth2TokenExchangeCredentialsProviderFactory( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); + } + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + } + ); + + server.WithLock( + [&]() { + server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 1})"; + } + ); + + Sleep(TDuration::Seconds(1)); + server.Run( + [&]() { + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); + } + ); + } + + Y_UNIT_TEST(UpdatesToken) { + UpdatesToken(false); + } + + Y_UNIT_TEST(UpdatesTokenFromConfig) { + UpdatesToken(true); + } + + Y_UNIT_TEST(UsesCachedToken) { + TCredentialsProviderFactoryPtr factory; + TInstant startTime; + + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("actor_token", "test_token"); + server.Check.ExpectedInputParams.emplace("actor_token_type", "test_token_type"); + server.Check.Response = R"({"access_token": "the_only_token", "token_type": "bearer", "expires_in": 20000})"; + server.Run( + [&]() { + factory = CreateOauth2TokenExchangeCredentialsProviderFactory( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .ActorTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); + startTime = TInstant::Now(); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer the_only_token"); + } + ); + + server.WithLock( + [&]() { + server.Check.StatusCode = HTTP_BAD_REQUEST; + server.Check.Response = R"(invalid response)"; + } + ); + + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer the_only_token"); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer the_only_token"); + } + + Y_UNIT_TEST(UpdatesTokenInBackgroud) { + TCredentialsProviderFactoryPtr factory; + TInstant startTime; + + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("actor_token", "test_token"); + server.Check.ExpectedInputParams.emplace("actor_token_type", "test_token_type"); + + for (int i = 0; i < 2; ++i) { + server.WithLock( + [&]() { + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 2})"; + } + ); + if (!factory) { + server.Run( + [&]() { + factory = CreateOauth2TokenExchangeCredentialsProviderFactory( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .ActorTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); + startTime = TInstant::Now(); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + } + ); + } + + server.WithLock( + [&]() { + server.Check.Reset(); + if (i == 0) { + server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 2})"; + } else { + server.Check.Response = R"({"access_token": "token_3", "token_type": "bearer", "expires_in": 2})"; + } + } + ); + + SleepUntil(startTime + TDuration::Seconds(1) + TDuration::MilliSeconds(5)); + const std::string token = factory->CreateProvider()->GetAuthInfo(); + TInstant halfTimeTokenValid = TInstant::Now(); + if (halfTimeTokenValid < startTime + TDuration::Seconds(2)) { // valid => got cached token, but async update must be run after half time token is valid + if (i == 0) { + UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_1"); + } else { + UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_2"); + } + do { + Sleep(TDuration::MilliSeconds(10)); + bool gotRequest = false; + server.WithLock( + [&]() { + if (server.Check.InputParams) { // InputParams are created => got the request + gotRequest = true; + } + } + ); + if (gotRequest) { + startTime = TInstant::Now(); // for second iteration + break; + } + } while (TInstant::Now() <= startTime + TDuration::Seconds(30)); + server.CheckExpectations(); + server.WithLock( + [&]() { + server.Check.Reset(); + server.Check.Response = R"(invalid response)"; // update must finish asyncronously + } + ); + Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token + if (i == 0) { // Finally check that we got updated token + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); + } else { + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_3"); + } + Cerr << "Checked backgroud update on " << i << " iteration" << Endl; + } + } + } + + Y_UNIT_TEST(UpdatesTokenAndRetriesErrors) { + TCredentialsProviderFactoryPtr factory; + + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 6})"; + server.Run( + [&]() { + factory = CreateOauth2TokenExchangeCredentialsProviderFactory( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + } + ); + + server.WithLock( + [&]() { + server.Check.Reset(); + server.Check.StatusCode = HTTP_BAD_REQUEST; // all errors are temporary, because the first attempt is always successful (in constructor) + server.Check.Response = R"({"error": "tmp", "error_description": "temporary error"})"; + } + ); + + Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + + auto waitRequest = [&](TDuration howLong) { + TInstant startTime = TInstant::Now(); + bool gotRequest = false; + do { + Sleep(TDuration::MilliSeconds(10)); + server.WithLock( + [&]() { + if (server.Check.InputParams) { // InputParams are created => got the request + gotRequest = true; + } + } + ); + if (gotRequest) { + break; + } + } while (TInstant::Now() <= startTime + howLong); + return gotRequest; + }; + + UNIT_ASSERT(waitRequest(TDuration::Seconds(30))); + + server.WithLock( + [&]() { + server.Check.Reset(); + server.Check.StatusCode = HTTP_OK; + server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 2})"; + } + ); + + UNIT_ASSERT(waitRequest(TDuration::Seconds(10))); + Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); + + server.WithLock( + [&]() { + server.Check.Reset(); + server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; + server.Check.Response = R"({})"; + } + ); + + Sleep(TDuration::Seconds(2)); + UNIT_ASSERT_EXCEPTION(factory->CreateProvider()->GetAuthInfo(), std::runtime_error); + } + + Y_UNIT_TEST(ShutdownWhileRefreshingToken) { + TCredentialsProviderFactoryPtr factory; + + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 6})"; + + server.Run( + [&]() { + factory = CreateOauth2TokenExchangeCredentialsProviderFactory( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + } + ); + + server.WithLock( + [&]() { + server.Check.Reset(); + server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; + server.Check.Response = R"({})"; + } + ); + + Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); + + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + + const TInstant shutdownStart = TInstant::Now(); + factory = nullptr; + const TInstant shutdownStop = TInstant::Now(); + Cerr << "Shutdown: " << (shutdownStop - shutdownStart) << Endl; + } + + Y_UNIT_TEST(ExchangesFromFileConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.ExpectedInputParams.emplace("audience", "test_aud"); + server.Check.ExpectedInputParams.emplace("scope", "s1 s2"); + server.Check.Response = R"({"access_token": "hello_token", "token_type": "bEareR", "expires_in": 42})"; + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .Audience("test_aud") + .AppendScope("s1") + .AppendScope("s2") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")), + "Bearer hello_token" + ); + + server.Check.ExpectedInputParams.emplace("audience", "test_aud_2"); + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendAudience("test_aud") + .AppendAudience("test_aud_2") + .AppendScope("s1") + .AppendScope("s2") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")), + "Bearer hello_token" + ); + + server.Check.ExpectedInputParams.erase("scope"); + server.Check.ExpectedInputParams.emplace("resource", "test_res"); + server.Check.ExpectedInputParams.emplace("resource", "test_res_2"); + server.Check.ExpectedInputParams.emplace("actor_token", "act_token"); + server.Check.ExpectedInputParams.emplace("actor_token_type", "act_token_type"); + server.Run( + TOauth2TokenExchangeParams() + .TokenEndpoint(server.GetEndpoint()) + .AppendAudience("test_aud") + .AppendAudience("test_aud_2") + .AppendResource("test_res") + .AppendResource("test_res_2") + .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) + .ActorTokenSource(CreateFixedTokenSource("act_token", "act_token_type")), + "Bearer hello_token" + ); + } + + Y_UNIT_TEST(SkipsUnknownFieldsInConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "test_grant_type"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "test_requested_token_type"); + server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); + server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); + server.Check.ExpectedInputParams.emplace("resource", "r1"); + server.Check.ExpectedInputParams.emplace("resource", "r2"); + server.Check.Response = R"({"access_token": "received_token", "token_type": "bEareR", "expires_in": 42})"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", "bla-bla-bla") // use explicit endpoint via param + .Field("unknown", "unknown value") + .Field("grant-type", "test_grant_type") + .Field("requested-token-type", "test_requested_token_type") + .Array("res") + .Value("r1") + .Value("r2") + .Build() + .SubMap("subject-credentials") + .Field("type", "Fixed") + .Field("token", "test_token") + .Field("token-type", "test_token_type") + .Field("unknown", "unknown value") + .Build() + .Build(), + "Bearer received_token", + true, + server.GetEndpoint() + ); + } + + Y_UNIT_TEST(JwtTokenSourceInConfig) { + TTestTokenExchangeServer server; + server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); + server.Check.SubjectJwtCheck.emplace() + .TokenTtl(TDuration::Hours(24)) + .KeyId("test_key_id") + .Issuer("test_iss") + .Subject("test_sub") + .Audience("test_aud") + .Id("test_jti") + .Alg(TestRSAPublicKeyContent); + server.Check.ActorJwtCheck.emplace() + .AppendAudience("a1") + .AppendAudience("a2"); + server.Check.Response = R"({"access_token": "received_token", "token_type": "bEareR", "expires_in": 42})"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("ttl", "24h") + .Field("kid", "test_key_id") + .Field("iss", "test_iss") + .Field("sub", "test_sub") + .Field("aud", "test_aud") + .Field("jti", "test_jti") + .Field("alg", "rs384") + .Field("private-key", TestRSAPrivateKeyContent) + .Field("unknown", "unknown value") + .Build() + .SubMap("actor-credentials") + .Field("type", "JWT") + .Array("aud") + .Value("a1") + .Value("a2") + .Build() + .Field("alg", "RS256") + .Field("private-key", TestRSAPrivateKeyContent) + .Build() + .Build(), + "Bearer received_token" + ); + + // Other signing methods + server.Check.SubjectJwtCheck.emplace() + .Id("jti") + .Alg(Base64Decode(TestHMACSecretKeyBase64Content)); + server.Check.ActorJwtCheck.emplace() + .Alg(TestECPublicKeyContent) + .Issuer("iss"); + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("jti", "jti") + .Field("alg", "HS384") + .Field("private-key", TestHMACSecretKeyBase64Content) + .Build() + .SubMap("actor-credentials") + .Field("type", "JWT") + .Field("alg", "ES256") + .Field("private-key", TestECPrivateKeyContent) + .Field("iss", "iss") + .Build() + .Build(), + "Bearer received_token" + ); + } + + Y_UNIT_TEST(BadConfigParams) { + TTestTokenExchangeServer server; + server.Check.ExpectRequest = false; + + // wrong format + TTempFile cfg(MakeTempName(nullptr, "oauth2_cfg")); + TUnbufferedFileOutput(cfg.Name()).Write(""); // empty + server.Check.ExpectedErrorPart = "Failed to parse config file"; + server.RunFromConfig(cfg.Name()); + + TUnbufferedFileOutput(cfg.Name()).Write("not a json"); + server.Check.ExpectedErrorPart = "Failed to parse config file"; + server.RunFromConfig(cfg.Name()); + + TUnbufferedFileOutput(cfg.Name()).Write("[\"not a map\"]"); + server.Check.ExpectedErrorPart = "Not a map"; + server.RunFromConfig(cfg.Name()); + + server.Check.ExpectedErrorPart = "No \"type\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type1", "unknown") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "Incorrect \"type\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "unknown") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "Failed to parse \"iss\""; // must be string + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Array("iss") + .Value("1") + .Build() + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "No \"token\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "fixed") + .Field("token-type", "test_token_type") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "No \"token-type\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "fixed") + .Field("token", "test_token") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "No \"alg\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("private-key", TestRSAPrivateKeyContent) + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "No \"private-key\" parameter"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "rs256") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "Failed to parse \"ttl\""; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "rs256") + .Field("private-key", TestRSAPrivateKeyContent) + .Field("ttl", "-1s") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "Algorithm \"algorithm\" is not supported. Supported algorithms are: "; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "algorithm") + .Field("private-key", TestRSAPrivateKeyContent) + .Build() + .Build() + ); +#ifdef YDB_SDK_USE_NEW_JWT + server.Check.ExpectedErrorPart = "failed to load key"; +#else + server.Check.ExpectedErrorPart = "failed to load private key"; +#endif + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "rs256") + .Field("private-key", "not a key") + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "failed to decode HMAC secret from Base64"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "hs256") + .Field("private-key", "\n\n") + .Build() + .Build() + ); + +#ifdef YDB_SDK_USE_NEW_JWT + server.Check.ExpectedErrorPart = "invalid key size"; +#else + server.Check.ExpectedErrorPart = "failed to load private key"; +#endif + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "es256") + .Field("private-key", TestRSAPrivateKeyContent) // Need EC key + .Build() + .Build() + ); + +#ifdef YDB_SDK_USE_NEW_JWT + server.Check.ExpectedErrorPart = "failed to load key"; +#else + server.Check.ExpectedErrorPart = "failed to load private key"; +#endif + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .SubMap("subject-credentials") + .Field("type", "jwt") + .Field("alg", "ps512") + .Field("private-key", TestHMACSecretKeyBase64Content) // Need RSA key + .Build() + .Build() + ); + + server.Check.ExpectedErrorPart = "Not a map"; + server.RunFromConfig( + TTestConfigFile() + .Field("token-endpoint", server.GetEndpoint()) + .Array("subject-credentials") + .Value("42") + .Build() + .Build() + ); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_check_helper.h b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_check_helper.h new file mode 100644 index 000000000000..3b23e2fbe2d0 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_check_helper.h @@ -0,0 +1,98 @@ +#include +#include + +#include + +extern const std::string TestRSAPrivateKeyContent; +extern const std::string TestRSAPublicKeyContent; + +struct TJwtCheck { + using TSelf = TJwtCheck; +#ifdef YDB_SDK_USE_NEW_JWT + using TDecodedJwt = jwt::decoded_jwt; +#else + using TDecodedJwt = jwt::decoded_jwt; +#endif + + struct IAlgCheck { + virtual void Check(const TDecodedJwt& decoded) const = 0; + virtual ~IAlgCheck() = default; + }; + + template + struct TAlgCheck : public IAlgCheck { + TAlgCheck(const std::string& publicKey) + : Alg(publicKey) + {} + + void Check(const TDecodedJwt& decoded) const override { + UNIT_ASSERT_VALUES_EQUAL(decoded.get_algorithm(), Alg.name()); + const std::string data = decoded.get_header_base64() + "." + decoded.get_payload_base64(); + const std::string signature = decoded.get_signature(); +#ifdef YDB_SDK_USE_NEW_JWT + std::error_code ec; + Alg.verify(data, signature, ec); + if (ec) { + throw std::runtime_error(ec.message()); + } +#else + Alg.verify(data, signature); // Throws +#endif + } + + TAlg Alg; + }; + + template + TSelf& Alg(const std::string& publicKey) { + Alg_.reset(new TAlgCheck(publicKey)); + return *this; + } + std::unique_ptr Alg_ = std::make_unique>(TestRSAPublicKeyContent); + + FLUENT_SETTING_OPTIONAL(std::string, KeyId); + + FLUENT_SETTING_OPTIONAL(std::string, Issuer); + FLUENT_SETTING_OPTIONAL(std::string, Subject); + FLUENT_SETTING_OPTIONAL(std::string, Id); + FLUENT_SETTING_VECTOR_OR_SINGLE(std::string, Audience); + FLUENT_SETTING_DEFAULT(TDuration, TokenTtl, TDuration::Hours(1)); + + void Check(const std::string& token) { + TDecodedJwt decoded(token); + + UNIT_ASSERT_VALUES_EQUAL(decoded.get_type(), "JWT"); + Alg_->Check(decoded); + + const auto now = std::chrono::system_clock::from_time_t(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now())); + UNIT_ASSERT(decoded.get_issued_at() >= now - std::chrono::minutes(10)); + UNIT_ASSERT(decoded.get_issued_at() <= now); + UNIT_ASSERT_EQUAL(decoded.get_expires_at() - decoded.get_issued_at(), std::chrono::seconds(TokenTtl_.Seconds())); + +#define CHECK_JWT_CLAIM(claim, option) \ + if (option) { \ + UNIT_ASSERT(decoded.has_ ## claim()); \ + UNIT_ASSERT_VALUES_EQUAL(decoded.get_ ## claim(), *option); \ + } else { \ + UNIT_ASSERT(!decoded.has_ ## claim()); \ + } + + CHECK_JWT_CLAIM(key_id, KeyId_); + CHECK_JWT_CLAIM(issuer, Issuer_); + CHECK_JWT_CLAIM(subject, Subject_); + CHECK_JWT_CLAIM(id, Id_); + +#undef CHECK_JWT_CLAIM + + if (!Audience_.empty()) { + UNIT_ASSERT(decoded.has_audience()); + const std::set aud = decoded.get_audience(); + UNIT_ASSERT_VALUES_EQUAL(aud.size(), Audience_.size()); + for (const std::string& a : Audience_) { + UNIT_ASSERT(aud.contains(a)); + } + } else { + UNIT_ASSERT(!decoded.has_audience()); + } + } +}; diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_token_source_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_token_source_ut.cpp new file mode 100644 index 000000000000..ddc9ecb05c7e --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/jwt_token_source_ut.cpp @@ -0,0 +1,63 @@ +#include +#include "jwt_check_helper.h" + +#include + +#include + +using namespace NYdb; + +extern const std::string TestRSAPrivateKeyContent = "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC75/JS3rMcLJxv\nFgpOzF5+2gH+Yig3RE2MTl9uwC0BZKAv6foYr7xywQyWIK+W1cBhz8R4LfFmZo2j\nM0aCvdRmNBdW0EDSTnHLxCsFhoQWLVq+bI5f5jzkcoiioUtaEpADPqwgVULVtN/n\nnPJiZ6/dU30C3jmR6+LUgEntUtWt3eq3xQIn5lG3zC1klBY/HxtfH5Hu8xBvwRQT\nJnh3UpPLj8XwSmriDgdrhR7o6umWyVuGrMKlLHmeivlfzjYtfzO1MOIMG8t2/zxG\nR+xb4Vwks73sH1KruH/0/JMXU97npwpe+Um+uXhpldPygGErEia7abyZB2gMpXqr\nWYKMo02NAgMBAAECggEAO0BpC5OYw/4XN/optu4/r91bupTGHKNHlsIR2rDzoBhU\nYLd1evpTQJY6O07EP5pYZx9mUwUdtU4KRJeDGO/1/WJYp7HUdtxwirHpZP0lQn77\nuccuX/QQaHLrPekBgz4ONk+5ZBqukAfQgM7fKYOLk41jgpeDbM2Ggb6QUSsJISEp\nzrwpI/nNT/wn+Hvx4DxrzWU6wF+P8kl77UwPYlTA7GsT+T7eKGVH8xsxmK8pt6lg\nsvlBA5XosWBWUCGLgcBkAY5e4ZWbkdd183o+oMo78id6C+PQPE66PLDtHWfpRRmN\nm6XC03x6NVhnfvfozoWnmS4+e4qj4F/emCHvn0GMywKBgQDLXlj7YPFVXxZpUvg/\nrheVcCTGbNmQJ+4cZXx87huqwqKgkmtOyeWsRc7zYInYgraDrtCuDBCfP//ZzOh0\nLxepYLTPk5eNn/GT+VVrqsy35Ccr60g7Lp/bzb1WxyhcLbo0KX7/6jl0lP+VKtdv\nmto+4mbSBXSM1Y5BVVoVgJ3T/wKBgQDsiSvPRzVi5TTj13x67PFymTMx3HCe2WzH\nJUyepCmVhTm482zW95pv6raDr5CTO6OYpHtc5sTTRhVYEZoEYFTM9Vw8faBtluWG\nBjkRh4cIpoIARMn74YZKj0C/0vdX7SHdyBOU3bgRPHg08Hwu3xReqT1kEPSI/B2V\n4pe5fVrucwKBgQCNFgUxUA3dJjyMES18MDDYUZaRug4tfiYouRdmLGIxUxozv6CG\nZnbZzwxFt+GpvPUV4f+P33rgoCvFU+yoPctyjE6j+0aW0DFucPmb2kBwCu5J/856\nkFwCx3blbwFHAco+SdN7g2kcwgmV2MTg/lMOcU7XwUUcN0Obe7UlWbckzQKBgQDQ\nnXaXHL24GGFaZe4y2JFmujmNy1dEsoye44W9ERpf9h1fwsoGmmCKPp90az5+rIXw\nFXl8CUgk8lXW08db/r4r+ma8Lyx0GzcZyplAnaB5/6j+pazjSxfO4KOBy4Y89Tb+\nTP0AOcCi6ws13bgY+sUTa/5qKA4UVw+c5zlb7nRpgwKBgGXAXhenFw1666482iiN\ncHSgwc4ZHa1oL6aNJR1XWH+aboBSwR+feKHUPeT4jHgzRGo/aCNHD2FE5I8eBv33\nof1kWYjAO0YdzeKrW0rTwfvt9gGg+CS397aWu4cy+mTI+MNfBgeDAIVBeJOJXLlX\nhL8bFAuNNVrCOp79TNnNIsh7\n-----END PRIVATE KEY-----\n"; +extern const std::string TestRSAPublicKeyContent = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu+fyUt6zHCycbxYKTsxe\nftoB/mIoN0RNjE5fbsAtAWSgL+n6GK+8csEMliCvltXAYc/EeC3xZmaNozNGgr3U\nZjQXVtBA0k5xy8QrBYaEFi1avmyOX+Y85HKIoqFLWhKQAz6sIFVC1bTf55zyYmev\n3VN9At45kevi1IBJ7VLVrd3qt8UCJ+ZRt8wtZJQWPx8bXx+R7vMQb8EUEyZ4d1KT\ny4/F8Epq4g4Ha4Ue6OrplslbhqzCpSx5nor5X842LX8ztTDiDBvLdv88RkfsW+Fc\nJLO97B9Sq7h/9PyTF1Pe56cKXvlJvrl4aZXT8oBhKxImu2m8mQdoDKV6q1mCjKNN\njQIDAQAB\n-----END PUBLIC KEY-----\n"; +extern const std::string TestECPrivateKeyContent = "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIB6fv25gf7P/7fkjW/2kcKICUhHeOygkFeUJ/ylyU3hloAoGCCqGSM49\nAwEHoUQDQgAEvkKy92hpLiT0GEpzFkYBEWWnkAGTTA6141H0oInA9X30eS0RObAa\nmVY8yD39NI7Nj03hBxEa4Z0tOhrq9cW8eg==\n-----END EC PRIVATE KEY-----\n"; +extern const std::string TestECPublicKeyContent = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvkKy92hpLiT0GEpzFkYBEWWnkAGT\nTA6141H0oInA9X30eS0RObAamVY8yD39NI7Nj03hBxEa4Z0tOhrq9cW8eg==\n-----END PUBLIC KEY-----\n"; +extern const std::string TestHMACSecretKeyBase64Content = "VGhlIHdvcmxkIGhhcyBjaGFuZ2VkLgpJIHNlZSBpdCBpbiB0aGUgd2F0ZXIuCkkgZmVlbCBpdCBpbiB0aGUgRWFydGguCkkgc21lbGwgaXQgaW4gdGhlIGFpci4KTXVjaCB0aGF0IG9uY2Ugd2FzIGlzIGxvc3QsCkZvciBub25lIG5vdyBsaXZlIHdobyByZW1lbWJlciBpdC4K"; + +Y_UNIT_TEST_SUITE(JwtTokenSourceTest) { + Y_UNIT_TEST(Encodes) { + auto source = CreateJwtTokenSource( + TJwtTokenSourceParams() + .KeyId("test_key_id") + .SigningAlgorithm("", TestRSAPrivateKeyContent) + .Issuer("test_issuer") + .Subject("test_subject") + .Audience("test_audience") + ); + + TJwtCheck check; + check + .KeyId("test_key_id") + .Issuer("test_issuer") + .Audience("test_audience") + .Subject("test_subject"); + + auto t = source->GetToken(); + UNIT_ASSERT_VALUES_EQUAL(t.TokenType, "urn:ietf:params:oauth:token-type:jwt"); + check.Check(t.Token); + } + + Y_UNIT_TEST(BadParams) { + UNIT_ASSERT_EXCEPTION_CONTAINS(CreateJwtTokenSource( + TJwtTokenSourceParams() + .KeyId("test_key_id") + .Issuer("test_issuer") + .Subject("test_subject") + .Audience("test_audience") + ), std::invalid_argument, "no signing algorithm"); + + UNIT_ASSERT_EXCEPTION_CONTAINS(CreateJwtTokenSource( + TJwtTokenSourceParams() + .KeyId("test_key_id") + .SigningAlgorithm("", TestRSAPrivateKeyContent) + .TokenTtl(TDuration::Zero()) + ), std::invalid_argument, "token TTL must be positive"); + + UNIT_ASSERT_EXCEPTION_CONTAINS(CreateJwtTokenSource( + TJwtTokenSourceParams() + .SigningAlgorithm("", TestRSAPrivateKeyContent) + .AppendAudience("aud") + .AppendAudience("aud2") + .AppendAudience("") + ), std::invalid_argument, "empty audience"); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/ya.make b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/ya.make new file mode 100644 index 000000000000..5cafeeccbd71 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/ya.make @@ -0,0 +1,16 @@ +UNITTEST() + +SRCS( + credentials_ut.cpp + jwt_token_source_ut.cpp +) + +PEERDIR( + contrib/libs/jwt-cpp + library/cpp/http/server + library/cpp/json + library/cpp/string_utils/base64 + ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/params/params_ut.cpp similarity index 96% rename from ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp rename to ydb/public/sdk/cpp/tests/unit/client/params/params_ut.cpp index 1b516de577f1..1919774ff0c7 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/params/params_ut.cpp @@ -1,5 +1,6 @@ +#include + #include -#include #include #include @@ -85,7 +86,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { .EndOptional() .Build(); - TMap paramsMap; + std::map paramsMap; paramsMap.emplace("$param1", param1Type); paramsMap.emplace("$param2", param2Type); @@ -129,7 +130,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { .EndOptional() .Build(); - TMap paramsMap; + std::map paramsMap; paramsMap.emplace("$param1", param1Type); paramsMap.emplace("$param2", param2Type); @@ -187,7 +188,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { .EndOptional() .Build(); - TMap paramsMap; + std::map paramsMap; paramsMap.emplace("$param1", param1Type); paramsMap.emplace("$param2", param2Type); @@ -217,7 +218,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { .EndOptional() .Build(); - TMap paramsMap; + std::map paramsMap; paramsMap.emplace("$param1", param1Type); paramsMap.emplace("$param2", param2Type); diff --git a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make b/ydb/public/sdk/cpp/tests/unit/client/params/ya.make similarity index 62% rename from ydb/public/sdk/cpp/client/ydb_params/ut/ya.make rename to ydb/public/sdk/cpp/tests/unit/client/params/ya.make index 32e3e9ca56ba..6a3860a481f0 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make +++ b/ydb/public/sdk/cpp/tests/unit/client/params/ya.make @@ -1,4 +1,6 @@ -UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_params) +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) IF (SANITIZER_TYPE == "thread") SIZE(LARGE) @@ -10,6 +12,7 @@ ENDIF() FORK_SUBTESTS() PEERDIR( + ydb/public/sdk/cpp/src/client/params ydb/public/lib/yson_value ) diff --git a/ydb/public/sdk/cpp/tests/unit/client/result/result_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/result/result_ut.cpp new file mode 100644 index 000000000000..5fef90a48329 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/result/result_ut.cpp @@ -0,0 +1,237 @@ +#include +#include +#include + +#include + +#include +#include + +#include + +using namespace NYdb; + +Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { + Y_UNIT_TEST(ListResultSet) { + const std::string resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " list_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 42\n" + " }\n" + " items {\n" + " int32_value: 43\n" + " }\n" + " items {\n" + " int32_value: 44\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 45\n" + " }\n" + " items {\n" + " int32_value: 46\n" + " }\n" + " items {\n" + " int32_value: 47\n" + " }\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + int expectedVal = 42; + auto& column0 = rsParser.ColumnParser(0); + while (rsParser.TryNextRow()) { + column0.OpenList(); + while (column0.TryNextListItem()) { + UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); + } + } + } + + Y_UNIT_TEST(OptionalDictResultSet) { + const std::string resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " dict_type {\n" + " key {\n" + " type_id: INT32\n" + " }\n" + " payload {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " pairs {\n" + " key {\n" + " int32_value: 42\n" + " }\n" + " payload {\n" + " bytes_value: \"abc\"\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " pairs {\n" + " key {\n" + " int32_value: 43\n" + " }\n" + " payload {\n" + " bytes_value: \"zxc\"\n" + " }\n" + " }\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + auto& column0 = rsParser.ColumnParser(0); + + UNIT_ASSERT(rsParser.TryNextRow()); + column0.OpenOptional(); + UNIT_ASSERT(!column0.IsNull()); + column0.OpenDict(); + UNIT_ASSERT(column0.TryNextDictItem()); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 42); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "abc"); + + UNIT_ASSERT(rsParser.TryNextRow()); + column0.OpenOptional(); + UNIT_ASSERT(!column0.IsNull()); + column0.OpenDict(); + UNIT_ASSERT(column0.TryNextDictItem()); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + column0.DictKey(); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + } + + Y_UNIT_TEST(Utf8OptionalResultSet) { + const std::string resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " type_id: UTF8\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " text_value: \"йцукен\"\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " null_flag_value: NULL_VALUE\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + auto& column0 = rsParser.ColumnParser(0); + + int row = 0; + while (rsParser.TryNextRow()) { + column0.OpenOptional(); + + if (row == 0) { + UNIT_ASSERT(!column0.IsNull()); + UNIT_ASSERT_EQUAL(column0.GetUtf8(), "йцукен"); + } else { + UNIT_ASSERT(column0.IsNull()); + } + + row++; + } + } + + Y_UNIT_TEST(ListCorruptedResultSet) { + const std::string resultSetString = + "columns {\n" + " name: \"colName1\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " type_id: UTF8\n" + " }\n" + " }\n" + " }\n" + "}\n" + "columns {\n" + " name: \"colName2\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " type_id: UTF8\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " bytes_value: \"test content\"\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 2); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 1); + + UNIT_ASSERT_EXCEPTION_CONTAINS(rsParser.TryNextRow(), TContractViolation, "Corrupted data: row 0 contains 1 column(s), but metadata contains 2 column(s)"); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/result/ya.make b/ydb/public/sdk/cpp/tests/unit/client/result/ya.make new file mode 100644 index 000000000000..845cd3c445ee --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/result/ya.make @@ -0,0 +1,23 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +IF (SANITIZER_TYPE == "thread") + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +SRCS( + result_ut.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/params +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/value/value_ut.cpp similarity index 96% rename from ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp rename to ydb/public/sdk/cpp/tests/unit/client/value/value_ut.cpp index c7dd61b9fe68..da604232fad9 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/value/value_ut.cpp @@ -1,13 +1,14 @@ +#include +#include + #include -#include -#include +#include #include #include #include #include -#include #include namespace NYdb { @@ -61,7 +62,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); UNIT_ASSERT_NO_DIFF(FormatType(protoType), R"(Struct<'Member1':Uint32,'Member2':List,'Member3':Tuple>)"); @@ -87,7 +88,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); UNIT_ASSERT_NO_DIFF(FormatType(protoType), R"(Dict>)"); @@ -104,7 +105,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); UNIT_ASSERT_NO_DIFF(FormatType(protoType), R"(Tagged)"); @@ -287,10 +288,10 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); Ydb::Value protoValue; - NProtoBuf::TextFormat::ParseFromString(protoValueStr, &protoValue); + google::protobuf::TextFormat::ParseFromString(protoValueStr, &protoValue); TValue value(TType(protoType), protoValue); @@ -341,10 +342,10 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); Ydb::Value protoValue; - NProtoBuf::TextFormat::ParseFromString(protoValueStr, &protoValue); + google::protobuf::TextFormat::ParseFromString(protoValueStr, &protoValue); TValue value(TType(protoType), protoValue); @@ -428,10 +429,10 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); Ydb::Value protoValue; - NProtoBuf::TextFormat::ParseFromString(protoValueStr, &protoValue); + google::protobuf::TextFormat::ParseFromString(protoValueStr, &protoValue); TValue value(TType(protoType), protoValue); @@ -507,25 +508,25 @@ Y_UNIT_TEST_SUITE(YdbValue) { )"; Ydb::Type protoType; - NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); + google::protobuf::TextFormat::ParseFromString(protoTypeStr, &protoType); Ydb::Value protoValue; - NProtoBuf::TextFormat::ParseFromString(protoValueStr, &protoValue); + google::protobuf::TextFormat::ParseFromString(protoValueStr, &protoValue); TValue value(TType(protoType), protoValue); TValueParser parser(value); parser.OpenTuple(); UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalUtf8(), "SomeUtf"); + UNIT_ASSERT(parser.GetOptionalUtf8() == "SomeUtf"); UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalInt8(), -5); + UNIT_ASSERT(parser.GetOptionalInt8() == -5); UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalDouble(), TMaybe()); + UNIT_ASSERT(parser.GetOptionalDouble() == std::optional()); UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalUint64(), (ui64)7); + UNIT_ASSERT(parser.GetOptionalUint64() == (ui64)7); UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalDyNumber(), "12.345"); + UNIT_ASSERT(parser.GetOptionalDyNumber() == "12.345"); parser.CloseTuple(); } @@ -781,8 +782,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { " null_flag_value: NULL_VALUE\n" "}\n"; - TString protoValueStr; - NProtoBuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); + TStringType protoValueStr; + google::protobuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); UNIT_ASSERT_NO_DIFF(protoValueStr, expectedProtoValueStr); } @@ -843,8 +844,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { " }\n" "}\n"; - TString protoValueStr; - NProtoBuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); + TStringType protoValueStr; + google::protobuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); UNIT_ASSERT_NO_DIFF(protoValueStr, expectedProtoValueStr); } @@ -1359,7 +1360,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { } Y_UNIT_TEST(CorrectUuid) { - TString uuidStr = "5ca32c22-841b-11e8-adc0-fa7ae01bbebc"; + std::string uuidStr = "5ca32c22-841b-11e8-adc0-fa7ae01bbebc"; TUuidValue uuid(uuidStr); UNIT_ASSERT_VALUES_EQUAL(uuidStr, uuid.ToString()); } diff --git a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make b/ydb/public/sdk/cpp/tests/unit/client/value/ya.make similarity index 59% rename from ydb/public/sdk/cpp/client/ydb_value/ut/ya.make rename to ydb/public/sdk/cpp/tests/unit/client/value/ya.make index 45d6c80561ed..2aba55b2dc79 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make +++ b/ydb/public/sdk/cpp/tests/unit/client/value/ya.make @@ -1,4 +1,6 @@ -UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_value) +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) IF (SANITIZER_TYPE == "thread") SIZE(LARGE) @@ -12,7 +14,8 @@ FORK_SUBTESTS() PEERDIR( ydb/public/lib/json_value ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_params + ydb/public/sdk/cpp/src/client/value + ydb/public/sdk/cpp/src/client/params ) SRCS( diff --git a/ydb/public/sdk/cpp/tests/unit/client/ya.make b/ydb/public/sdk/cpp/tests/unit/client/ya.make new file mode 100644 index 000000000000..01b80c22bee5 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/ya.make @@ -0,0 +1,11 @@ +RECURSE( + coordination + discovery_mutator + draft + driver + endpoints + oauth2_token_exchange + params + result + value +) diff --git a/ydb/public/sdk/cpp/tests/unit/library/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/library/CMakeLists.txt new file mode 100644 index 000000000000..bb3ffee93000 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(decimal) +add_subdirectory(grpc_client) +add_subdirectory(issue) +add_subdirectory(operation_id) diff --git a/ydb/public/sdk/cpp/tests/unit/library/decimal/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/library/decimal/CMakeLists.txt new file mode 100644 index 000000000000..6675cc27ab5f --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/decimal/CMakeLists.txt @@ -0,0 +1,11 @@ +add_ydb_test(NAME yql-public-decimal_ut + SOURCES + yql_decimal_ut.cpp + yql_wide_int_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + yql-public-decimal + LABELS + unit +) diff --git a/ydb/public/sdk/cpp/tests/unit/library/decimal/ya.make b/ydb/public/sdk/cpp/tests/unit/library/decimal/ya.make new file mode 100644 index 000000000000..e93b6a7e1dea --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/decimal/ya.make @@ -0,0 +1,14 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +SRCS( + yql_decimal_ut.cpp + yql_wide_int_ut.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/decimal +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_decimal_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_decimal_ut.cpp new file mode 100644 index 000000000000..8052da4c2934 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_decimal_ut.cpp @@ -0,0 +1,195 @@ +#include + +#include + +namespace NYdb::NDecimal { + +Y_UNIT_TEST_SUITE(TYqlDecimalTest) { + void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const std::string& expected) { + std::string result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); + TInt128 parsed = FromString(result, precision, scale); + UNIT_ASSERT(parsed == v); + } + + void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { + std::string result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, ""); + } + + Y_UNIT_TEST(TestZeroFormat) { + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 0), "0"); + } + + Y_UNIT_TEST(TestZeroScale) { + SimplePositiveTest(1, 5, 0, "1"); + SimplePositiveTest(10, 5, 0, "10"); + SimplePositiveTest(100, 5, 0, "100"); + SimplePositiveTest(1000, 5, 0, "1000"); + SimplePositiveTest(10000, 5, 0, "10000"); + SimpleNegativeFormatTest(100000, 5, 0); + SimpleNegativeFormatTest(1000000, 5, 0); + + // negative numbers + SimplePositiveTest(-1, 5, 0, "-1"); + SimplePositiveTest(-10, 5, 0, "-10"); + SimplePositiveTest(-100, 5, 0, "-100"); + SimplePositiveTest(-1000, 5, 0, "-1000"); + SimplePositiveTest(-10000, 5, 0, "-10000"); + SimpleNegativeFormatTest(-100000, 5, 0); + } + + Y_UNIT_TEST(TestFormats) { + // we have no trailing zeros + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(10, 15, 6, "0.00001"); + SimplePositiveTest(100, 15, 6, "0.0001"); + SimplePositiveTest(1000, 15, 6, "0.001"); + SimplePositiveTest(10000, 15, 6, "0.01"); + SimplePositiveTest(100000, 15, 6, "0.1"); + SimplePositiveTest(1000000, 15, 6, "1"); + SimplePositiveTest(10000000, 15, 6, "10"); + SimplePositiveTest(100000000, 15, 6, "100"); + + SimplePositiveTest(2020000, 15, 6, "2.02"); + SimplePositiveTest(3003000, 15, 6, "3.003"); + + // negative numbers + SimplePositiveTest(-1, 15, 6, "-0.000001"); + SimplePositiveTest(-10, 15, 6, "-0.00001"); + SimplePositiveTest(-100, 15, 6, "-0.0001"); + SimplePositiveTest(-1000, 15, 6, "-0.001"); + SimplePositiveTest(-10000, 15, 6, "-0.01"); + SimplePositiveTest(-100000, 15, 6, "-0.1"); + SimplePositiveTest(-1000000, 15, 6, "-1"); + SimplePositiveTest(-10000000, 15, 6, "-10"); + SimplePositiveTest(-100000000, 15, 6, "-100"); + + SimplePositiveTest(-2020000, 15, 6, "-2.02"); + SimplePositiveTest(-3003000, 15, 6, "-3.003"); + + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(12, 15, 6, "0.000012"); + SimplePositiveTest(123, 15, 6, "0.000123"); + SimplePositiveTest(1234, 15, 6, "0.001234"); + SimplePositiveTest(12345, 15, 6, "0.012345"); + SimplePositiveTest(123456, 15, 6, "0.123456"); + SimplePositiveTest(1234567, 15, 6, "1.234567"); + SimplePositiveTest(12345678, 15, 6, "12.345678"); + SimplePositiveTest(123456789, 15, 6, "123.456789"); + SimplePositiveTest(1234567898, 15, 6, "1234.567898"); + SimplePositiveTest(12345678987ll, 15, 6, "12345.678987"); + SimplePositiveTest(123456789876ll, 15, 6, "123456.789876"); + } + + Y_UNIT_TEST(TestHugeNumberFormat) { + TInt128 max120 = Inf() - 1; + const char max120String[] = "99999999999999999999999999999999999"; // 35 digits + static_assert(sizeof(max120String) == 36, "sizeof(max120String) == 36"); + SimplePositiveTest(max120, MaxPrecision, 0, max120String); + SimplePositiveTest(max120 + 1, MaxPrecision, 0, "inf"); + + TInt128 min120 = -Inf() + 1; + const char min120String[] = "-99999999999999999999999999999999999"; + static_assert(sizeof(min120String) == 37, "sizeof(min120String) == 37"); + SimplePositiveTest(min120, MaxPrecision, 0, min120String); + SimplePositiveTest(min120 - 1, MaxPrecision, 0, "-inf"); + + // take spot for sign and zero before dot + const char min120StringAfterDot[] = "-0.99999999999999999999999999999999999"; // 35 by nine + leading zero + static_assert(sizeof(min120StringAfterDot) == 39, "sizeof(min120StringAfterDot) == 39"); + SimplePositiveTest(min120, MaxPrecision, MaxPrecision, min120StringAfterDot); + + SimpleNegativeFormatTest(1, MaxPrecision + 1, MaxPrecision + 1); + SimpleNegativeFormatTest(1, MaxPrecision + 1, 0); + SimpleNegativeFormatTest(1, 2, 3); + } + + Y_UNIT_TEST(TestFormStringRoundToEven) { + UNIT_ASSERT(FromString(".51", 1, 0) == 1); + UNIT_ASSERT(FromString("-0.51", 1, 0) == -1); + + UNIT_ASSERT(FromString("+00000008.5", 1, 0) == 8); + UNIT_ASSERT(FromString("-8.5000000000000000000000000000000", 1, 0) == -8); + + UNIT_ASSERT(FromString("00008.51", 1, 0) == 9); + UNIT_ASSERT(FromString("-8.5000000000000000000000000000001", 1, 0) == -9); + + UNIT_ASSERT(FromString("09.499999999999999999999999999999999999999999999999999999999", 1, 0) == 9); + UNIT_ASSERT(FromString("-9.499999999999999999999999999999999999999999999999999999999", 1, 0) == -9); + + UNIT_ASSERT(FromString("9.50", 2, 0) == 10); + UNIT_ASSERT(FromString("-9.5", 2, 0) == -10); + + UNIT_ASSERT(FromString("+0.9949", 2, 2) == 99); + UNIT_ASSERT(FromString("-0.9949", 2, 2) == -99); + } + + Y_UNIT_TEST(TestInfinityValues) { + UNIT_ASSERT(FromString("+1", 1, 1) == Inf()); + UNIT_ASSERT(FromString("-1", 1, 1) == -Inf()); + + UNIT_ASSERT(FromString("10.000", 1, 0) == Inf()); + UNIT_ASSERT(FromString("-10.000", 1, 0) == -Inf()); + + UNIT_ASSERT(FromString("9.500", 1, 0) == Inf()); + UNIT_ASSERT(FromString("-9.500", 1, 0) == -Inf()); + + UNIT_ASSERT(FromString("+0.950", 1, 1) == Inf()); + UNIT_ASSERT(FromString("-0.950", 1, 1) == -Inf()); + + UNIT_ASSERT(FromString("+0.9950", 2, 2) == Inf()); + UNIT_ASSERT(FromString("-0.9950", 2, 2) == -Inf()); + + UNIT_ASSERT(FromString("9999999999999999999999999999999999999.5", 35, 0) == Inf()); + UNIT_ASSERT(FromString("-9999999999999999999999999999999999999.5", 35, 0) == -Inf()); + } + + Y_UNIT_TEST(TestInvalidValues) { + UNIT_ASSERT(IsValid("+999999999999999991234567890.039493804903849038490312345678909999999999999999990")); + + UNIT_ASSERT(!IsValid("")); // empty + UNIT_ASSERT(!IsValid("12.2.3")); // double dot + UNIT_ASSERT(!IsValid("+-12")); // extra sign + UNIT_ASSERT(!IsValid("463786378O74674")); // letter inside + + UNIT_ASSERT(IsError(FromString("", 35, 15))); // empty + UNIT_ASSERT(IsError(FromString("12.2.3", 35, 15))); // double dot + UNIT_ASSERT(IsError(FromString("+-12", 35, 15))); // extra sign + UNIT_ASSERT(IsError(FromString("463786378O74674", 35, 15))); // letter inside + UNIT_ASSERT(IsError(FromString("+7.039493804E1", 35, 5))); // letter in tail after scale + } + + Y_UNIT_TEST(TestSpecialAsString) { + UNIT_ASSERT(IsValid("Nan")); + UNIT_ASSERT(IsValid("INF")); + UNIT_ASSERT(IsValid("-inf")); + + UNIT_ASSERT_VALUES_EQUAL(ToString(Nan(), 10, 2), "nan"); + + UNIT_ASSERT_VALUES_EQUAL(ToString(+Inf(), 10, 2), "inf"); + UNIT_ASSERT_VALUES_EQUAL(ToString(-Inf(), 10, 2), "-inf"); + + UNIT_ASSERT(IsNan(FromString("nan", 10, 2))); + UNIT_ASSERT(IsInf(FromString("+INf", MaxPrecision, 6))); + UNIT_ASSERT(IsInf(FromString("-inF", 4, 2))); + } + + Y_UNIT_TEST(TestToStringOfNonNormal) { + // above Inf + for (TInt128 i = Inf() + 2, end = Inf() + 100; i < end; i++) { + UNIT_ASSERT(!IsNormal(i)); + UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); + } + + // below -Inf + for (TInt128 i = -Inf() - 2, end = -Inf() - 100; i < end; i--) { + UNIT_ASSERT(!IsNormal(i)); + UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); + } + } +} + +} diff --git a/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_wide_int_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_wide_int_ut.cpp new file mode 100644 index 000000000000..d19427f84897 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/decimal/yql_wide_int_ut.cpp @@ -0,0 +1,466 @@ +#include +#include + +namespace NYdb::NDecimal { + +Y_UNIT_TEST_SUITE(TYqlWideIntTest) { + template + void TestUnary(const T aa) { + using Test = TWide::Type>; + const Test at(aa); + static_assert(sizeof(at) == sizeof(aa), "Bad wide int size!"); + + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast(aa), static_cast(at)); +#ifndef _win_ + UNIT_ASSERT(static_cast(aa) == static_cast(at)); + UNIT_ASSERT(static_cast(aa) == static_cast(at)); +#endif + + { + const auto exp = ~aa; + const auto tst = ~at; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = +aa; + const auto tst = +at; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = -aa; + const auto tst = -at; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + auto exp = aa; + auto tst = at; + ++exp; + ++tst; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + auto exp = aa; + auto tst = at; + --exp; + --tst; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + auto exp = aa; + auto tst = at; + exp++; + tst++; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + auto exp = aa; + auto tst = at; + exp--; + tst--; + UNIT_ASSERT(T(tst) == T(exp)); + } + } + + template + void TestBinary(const T ll, const T rr) { + using Test = TWide::Type>; + const Test lt(ll), rt(rr); + + { + const auto exp = ll & rr; + const auto tst = lt & rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll | rr; + const auto tst = lt | rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll ^ rr; + const auto tst = lt ^ rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll + rr; + const auto tst = lt + rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll - rr; + const auto tst = lt - rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + if (rr > 0 && rr < T(sizeof(T) << 3U)) + { + const auto exp = ll >> rr; + const auto tst = lt >> rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + if (rr > 0 && rr < T(sizeof(T) << 3U)) + { + const auto exp = ll << rr; + const auto tst = lt << rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll * rr; + const auto tst = lt * rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + if (rr) + { + const auto exp = ll / rr; + const auto tst = lt / rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + if (rr) + { + const auto exp = ll % rr; + const auto tst = lt % rt; + UNIT_ASSERT(T(tst) == T(exp)); + } + + { + const auto exp = ll == rr; + const auto tst = lt == rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + + { + const auto exp = ll != rr; + const auto tst = lt != rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + + { + const auto exp = ll > rr; + const auto tst = lt > rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + + { + const auto exp = ll < rr; + const auto tst = lt < rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + + { + const auto exp = ll >= rr; + const auto tst = lt >= rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + + { + const auto exp = ll <= rr; + const auto tst = lt <= rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } + } + + template + void TestsForUnsignedType() { + static_assert(std::is_unsigned::value, "Tests for unsigned type."); + TestUnary(2U); + TestUnary(4U); + TestUnary(17U); + TestUnary(42U); + TestUnary(127U); + TestUnary(128U); + TestUnary(129U); + TestUnary(200U); + TestUnary(255U); + TestUnary(256U); + TestUnary(257U); + + TestUnary(std::numeric_limits::min()); + TestUnary(std::numeric_limits::max()); + TestUnary(std::numeric_limits::max() - 1U); + TestUnary(std::numeric_limits::max() >> 1U); + TestUnary(std::numeric_limits::max() >> 3U); + TestUnary(std::numeric_limits::max() >> 7U); + + + TestUnary(std::numeric_limits::min() + 1U); + TestUnary(std::numeric_limits::max() - 1U); + + TestUnary(std::numeric_limits::min() + 3U); + TestUnary(std::numeric_limits::max() - 3U); + + TestUnary(std::numeric_limits::min() + 7U); + TestUnary(std::numeric_limits::max() - 7U); + + + TestBinary(1U, 1U); + TestBinary(7U, 31U); + TestBinary(30000U, 13U); + TestBinary(127U, 13U); + TestBinary(128U, 19U); + TestBinary(129U, 17U); + + + TestBinary(std::numeric_limits::min(), 7U); + TestBinary(std::numeric_limits::max(), 7U); + + TestBinary(std::numeric_limits::min(), 8U); + TestBinary(std::numeric_limits::max(), 8U); + + TestBinary(std::numeric_limits::min(), 9U); + TestBinary(std::numeric_limits::max(), 9U); + + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min() + 1U); + + + TestBinary(std::numeric_limits::max(), std::numeric_limits::max()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min()); + TestBinary(std::numeric_limits::min(), std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min()); + + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() + 1U); + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min() + 1U); + + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min()); + TestBinary(std::numeric_limits::min() + 1U, std::numeric_limits::min()); + + TestBinary(std::numeric_limits::max(), 1U); + TestBinary(std::numeric_limits::min(), 1U); + TestBinary(std::numeric_limits::max() - 1U, 1U); + TestBinary(std::numeric_limits::min() + 1U, 1U); + + TestBinary(std::numeric_limits::max(), 7U); + TestBinary(std::numeric_limits::min(), 7U); + TestBinary(std::numeric_limits::max() - 1U, 7U); + TestBinary(std::numeric_limits::min() + 1U, 7U); + + TestBinary(std::numeric_limits::max() >> 1U, std::numeric_limits::min() >> 1U); + TestBinary(std::numeric_limits::max() >> 1U, std::numeric_limits::min() + 1U); + TestBinary(std::numeric_limits::max() - 1U, std::numeric_limits::min() >> 1U); + TestBinary(std::numeric_limits::max() >> 1U, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() >> 1U); + + TestBinary(std::numeric_limits::max() >> 3U, std::numeric_limits::min() >> 3U); + TestBinary(std::numeric_limits::max() >> 3U, std::numeric_limits::min() + 3U); + TestBinary(std::numeric_limits::max() - 3U, std::numeric_limits::min() >> 3U); + TestBinary(std::numeric_limits::max() >> 3U, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() >> 3U); + } + + template + void TestsForSignedType() { + static_assert(std::is_signed::value, "Tests for signed type."); + TestUnary(0); + + TestUnary(1); + TestUnary(-1); + + TestUnary(2); + TestUnary(-2); + + TestUnary(3); + TestUnary(-3); + + TestUnary(4); + TestUnary(-4); + + TestUnary(17); + TestUnary(-17); + + TestUnary(42); + TestUnary(-42); + + TestUnary(127); + TestUnary(-127); + + TestUnary(128); + TestUnary(-128); + + TestUnary(129); + TestUnary(-129); + + TestUnary(200); + TestUnary(-200); + + TestUnary(255); + TestUnary(-255); + + TestUnary(256); + TestUnary(-256); + + TestUnary(257); + TestUnary(-257); + + TestUnary(258); + TestUnary(-258); + + TestUnary(std::numeric_limits::min()); + TestUnary(std::numeric_limits::max()); + + TestUnary(std::numeric_limits::min() + 1); + TestUnary(std::numeric_limits::max() - 1); + + TestUnary(std::numeric_limits::min() + 2); + TestUnary(std::numeric_limits::max() - 2); + + TestUnary(std::numeric_limits::min() + 3); + TestUnary(std::numeric_limits::max() - 3); + + TestUnary(std::numeric_limits::min() + 7); + TestUnary(std::numeric_limits::max() - 7); + + TestUnary(std::numeric_limits::min() >> 1); + TestUnary(std::numeric_limits::max() >> 1); + + TestUnary(std::numeric_limits::min() >> 3); + TestUnary(std::numeric_limits::max() >> 3); + + TestUnary(std::numeric_limits::min() >> 7); + TestUnary(std::numeric_limits::max() >> 7); + + + TestUnary(std::numeric_limits::min() + 1); + TestUnary(std::numeric_limits::max() - 1); + + TestUnary(std::numeric_limits::min() + 3); + TestUnary(std::numeric_limits::max() - 3); + + TestUnary(std::numeric_limits::min() + 7); + TestUnary(std::numeric_limits::max() - 7); + + + TestBinary(0, 0); + TestBinary(1, 0); + TestBinary(0, -1); + + TestBinary(1, 1); + TestBinary(1, -1); + TestBinary(-1, 1); + TestBinary(-1, -1); + + TestBinary(7, 42); + TestBinary(-7, 42); + + TestBinary(0, -43); + + TestBinary(-30000, 64); + TestBinary(30000, -64); + TestBinary(30000, 64); + + TestBinary(21, 0); + TestBinary(13, -127); + TestBinary(-19, 128); + TestBinary(-77, -129); + TestBinary(13, 127); + TestBinary(19, 128); + TestBinary(77, 129); + + TestBinary(std::numeric_limits::max(), -1); + + TestBinary(std::numeric_limits::min(), -7); + TestBinary(std::numeric_limits::max(), -7); + + TestBinary(std::numeric_limits::min(), -8); + TestBinary(std::numeric_limits::max(), -8); + + TestBinary(std::numeric_limits::min(), -9); + TestBinary(std::numeric_limits::max(), -9); + + TestBinary(std::numeric_limits::min(), std::numeric_limits::min() >> 5); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() >> 5); + + + TestBinary(std::numeric_limits::min(), 7); + TestBinary(std::numeric_limits::max(), 7); + + TestBinary(std::numeric_limits::min(), 8); + TestBinary(std::numeric_limits::max(), 8); + + TestBinary(std::numeric_limits::min(), 9); + TestBinary(std::numeric_limits::max(), 9); + + TestBinary(std::numeric_limits::max() - 1, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max() - 1, std::numeric_limits::min() + 1); + + + TestBinary(std::numeric_limits::max(), std::numeric_limits::max()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min()); + TestBinary(std::numeric_limits::min(), std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min()); + + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() + 1); + TestBinary(std::numeric_limits::max() - 1, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max() - 1, std::numeric_limits::min() + 1); + + TestBinary(std::numeric_limits::max(), 0); + TestBinary(std::numeric_limits::min(), 0); + TestBinary(std::numeric_limits::max() - 1, 0); + TestBinary(std::numeric_limits::min() + 1, 0); + + TestBinary(std::numeric_limits::max(), 1); + TestBinary(std::numeric_limits::min(), 1); + TestBinary(std::numeric_limits::max() - 1, 1); + TestBinary(std::numeric_limits::min() + 1, 1); + + TestBinary(std::numeric_limits::max(), 7); + TestBinary(std::numeric_limits::min(), 7); + TestBinary(std::numeric_limits::max() - 1, 7); + TestBinary(std::numeric_limits::min() + 1, 7); + + TestBinary(std::numeric_limits::max() >> 1, std::numeric_limits::min() >> 1); + TestBinary(std::numeric_limits::max() >> 1, std::numeric_limits::min() + 1); + TestBinary(std::numeric_limits::max() - 1, std::numeric_limits::min() >> 1); + TestBinary(std::numeric_limits::max() >> 1, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() >> 1); + + TestBinary(std::numeric_limits::max() >> 3, std::numeric_limits::min() >> 3); + TestBinary(std::numeric_limits::max() >> 3, std::numeric_limits::min() + 3); + TestBinary(std::numeric_limits::max() - 3, std::numeric_limits::min() >> 3); + TestBinary(std::numeric_limits::max() >> 3, std::numeric_limits::min()); + TestBinary(std::numeric_limits::max(), std::numeric_limits::min() >> 3); + } + + Y_UNIT_TEST(CheckUnsignedByCompilerIntegrals) { + TestsForUnsignedType(); + TestsForUnsignedType(); +#ifndef _win_ + TestsForUnsignedType(); +#endif + } + +#ifndef _ubsan_enabled_ +#ifndef _msan_enabled_ + Y_UNIT_TEST(CheckSignedByCompilerIntegrals) { + TestsForSignedType(); + TestsForSignedType(); +#ifndef _win_ + TestsForSignedType(); +#endif + } +#endif +#endif +} + +} diff --git a/ydb/public/sdk/cpp/tests/unit/library/grpc_client/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/CMakeLists.txt new file mode 100644 index 000000000000..620eff01f281 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/CMakeLists.txt @@ -0,0 +1,9 @@ +add_ydb_test(NAME grpc-client-low_ut + SOURCES + grpc_client_low_ut.cpp + LINK_LIBRARIES + cpp-testing-unittest_main + grpc-client + LABELS + unit +) diff --git a/ydb/library/grpc/client/ut/grpc_client_low_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/grpc_client_low_ut.cpp similarity index 96% rename from ydb/library/grpc/client/ut/grpc_client_low_ut.cpp rename to ydb/public/sdk/cpp/tests/unit/library/grpc_client/grpc_client_low_ut.cpp index b5d6f6b8c158..35a89328cb7f 100644 --- a/ydb/library/grpc/client/ut/grpc_client_low_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/grpc_client_low_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -58,4 +58,4 @@ Y_UNIT_TEST_SUITE(ChannelPoolTests) { UNIT_ASSERT_C(allDeleted, "expired stubsHolders were not deleted after timeout"); } -} // ChannelPoolTests ut suite \ No newline at end of file +} // ChannelPoolTests ut suite diff --git a/ydb/public/sdk/cpp/tests/unit/library/grpc_client/ya.make b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/ya.make new file mode 100644 index 000000000000..690af039e127 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/grpc_client/ya.make @@ -0,0 +1,15 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +FORK_SUBTESTS() + +SRCS( + grpc_client_low_ut.cpp +) + +PEERDIR( + ydb/public/sdk/cpp/src/library/grpc/client +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/library/issue/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/library/issue/CMakeLists.txt new file mode 100644 index 000000000000..49e833e9cf78 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/issue/CMakeLists.txt @@ -0,0 +1,12 @@ +add_ydb_test(NAME yql-public-issue_ut + SOURCES + utf8_ut.cpp + yql_issue_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + yql-public-issue + unicode-normalization + LABELS + unit +) diff --git a/ydb/public/sdk/cpp/tests/unit/library/issue/utf8_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/issue/utf8_ut.cpp new file mode 100644 index 000000000000..283ca65cffc2 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/issue/utf8_ut.cpp @@ -0,0 +1,18 @@ +#include + +#include + +Y_UNIT_TEST_SUITE(TUtf8Tests) { + Y_UNIT_TEST(Simple) { + UNIT_ASSERT(NYdb::NIssue::IsUtf8("")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("\x01_ASCII_\x7F")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("Привет!")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("\xF0\x9F\x94\xA2")); + + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xf5\x80\x80\x80")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xed\xa6\x80")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xF0\x9F\x94")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xE3\x85\xB6\xE7\x9C\xB0\xE3\x9C\xBA\xE2\xAA\x96\xEE\xA2\x8C\xEC\xAF\xB8\xE1\xB2\xBB\xEC\xA3\x9C\xE3\xAB\x8B\xEC\x95\x92\xE1\x8A\xBF\xE2\x8E\x86\xEC\x9B\x8D\xE2\x8E\xAE\xE3\x8A\xA3\xE0\xAC\xBC\xED\xB6\x85")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xc0\xbe\xd0\xb1\xd0\xbd\xd0\xbe\xd0\xb2\xd0\xbb\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8e")); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/library/issue/ya.make b/ydb/public/sdk/cpp/tests/unit/library/issue/ya.make new file mode 100644 index 000000000000..05488b48b6ef --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/issue/ya.make @@ -0,0 +1,17 @@ +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) + +FORK_SUBTESTS() + +SRCS( + utf8_ut.cpp + yql_issue_ut.cpp +) + +PEERDIR( + library/cpp/unicode/normalization + ydb/public/sdk/cpp/src/library/issue +) + +END() diff --git a/ydb/public/sdk/cpp/tests/unit/library/issue/yql_issue_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/issue/yql_issue_ut.cpp new file mode 100644 index 000000000000..2f90693b4ffb --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/issue/yql_issue_ut.cpp @@ -0,0 +1,101 @@ +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include + +#include + +using namespace google::protobuf; +using namespace NYdb::NIssue; + +Y_UNIT_TEST_SUITE(IssueTest) { + Y_UNIT_TEST(Ascii) { + TIssue issue1("тест abc"); + UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); + TIssue issue2("\xFF abc"); + UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); + TIssue issue3(""); + UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); + TIssue issue4("abc"); + UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); + } +} + +Y_UNIT_TEST_SUITE(ToMessage) { + Y_UNIT_TEST(NonUtf8) { + const std::string nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + TIssue issue; + issue.SetMessage(nonUtf8String); + + Ydb::Issue::IssueMessage msg; + IssueToMessage(issue, &msg); + NYdb::TStringType serialized; + UNIT_ASSERT(msg.SerializeToString(&serialized)); + Ydb::Issue::IssueMessage msg2; + UNIT_ASSERT(msg2.ParseFromString(serialized)); + } +} + +Y_UNIT_TEST_SUITE(ToOneLineStringTest) { + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive("subsuberror")); + issue.AddSubIssue(MakeIntrusive(subissue)); + + TIssues issues; + issues.AddIssue(issue); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: {
: Error: suberror subissue: {
: Error: subsuberror } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); + issue.AddSubIssue(MakeIntrusive("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: {
: Error: suberror } } { abc.file:2:100: Error: my message } ]"); + } +} + +Y_UNIT_TEST_SUITE(EscapeNonUtf8) { + Y_UNIT_TEST(Escape) { + const std::string nonUtf8String = "\xfe\xfa\xf5\xc2"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + + // Check that our escaping correctly processes unicode pairs + const std::string toNormalize = "Ёлка"; + const std::string nfd = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol + const std::string nfc = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter + UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); + std::pair nonUtf8Messages[] = { + { nonUtf8String, "????" }, + { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, + { nfd, nfd }, + { nfc, nfc }, + { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, + { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, + }; + + for (const auto& [src, dst] : nonUtf8Messages) { + TIssue issue(src); + UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); + } + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/library/operation_id/CMakeLists.txt b/ydb/public/sdk/cpp/tests/unit/library/operation_id/CMakeLists.txt new file mode 100644 index 000000000000..86d3fd5131de --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/operation_id/CMakeLists.txt @@ -0,0 +1,11 @@ +add_ydb_test(NAME operation_id_ut + SOURCES + operation_id_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + library-operation_id + cpp-testing-unittest + LABELS + unit +) diff --git a/ydb/public/lib/operation_id/operation_id_ut.cpp b/ydb/public/sdk/cpp/tests/unit/library/operation_id/operation_id_ut.cpp similarity index 62% rename from ydb/public/lib/operation_id/operation_id_ut.cpp rename to ydb/public/sdk/cpp/tests/unit/library/operation_id/operation_id_ut.cpp index 3ece76f73f96..8c661a1b3c35 100644 --- a/ydb/public/lib/operation_id/operation_id_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/library/operation_id/operation_id_ut.cpp @@ -1,4 +1,5 @@ -#include "operation_id.h" +#include +#include #include #include @@ -7,21 +8,21 @@ namespace NKikimr { namespace NOperationId { Y_UNIT_TEST_SUITE(OperationIdTest) { - const TString PreparedQueryId = "9d629c27-2c3036b3-4b180476-64435bca"; + const std::string PreparedQueryId = "9d629c27-2c3036b3-4b180476-64435bca"; Y_UNIT_TEST(ConvertKindOnly) { Ydb::TOperationId proto; - proto.SetKind(Ydb::TOperationId::OPERATION_DDL); + proto.set_kind(Ydb::TOperationId::OPERATION_DDL); auto str = ProtoToString(proto); UNIT_ASSERT_EQUAL(str, "ydb://operation/1"); auto newProto = TOperationId(str); - UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); - UNIT_ASSERT_EQUAL(newProto.DataSize(), 0); + UNIT_ASSERT_EQUAL(newProto.GetProto().kind(), proto.kind()); + UNIT_ASSERT_EQUAL(newProto.GetProto().data_size(), 0); } Y_UNIT_TEST(PreparedQueryIdCompatibleFormatter) { Ydb::TOperationId opId; - opId.SetKind(Ydb::TOperationId::PREPARED_QUERY_ID); + opId.set_kind(Ydb::TOperationId::PREPARED_QUERY_ID); AddOptionalValue(opId, "id", PreparedQueryId); auto result = ProtoToString(opId); UNIT_ASSERT_VALUES_EQUAL(FormatPreparedQueryIdCompat(PreparedQueryId), result); @@ -29,28 +30,28 @@ Y_UNIT_TEST_SUITE(OperationIdTest) { Y_UNIT_TEST(PreparedQueryIdDecode) { const auto queryId = FormatPreparedQueryIdCompat(PreparedQueryId); - TString decodedString; + std::string decodedString; bool decoded = DecodePreparedQueryIdCompat(queryId, decodedString); UNIT_ASSERT(decoded); UNIT_ASSERT_VALUES_EQUAL(PreparedQueryId, decodedString); } Y_UNIT_TEST(PreparedQueryIdDecodeRawString) { - TString decodedString; + std::string decodedString; bool decoded = DecodePreparedQueryIdCompat(PreparedQueryId, decodedString); UNIT_ASSERT(!decoded); UNIT_ASSERT(decodedString.empty()); } Y_UNIT_TEST(PreparedQueryIdDecodeInvalidString) { - TString decodedString; + std::string decodedString; UNIT_ASSERT_EXCEPTION( - DecodePreparedQueryIdCompat(TString("ydb://preparedqueryid/4?id="), decodedString), yexception); + DecodePreparedQueryIdCompat(std::string("ydb://preparedqueryid/4?id="), decodedString), yexception); UNIT_ASSERT(decodedString.empty()); } Y_UNIT_TEST(FormatPrefixShorter) { - UNIT_ASSERT(TString("ydb://preparedqueryid/4?id=").size() < PreparedQueryId.size()); + UNIT_ASSERT(std::string("ydb://preparedqueryid/4?id=").size() < PreparedQueryId.size()); } #if 0 @@ -60,20 +61,20 @@ Y_UNIT_TEST_SUITE(OperationIdTest) { auto result = FormatPreparedQueryIdCompat(PreparedQueryId); x += result.size(); } - Cerr << x << Endl; + std::cerr << x << std::endl; } Y_UNIT_TEST(PreparedQueryIdDecodePerf) { ui64 x = 0; for (int i = 0; i < 10000000; i++) { const auto queryId = FormatPreparedQueryIdCompat(PreparedQueryId); - TString decodedString; + std::string decodedString; bool decoded = DecodePreparedQueryIdCompat(queryId, decodedString); UNIT_ASSERT(decoded); UNIT_ASSERT_VALUES_EQUAL(PreparedQueryId, decodedString); x += decodedString.size(); } - Cerr << x << Endl; + std::cerr << x << std::endl; } Y_UNIT_TEST(PreparedQueryIdOldFormatterPerf) { @@ -85,36 +86,36 @@ Y_UNIT_TEST_SUITE(OperationIdTest) { auto result = ProtoToString(opId); x += result.size(); } - Cerr << x << Endl; + std::cerr << x << std::endl; } #endif Y_UNIT_TEST(ConvertKindAndValues) { Ydb::TOperationId proto; - proto.SetKind(Ydb::TOperationId::OPERATION_DDL); + proto.set_kind(Ydb::TOperationId::OPERATION_DDL); { - auto data = proto.AddData(); - data->SetKey("key1"); - data->SetValue("value1"); + auto data = proto.add_data(); + data->set_key("key1"); + data->set_value("value1"); } { - auto data = proto.AddData(); - data->SetKey("txId"); - data->SetValue("42"); + auto data = proto.add_data(); + data->set_key("txId"); + data->set_value("42"); } auto str = ProtoToString(proto); UNIT_ASSERT_EQUAL(str, "ydb://operation/1?key1=value1&txId=42"); auto newProto = TOperationId(str); - UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); - UNIT_ASSERT_EQUAL(newProto.DataSize(), 2); + UNIT_ASSERT_EQUAL(newProto.GetProto().kind(), proto.kind()); + UNIT_ASSERT_EQUAL(newProto.GetProto().data_size(), 2); { - auto data = newProto.GetData(0); - UNIT_ASSERT_EQUAL(data.GetKey(), "key1"); - UNIT_ASSERT_EQUAL(data.GetValue(), "value1"); + auto data = newProto.GetProto().data(0); + UNIT_ASSERT_EQUAL(data.key(), "key1"); + UNIT_ASSERT_EQUAL(data.value(), "value1"); } { - auto data = newProto.GetData(1); - UNIT_ASSERT_EQUAL(data.GetKey(), "txId"); - UNIT_ASSERT_EQUAL(data.GetValue(), "42"); + auto data = newProto.GetProto().data(1); + UNIT_ASSERT_EQUAL(data.key(), "txId"); + UNIT_ASSERT_EQUAL(data.value(), "42"); } } } diff --git a/ydb/public/lib/operation_id/ut/ya.make b/ydb/public/sdk/cpp/tests/unit/library/operation_id/ya.make similarity index 58% rename from ydb/public/lib/operation_id/ut/ya.make rename to ydb/public/sdk/cpp/tests/unit/library/operation_id/ya.make index a22fd22fba0e..f2b1c5a95b84 100644 --- a/ydb/public/lib/operation_id/ut/ya.make +++ b/ydb/public/sdk/cpp/tests/unit/library/operation_id/ya.make @@ -1,4 +1,6 @@ -UNITTEST_FOR(ydb/public/lib/operation_id) +UNITTEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) FORK_SUBTESTS() @@ -12,6 +14,7 @@ SRCS( PEERDIR( library/cpp/testing/unittest + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/public/sdk/cpp/tests/unit/library/ya.make b/ydb/public/sdk/cpp/tests/unit/library/ya.make new file mode 100644 index 000000000000..7063b9a8485d --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/library/ya.make @@ -0,0 +1,6 @@ +RECURSE( + decimal + grpc_client + issue + operation_id +) diff --git a/ydb/public/sdk/cpp/tests/unit/ya.make b/ydb/public/sdk/cpp/tests/unit/ya.make new file mode 100644 index 000000000000..736afe96ef74 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/ya.make @@ -0,0 +1,4 @@ +RECURSE( + client + library +) diff --git a/ydb/public/sdk/cpp/tests/ya.make b/ydb/public/sdk/cpp/tests/ya.make new file mode 100644 index 000000000000..3bfdc86670df --- /dev/null +++ b/ydb/public/sdk/cpp/tests/ya.make @@ -0,0 +1,4 @@ +RECURSE( + integration + unit +) diff --git a/ydb/public/sdk/cpp/ya.make b/ydb/public/sdk/cpp/ya.make index e5c992d321d0..6c59f7843612 100644 --- a/ydb/public/sdk/cpp/ya.make +++ b/ydb/public/sdk/cpp/ya.make @@ -1,65 +1,10 @@ RECURSE( - client/draft - client/draft/ut - client/extensions - client/helpers - client/impl/ydb_endpoints - client/impl/ydb_endpoints/ut - client/impl/ydb_internal/common - client/impl/ydb_internal/db_driver_state - client/impl/ydb_internal/grpc_connections - client/impl/ydb_internal/logger - client/impl/ydb_internal/make_request - client/impl/ydb_internal/plain_status - client/impl/ydb_internal/thread_pool - client/impl/ydb_internal/value_helpers - client/impl/ydb_stats - client/resources - client/ydb_bsconfig - client/ydb_common_client - client/ydb_common_client/impl - client/ydb_coordination - client/ydb_coordination/ut - client/ydb_datastreams - client/ydb_debug - client/ydb_discovery - client/ydb_driver - client/ydb_driver/ut - client/ydb_export - client/ydb_extension - client/ydb_federated_topic - client/ydb_federated_topic/impl - client/ydb_federated_topic/ut - client/ydb_import - client/ydb_operation - client/ydb_params - client/ydb_params/ut - client/ydb_persqueue_core - client/ydb_persqueue_public - client/ydb_persqueue_public/impl - client/ydb_persqueue_public/ut - client/ydb_persqueue_public/ut/ut_utils - client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut - client/ydb_proto - client/ydb_query - client/ydb_rate_limiter - client/ydb_result - client/ydb_result/ut - client/ydb_scheme - client/ydb_table - client/ydb_table/impl - client/ydb_table/query_stats - client/ydb_topic - client/ydb_topic/codecs - client/ydb_topic/impl - client/ydb_topic/ut - client/ydb_types - client/ydb_types/credentials - client/ydb_types/exceptions - client/ydb_types/fatal_error_handlers - client/ydb_types/operation - client/ydb_types/status - client/ydb_value - client/ydb_value/ut + adapters + client examples + src +) + +RECURSE_FOR_TESTS( + tests ) diff --git a/ydb/services/auth/grpc_service.cpp b/ydb/services/auth/grpc_service.cpp index acf0853a35f3..4a38caedc1e3 100644 --- a/ydb/services/auth/grpc_service.cpp +++ b/ydb/services/auth/grpc_service.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace NKikimr { namespace NGRpcService { diff --git a/ydb/services/auth/ya.make b/ydb/services/auth/ya.make index f01fd127d81d..8970c34c0e0f 100644 --- a/ydb/services/auth/ya.make +++ b/ydb/services/auth/ya.make @@ -11,7 +11,7 @@ PEERDIR( ydb/core/protos ydb/library/login ydb/public/api/grpc - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/backup/grpc_service.cpp b/ydb/services/backup/grpc_service.cpp index 706551a22900..84618e815491 100644 --- a/ydb/services/backup/grpc_service.cpp +++ b/ydb/services/backup/grpc_service.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace NKikimr { namespace NGRpcService { diff --git a/ydb/services/backup/ya.make b/ydb/services/backup/ya.make index f01fd127d81d..8970c34c0e0f 100644 --- a/ydb/services/backup/ya.make +++ b/ydb/services/backup/ya.make @@ -11,7 +11,7 @@ PEERDIR( ydb/core/protos ydb/library/login ydb/public/api/grpc - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/bsconfig/bsconfig_ut.cpp b/ydb/services/bsconfig/bsconfig_ut.cpp index 6eacf8c13f62..eab8effaa01e 100644 --- a/ydb/services/bsconfig/bsconfig_ut.cpp +++ b/ydb/services/bsconfig/bsconfig_ut.cpp @@ -9,9 +9,9 @@ #include -#include +#include -#include +#include #include #include #include diff --git a/ydb/services/cms/cms_ut.cpp b/ydb/services/cms/cms_ut.cpp index 5693219c2790..b78a33bd4812 100644 --- a/ydb/services/cms/cms_ut.cpp +++ b/ydb/services/cms/cms_ut.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include #include @@ -24,10 +24,10 @@ #include // new grpc client -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/ydb/services/cms/ut/ya.make b/ydb/services/cms/ut/ya.make index 6b7552357ded..e4f5efaf7dd4 100644 --- a/ydb/services/cms/ut/ya.make +++ b/ydb/services/cms/ut/ya.make @@ -12,7 +12,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/svnversion ydb/core/testlib/default diff --git a/ydb/services/cms/ya.make b/ydb/services/cms/ya.make index 629dbc6b3b10..fe23bf4a3724 100644 --- a/ydb/services/cms/ya.make +++ b/ydb/services/cms/ya.make @@ -10,8 +10,8 @@ PEERDIR( ydb/core/mind ydb/library/aclib ydb/public/api/grpc - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/resources + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/resources ) END() diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index 95653817cd1e..b39a5518880f 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -2,12 +2,12 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -97,7 +97,9 @@ class TDatastreamsTestServer { KikimrServer = std::make_unique(std::move(appConfig)); ui16 grpc = KikimrServer->GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driverConfig = TDriverConfig().SetEndpoint(location).SetLog(CreateLogBackend("cerr", TLOG_DEBUG)); + auto driverConfig = TDriverConfig() + .SetEndpoint(location) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", TLOG_DEBUG).Release())); if (secure) { driverConfig.UseSecureConnection(TString(NYdbSslTestData::CaCrt)); } else { @@ -1447,7 +1449,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { for (const auto& item : dataReceivedEvent->GetMessages()) { Cerr << item.DebugString(true) << Endl; UNIT_ASSERT_VALUES_EQUAL(item.GetData(), item.GetPartitionKey()); - auto hashKey = item.GetExplicitHash().empty() ? HexBytesToDecimal(MD5::Calc(item.GetPartitionKey())) : BytesToDecimal(item.GetExplicitHash()); + auto hashKey = item.GetExplicitHash().empty() ? HexBytesToDecimal(MD5::Calc(item.GetPartitionKey())) : BytesToDecimal(TString{item.GetExplicitHash()}); UNIT_ASSERT_VALUES_EQUAL(NKikimr::NDataStreams::V1::ShardFromDecimal(hashKey, 5), item.GetPartitionStream()->GetPartitionId()); UNIT_ASSERT(item.GetIp().empty()); if (item.GetData() == dataStr) { diff --git a/ydb/services/datastreams/ut/ya.make b/ydb/services/datastreams/ut/ya.make index e02a6f819576..faf35e8333fc 100644 --- a/ydb/services/datastreams/ut/ya.make +++ b/ydb/services/datastreams/ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/services/datastreams) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() SIZE(MEDIUM) @@ -14,12 +18,12 @@ PEERDIR( ydb/core/persqueue/ut/common ydb/core/testlib/default ydb/core/tx/schemeshard/ut_helpers - ydb/library/grpc/client - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/library/grpc/client + ydb/public/sdk/cpp/src/client/topic ydb/services/datastreams - ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/topic/ut/ut_utils ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/datastreams/ya.make b/ydb/services/datastreams/ya.make index 8fdd95a53997..693235f095e8 100644 --- a/ydb/services/datastreams/ya.make +++ b/ydb/services/datastreams/ya.make @@ -16,9 +16,9 @@ PEERDIR( ydb/core/mind ydb/public/api/grpc ydb/public/api/grpc/draft - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/resources - ydb/public/sdk/cpp/client/ydb_datastreams + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/resources + ydb/public/sdk/cpp/src/client/datastreams ydb/services/lib/actors ydb/services/lib/sharding ydb/services/persqueue_v1 diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h b/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h index 150dd11b52b8..ece96617d623 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp index 9bd43e80399c..0953bcc7a4f6 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace NKikimr { namespace NGRpcProxy { diff --git a/ydb/services/discovery/ya.make b/ydb/services/discovery/ya.make index 97ef59e60b35..3f97b4d86684 100644 --- a/ydb/services/discovery/ya.make +++ b/ydb/services/discovery/ya.make @@ -9,7 +9,7 @@ PEERDIR( ydb/core/grpc_services ydb/core/mind ydb/public/api/grpc - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/dynamic_config/ut/ya.make b/ydb/services/dynamic_config/ut/ya.make index 724386477be0..42c7952fb905 100644 --- a/ydb/services/dynamic_config/ut/ya.make +++ b/ydb/services/dynamic_config/ut/ya.make @@ -12,7 +12,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/svnversion ydb/core/testlib/default diff --git a/ydb/services/dynamic_config/ya.make b/ydb/services/dynamic_config/ya.make index 629dbc6b3b10..fe23bf4a3724 100644 --- a/ydb/services/dynamic_config/ya.make +++ b/ydb/services/dynamic_config/ya.make @@ -10,8 +10,8 @@ PEERDIR( ydb/core/mind ydb/library/aclib ydb/public/api/grpc - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/resources + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/resources ) END() diff --git a/ydb/services/ext_index/ut/ut_ext_index.cpp b/ydb/services/ext_index/ut/ut_ext_index.cpp index 2083968943aa..04adc312ce5f 100644 --- a/ydb/services/ext_index/ut/ut_ext_index.cpp +++ b/ydb/services/ext_index/ut/ut_ext_index.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp index bf0a15b4f1d2..c8210a20899b 100644 --- a/ydb/services/fq/ut_integration/fq_ut.cpp +++ b/ydb/services/fq/ut_integration/fq_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/ydb/services/fq/ut_integration/ut_utils.cpp b/ydb/services/fq/ut_integration/ut_utils.cpp index d7087f8ea177..3c6df45c73e5 100644 --- a/ydb/services/fq/ut_integration/ut_utils.cpp +++ b/ydb/services/fq/ut_integration/ut_utils.cpp @@ -1,6 +1,6 @@ #include "ut_utils.h" -#include +#include #include #include #include diff --git a/ydb/services/fq/ut_integration/ut_utils.h b/ydb/services/fq/ut_integration/ut_utils.h index a627c45929c0..ea123f29484b 100644 --- a/ydb/services/fq/ut_integration/ut_utils.h +++ b/ydb/services/fq/ut_integration/ut_utils.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/ydb/services/fq/ut_integration/ya.make b/ydb/services/fq/ut_integration/ya.make index e03a8081af9c..3b7309a3f355 100644 --- a/ydb/services/fq/ut_integration/ya.make +++ b/ydb/services/fq/ut_integration/ya.make @@ -11,7 +11,7 @@ SRCS( PEERDIR( library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/svnversion ydb/core/fq/libs/control_plane_storage diff --git a/ydb/services/kesus/grpc_service.cpp b/ydb/services/kesus/grpc_service.cpp index fe89825a35ab..62e8b0ca49df 100644 --- a/ydb/services/kesus/grpc_service.cpp +++ b/ydb/services/kesus/grpc_service.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/services/kesus/ya.make b/ydb/services/kesus/ya.make index 40877d70819d..6a32f51a99a2 100644 --- a/ydb/services/kesus/ya.make +++ b/ydb/services/kesus/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/core/kesus/tablet ydb/public/api/grpc ydb/public/api/grpc/draft - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/keyvalue/grpc_service_ut.cpp b/ydb/services/keyvalue/grpc_service_ut.cpp index 8ef9375e5785..3740b4be51d7 100644 --- a/ydb/services/keyvalue/grpc_service_ut.cpp +++ b/ydb/services/keyvalue/grpc_service_ut.cpp @@ -9,9 +9,9 @@ #include -#include +#include -#include +#include #include #include #include diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp index 7a85c53371c2..55b0aea47b34 100644 --- a/ydb/services/lib/actors/pq_schema_actor.cpp +++ b/ydb/services/lib/actors/pq_schema_actor.cpp @@ -1,11 +1,11 @@ #include "pq_schema_actor.h" -#include +#include #include #include #include -#include +#include #include diff --git a/ydb/services/lib/actors/ya.make b/ydb/services/lib/actors/ya.make index dc2ac62bb0c9..5408a9c2455f 100644 --- a/ydb/services/lib/actors/ya.make +++ b/ydb/services/lib/actors/ya.make @@ -12,12 +12,12 @@ PEERDIR( ydb/core/metering ydb/core/mind ydb/core/protos - ydb/library/persqueue/obfuscate + ydb/public/sdk/cpp/src/library/persqueue/obfuscate ydb/library/persqueue/topic_parser ydb/public/api/grpc ydb/public/api/grpc/draft - ydb/public/lib/jwt - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/jwt + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/local_discovery/ya.make b/ydb/services/local_discovery/ya.make index 97ef59e60b35..3f97b4d86684 100644 --- a/ydb/services/local_discovery/ya.make +++ b/ydb/services/local_discovery/ya.make @@ -9,7 +9,7 @@ PEERDIR( ydb/core/grpc_services ydb/core/mind ydb/public/api/grpc - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/metadata/abstract/ya.make b/ydb/services/metadata/abstract/ya.make index 56a5857702df..d70ee6198a0b 100644 --- a/ydb/services/metadata/abstract/ya.make +++ b/ydb/services/metadata/abstract/ya.make @@ -19,6 +19,7 @@ PEERDIR( ydb/library/actors/core yql/essentials/core/expr_nodes ydb/public/api/protos + ydb/public/sdk/cpp/src/client/resources ) END() diff --git a/ydb/services/metadata/initializer/ut/ut_init.cpp b/ydb/services/metadata/initializer/ut/ut_init.cpp index 5207e20e9910..450ad891b4e0 100644 --- a/ydb/services/metadata/initializer/ut/ut_init.cpp +++ b/ydb/services/metadata/initializer/ut/ut_init.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/services/metadata/secret/ut/ut_secret.cpp b/ydb/services/metadata/secret/ut/ut_secret.cpp index 039cd2c3a8a6..5f59b8820445 100644 --- a/ydb/services/metadata/secret/ut/ut_secret.cpp +++ b/ydb/services/metadata/secret/ut/ut_secret.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/services/monitoring/ya.make b/ydb/services/monitoring/ya.make index 9dc6c1b87705..7b81cff77598 100644 --- a/ydb/services/monitoring/ya.make +++ b/ydb/services/monitoring/ya.make @@ -9,7 +9,7 @@ PEERDIR( ydb/core/grpc_services ydb/core/protos ydb/public/api/grpc - ydb/public/lib/operation_id + ydb/public/sdk/cpp/src/library/operation_id ) END() diff --git a/ydb/services/persqueue_v1/actors/schema_actors.cpp b/ydb/services/persqueue_v1/actors/schema_actors.cpp index 9fc6789921df..bfd6519c95ae 100644 --- a/ydb/services/persqueue_v1/actors/schema_actors.cpp +++ b/ydb/services/persqueue_v1/actors/schema_actors.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace NKikimr::NGRpcProxy::V1 { diff --git a/ydb/services/persqueue_v1/actors/write_session_actor.cpp b/ydb/services/persqueue_v1/actors/write_session_actor.cpp index 7c247fa8ead5..1107495bfa5f 100644 --- a/ydb/services/persqueue_v1/actors/write_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/write_session_actor.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/services/persqueue_v1/actors/write_session_actor.h b/ydb/services/persqueue_v1/actors/write_session_actor.h index 43434ecdbdf3..55cb5c6899b0 100644 --- a/ydb/services/persqueue_v1/actors/write_session_actor.h +++ b/ydb/services/persqueue_v1/actors/write_session_actor.h @@ -19,6 +19,7 @@ #include #include #include +#include #include diff --git a/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp b/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp index 0b70e58f063f..d2d737c82f81 100644 --- a/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp +++ b/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp @@ -1,7 +1,7 @@ #include -#include -#include -#include +#include +#include +#include #include namespace NKikimr::NPersQueueTests { @@ -110,7 +110,7 @@ Y_UNIT_TEST_SUITE(TFstClassSrcIdPQTest) { writer->Close(); NYdb::NPersQueue::TReadSessionSettings readerSettings; - readerSettings.ConsumerName("debug").AppendTopics(topic); + readerSettings.ConsumerName("debug").AppendTopics(std::string{topic}); auto reader = CreateReader(*driver, readerSettings); auto mbEv = GetNextMessageSkipAssignment(reader); UNIT_ASSERT(mbEv.Defined()); diff --git a/ydb/services/persqueue_v1/persqueue_compat_ut.cpp b/ydb/services/persqueue_v1/persqueue_compat_ut.cpp index 3b3fcd794831..58b84b3e772b 100644 --- a/ydb/services/persqueue_v1/persqueue_compat_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_compat_ut.cpp @@ -1,6 +1,6 @@ -#include -#include -#include +#include +#include +#include #include using namespace NYdb; @@ -106,7 +106,7 @@ Y_UNIT_TEST_SUITE(TPQCompatTest) { THashMap> clustersFound; for (auto i = 0u; i < paths.size() * 2; i++) { auto ev = readSession->GetEvent(true); - Y_ABORT_UNLESS(ev.Defined()); + Y_ABORT_UNLESS(ev.has_value()); auto* lockEvent = std::get_if(&*ev); if (!lockEvent) { if (auto* otherEv = std::get_if(&*ev)) { @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TPQCompatTest) { UNIT_ASSERT(paths.contains(path)); auto& clusters = clustersFound[path]; UNIT_ASSERT(!clusters.contains(cluster)); - clusters.insert(cluster); + clusters.insert(TString{cluster}); //lockEvent->Confirm(); } UNIT_ASSERT_VALUES_EQUAL(clustersFound.size(), paths.size()); diff --git a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp index 2c931d87681e..65b5d0f12b8d 100644 --- a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include @@ -23,7 +23,7 @@ #include #include -#include +#include namespace { const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; @@ -77,7 +77,9 @@ namespace NKikimr::NPersQueueTests { NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())) + .SetDatabase("/Root"); auto ydbDriver = MakeHolder(driverCfg); @@ -100,7 +102,9 @@ namespace NKikimr::NPersQueueTests { PrepareForGrpcNoDC(*server.AnnoyingClient); NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())) + .SetDatabase("/Root"); server.AnnoyingClient->GrantConnect("user1@" BUILTIN_ACL_DOMAIN); @@ -128,7 +132,7 @@ namespace NKikimr::NPersQueueTests { auto testReadFromTopic = [&](const TString& topicPath) { NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("user1").AppendTopics(topicPath); + settings.ConsumerName("user1").AppendTopics(std::string{topicPath}); auto reader = CreateReader(*ydbDriver, settings); for (int i = 0; i < 4; ++i) { @@ -155,8 +159,8 @@ namespace NKikimr::NPersQueueTests { Cerr << ">>>>> Create PersQueue client" << Endl; NYdb::TDriverConfig driverCfg; driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort) - .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)) - .SetDatabase("/Root"); + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())) + .SetDatabase("/Root"); auto ydbDriver = MakeHolder(driverCfg); auto persqueueClient = MakeHolder(*ydbDriver); @@ -250,7 +254,7 @@ namespace NKikimr::NPersQueueTests { std::shared_ptr reader; auto settings = NYdb::NPersQueue::TReadSessionSettings() - .AppendTopics(topic) + .AppendTopics(std::string{topic}) .ConsumerName(consumerName) .StartingMessageTimestamp(curTs) .ReadOnlyOriginal(true); @@ -264,7 +268,7 @@ namespace NKikimr::NPersQueueTests { Cerr << ">>>>> Iteration: " << i << " Got message: " << msg.GetData().substr(0, 16) << " :: " << msg.DebugString(false) << Endl << Flush; - auto count = ++map[msg.GetData()]; + auto count = ++map[TString{msg.GetData()}]; UNIT_ASSERT_C(count == 1, "Each message must be received once"); if (i == 0) { // First iteration. Filling ts and firstOffset vectors from received messages @@ -366,7 +370,9 @@ namespace NKikimr::NPersQueueTests { NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())) + .SetDatabase("/Root"); const auto monPort = TPortManager().GetPort(); auto Counters = server.CleverServer->GetGRpcServerRootCounters(); @@ -458,7 +464,7 @@ namespace NKikimr::NPersQueueTests { } NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName(consumerName).AppendTopics(topicName); + settings.ConsumerName(consumerName).AppendTopics(std::string{topicName}); settings.Header({{NYdb::YDB_APPLICATION_NAME, userAgent}}); auto reader = CreateReader(*ydbDriver, settings); @@ -575,7 +581,7 @@ namespace NKikimr::NPersQueueTests { auto driverCfg = NYdb::TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort) - .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())) .SetDatabase("/Root"); auto ydbDriver = MakeHolder(driverCfg); @@ -668,7 +674,7 @@ namespace NKikimr::NPersQueueTests { } NYdb::NTopic::TReadSessionSettings settings; - settings.ConsumerName(consumerName).AppendTopics(topicName); + settings.ConsumerName(consumerName).AppendTopics(std::string{topicName}); auto reader = CreateReader(*ydbDriver, settings, nullptr, userAgent); @@ -778,7 +784,7 @@ namespace NKikimr::NPersQueueTests { } { auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName("non_existing") - .AppendTopics(topic).DisableClusterDiscovery(true) + .AppendTopics(std::string{topic}).DisableClusterDiscovery(true) .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy())); @@ -786,8 +792,8 @@ namespace NKikimr::NPersQueueTests { future.Wait(TDuration::Seconds(10)); UNIT_ASSERT(future.HasValue()); - TMaybe event = reader->GetEvent(false); - UNIT_ASSERT(event.Defined()); + std::optional event = reader->GetEvent(false); + UNIT_ASSERT(event.has_value()); Cerr << "Got new read session event: " << DebugString(*event) << Endl; @@ -795,7 +801,7 @@ namespace NKikimr::NPersQueueTests { } { auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName(consumer) - .AppendTopics(topic).DisableClusterDiscovery(true) + .AppendTopics(std::string{topic}).DisableClusterDiscovery(true) .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy())); @@ -803,8 +809,8 @@ namespace NKikimr::NPersQueueTests { future.Wait(TDuration::Seconds(10)); UNIT_ASSERT(future.HasValue()); - TMaybe event = reader->GetEvent(false); - UNIT_ASSERT(event.Defined()); + std::optional event = reader->GetEvent(false); + UNIT_ASSERT(event.has_value()); Cerr << "Got new read session event: " << DebugString(*event) << Endl; diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index d242c1fcdd79..1b2fefc97685 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include @@ -40,12 +40,12 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include @@ -160,7 +160,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto driver = server.Server->AnnoyingClient->GetDriver(); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(topicPath).ReadMirrored("dc1"); + settings.ConsumerName("shared/user").AppendTopics(std::string{topicPath}).ReadMirrored("dc1"); Cerr << "=== Create reader\n"; auto reader = CreateReader(*driver, settings); @@ -1486,14 +1486,14 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { rcontext.AddMetadata("x-ydb-auth-ticket", "user@" BUILTIN_ACL_DOMAIN); - TVector>> permissions; + std::vector>> permissions; permissions.push_back({"user@" BUILTIN_ACL_DOMAIN, {"ydb.generic.read"}}); for (ui32 i = 0; i < 10; ++i) { permissions.push_back({"test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN, {"ydb.generic.read"}}); } server.ModifyTopicACL("/Root/PQ/rt3.dc1--acc--topic1", permissions); - TVector>> consumerPermissions; + std::vector>> consumerPermissions; consumerPermissions.push_back({"user@" BUILTIN_ACL_DOMAIN, {"ydb.generic.read", "ydb.granular.write_attributes"}}); for (ui32 i = 0; i < 10; ++i) { consumerPermissions.push_back({"test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN, {"ydb.generic.read", "ydb.granular.write_attributes"}}); @@ -1798,8 +1798,8 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { for (const auto& partInfo: desc.GetPartitions()) { Cerr << ">>>Describe result: partition id = " << partInfo.GetPartitionId() << ", "; auto stats = partInfo.GetPartitionStats(); - UNIT_ASSERT(stats.Defined()); - Cerr << "offsets: [ " << stats.Get()->GetStartOffset() << ", " << stats.Get()->GetEndOffset() << " )" << Endl; + UNIT_ASSERT(stats.has_value()); + Cerr << "offsets: [ " << stats.value().GetStartOffset() << ", " << stats.value().GetEndOffset() << " )" << Endl; } TAlterTopicSettings alterSettings; @@ -3285,7 +3285,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateSyncExecutorWrapper(); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); auto reader = CreateReader(*driver, settings); @@ -3382,7 +3382,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateThreadPoolExecutorWrapper(2); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); auto reader = CreateReader(*driver, settings); @@ -3521,7 +3521,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateThreadPoolExecutorWrapper(2); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); auto reader = CreateReader(*driver, settings); @@ -3660,7 +3660,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateThreadPoolExecutorWrapper(2); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); settings.MaxMemoryUsageBytes(5_MB); settings.Decompress(true); @@ -3713,7 +3713,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateThreadPoolExecutorWrapper(2); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); settings.MaxMemoryUsageBytes(maxMemoryUsageSize); settings.Decompress(decompress); @@ -3908,7 +3908,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto decompressor = CreateThreadPoolExecutorWrapper(2); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.DecompressionExecutor(decompressor); auto reader = CreateReader(*driver, settings); @@ -4105,7 +4105,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { { NYdb::NPersQueue::TReadSessionSettings settings; settings.ConsumerName(originallyProvidedConsumerName) - .AppendTopics(TString("account/topic1")).ReadOriginal({"dc1"}) + .AppendTopics(std::string{"account/topic1"}).ReadOriginal({"dc1"}) .Header({{NYdb::YDB_APPLICATION_NAME, userAgent}}); auto reader = CreateReader(*driver, settings); @@ -5632,7 +5632,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto ct = std::get_if(&*ev); UNIT_ASSERT(ct); writer->Write(std::move(ct->ContinuationToken), "1234567890"); - UNIT_ASSERT(ev.Defined()); + UNIT_ASSERT(ev.has_value()); while(true) { ev = writer->GetEvent(true); auto ack = std::get_if(&*ev); @@ -6864,7 +6864,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { .ReadOnlyOriginal(true) ); - TMaybe event = reader->GetEvent(true, 1); + std::optional event = reader->GetEvent(true, 1); auto createStream = std::get_if(&*event); UNIT_ASSERT(createStream); TString stepDescription = TStringBuilder() << "create stream for partition=" << partition @@ -6912,7 +6912,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto ct = std::get_if(&*ev); UNIT_ASSERT(ct); writer->Write(std::move(ct->ContinuationToken), "1234567890"); - UNIT_ASSERT(ev.Defined()); + UNIT_ASSERT(ev.has_value()); while(true) { ev = writer->GetEvent(true); auto ack = std::get_if(&*ev); @@ -6954,14 +6954,14 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto reader = CreateReader( *driver, NYdb::NPersQueue::TReadSessionSettings() - .AppendTopics(topic) + .AppendTopics(std::string{topic}) .ConsumerName("shared/user") .ReadOnlyOriginal(true) ); { - TMaybe event = reader->GetEvent(true, 1); + std::optional event = reader->GetEvent(true, 1); auto createStream = std::get_if(&*event); UNIT_ASSERT(createStream); Cerr << "Create stream event: " << createStream->DebugString() << Endl; @@ -6970,7 +6970,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { { auto future = reader->WaitEvent(); UNIT_ASSERT(future.Wait(TDuration::Seconds(10))); - TMaybe event = reader->GetEvent(true, 1); + std::optional event = reader->GetEvent(true, 1); auto partitionStatus = std::get_if(&*event); UNIT_ASSERT(partitionStatus); Cerr << "partition status: " << partitionStatus->DebugString() << Endl; @@ -6998,7 +6998,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { THashSet locksGot = {}; while(locksGot.size() < 2) { - TMaybe event = reader->GetEvent(true, 1); + std::optional event = reader->GetEvent(true, 1); auto createStream = std::get_if(&*event); UNIT_ASSERT(createStream); Cerr << "Create stream event: " << createStream->DebugString() << Endl; @@ -7020,7 +7020,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { locksGot.clear(); THashSet releasesGot = {}; while (locksGot.size() < 3) { - TMaybe event = reader2->GetEvent(true, 1); + std::optional event = reader2->GetEvent(true, 1); auto createStream = std::get_if(&*event); UNIT_ASSERT(createStream); Cerr << "Create stream event: " << createStream->DebugString() << Endl; @@ -7037,7 +7037,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { } while (!releasesGot.empty()) { - TMaybe event = reader->GetEvent(true, 1); + std::optional event = reader->GetEvent(true, 1); auto release = std::get_if(&*event); UNIT_ASSERT(release); UNIT_ASSERT_VALUES_EQUAL(release->GetPartitionStream()->GetTopicPath(), topic); @@ -7077,7 +7077,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { NYdb::NTopic::TWriteSessionSettings wSettings {topicFullName, "srcId", "srcId"}; wSettings.DirectWriteToPartition(false); auto writer = topicClient.CreateSimpleBlockingWriteSession(wSettings); - TVector> metadata = {{"key1", "val1"}, {"key2", "val2"}}; + std::vector> metadata = {{"key1", "val1"}, {"key2", "val2"}}; { auto message = NYdb::NTopic::TWriteMessage{"Somedata"}.MessageMeta(metadata); writer->Write(std::move(message)); @@ -7094,7 +7094,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto readSession = topicClient.CreateReadSession(rSettings); auto ev = readSession->GetEvent(true); - UNIT_ASSERT(ev.Defined()); + UNIT_ASSERT(ev.has_value()); auto spsEv = std::get_if(&*ev); UNIT_ASSERT(spsEv); spsEv->Confirm(); @@ -7194,7 +7194,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { Cerr << "Start reads\n"; while(totalMessages < 9) { auto ev = readSession->GetEvent(true); - UNIT_ASSERT(ev.Defined()); + UNIT_ASSERT(ev.has_value()); auto spsEv = std::get_if(&*ev); if (spsEv) { @@ -7234,7 +7234,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { NYdb::NPersQueue::TPersQueueClient persqueueClient(*driver); NYdb::NPersQueue::TReadSessionSettings settings; - settings.ConsumerName("shared/user").AppendTopics(SHORT_TOPIC_NAME).ReadOriginal({"dc1"}); + settings.ConsumerName("shared/user").AppendTopics(std::string{SHORT_TOPIC_NAME}).ReadOriginal({"dc1"}); settings.MaxMemoryUsageBytes(1_MB); settings.DisableClusterDiscovery(true); diff --git a/ydb/services/persqueue_v1/topic_yql_ut.cpp b/ydb/services/persqueue_v1/topic_yql_ut.cpp index 92534059a64a..be0185301e4f 100644 --- a/ydb/services/persqueue_v1/topic_yql_ut.cpp +++ b/ydb/services/persqueue_v1/topic_yql_ut.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace NKikimr::NPersQueueTests { diff --git a/ydb/services/persqueue_v1/ut/demo_tx.cpp b/ydb/services/persqueue_v1/ut/demo_tx.cpp index e2218261105b..aed95b55bf9e 100644 --- a/ydb/services/persqueue_v1/ut/demo_tx.cpp +++ b/ydb/services/persqueue_v1/ut/demo_tx.cpp @@ -1,10 +1,10 @@ #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include @@ -227,7 +227,7 @@ auto TTxFixture::CreateTopicReadSession(const TString& topic, const TString& consumer) -> TTopicReadSessionPtr { NYdb::NPersQueue::TReadSessionSettings settings; - settings.AppendTopics(topic); + settings.AppendTopics(std::string{topic}); settings.ConsumerName(consumer); settings.ReadOriginal({DC}); @@ -279,8 +279,8 @@ void TTxFixture::Call_UpdateOffsetsInTransaction(const NYdb::NTable::TTransactio Ydb::Topic::UpdateOffsetsInTransactionRequest request; Ydb::Topic::UpdateOffsetsInTransactionResponse response; - request.mutable_tx()->set_id(tx.GetId()); - request.mutable_tx()->set_session(tx.GetSession().GetId()); + request.mutable_tx()->set_id(TString{tx.GetId()}); + request.mutable_tx()->set_session(TString{tx.GetSession().GetId()}); request.set_consumer(consumer); auto *topic = request.mutable_topics()->Add(); @@ -335,10 +335,10 @@ void TTxFixture::Ensure_In_Table(const TString& tablePath, UNIT_ASSERT(rs.TryNextRow()); auto key = rs.ColumnParser("key").GetOptionalInt64(); - UNIT_ASSERT(key.Defined()); + UNIT_ASSERT(key.has_value()); auto value = rs.ColumnParser("value").GetOptionalInt64(); - UNIT_ASSERT(value.Defined()); + UNIT_ASSERT(value.has_value()); auto p = values.find(*key); UNIT_ASSERT(p != values.end()); diff --git a/ydb/services/persqueue_v1/ut/describes_ut/describe_topic_ut.cpp b/ydb/services/persqueue_v1/ut/describes_ut/describe_topic_ut.cpp index 96d71039dcd6..80529054c966 100644 --- a/ydb/services/persqueue_v1/ut/describes_ut/describe_topic_ut.cpp +++ b/ydb/services/persqueue_v1/ut/describes_ut/describe_topic_ut.cpp @@ -1,10 +1,11 @@ #include #include -#include +#include #include #include #include +#include namespace NKikimr::NPersQueueTests { diff --git a/ydb/services/persqueue_v1/ut/describes_ut/ic_cache_ut.cpp b/ydb/services/persqueue_v1/ut/describes_ut/ic_cache_ut.cpp index 5fbd9a4723f6..a8a28441f5a5 100644 --- a/ydb/services/persqueue_v1/ut/describes_ut/ic_cache_ut.cpp +++ b/ydb/services/persqueue_v1/ut/describes_ut/ic_cache_ut.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include diff --git a/ydb/services/persqueue_v1/ut/describes_ut/ya.make b/ydb/services/persqueue_v1/ut/describes_ut/ya.make index 88fd164896d1..2f850497c617 100644 --- a/ydb/services/persqueue_v1/ut/describes_ut/ya.make +++ b/ydb/services/persqueue_v1/ut/describes_ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/services/persqueue_v1) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) @@ -19,7 +23,8 @@ PEERDIR( ydb/core/testlib/default ydb/core/client/server ydb/services/persqueue_v1 - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/api/grpc ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/persqueue_v1/ut/functions_executor_wrapper.h b/ydb/services/persqueue_v1/ut/functions_executor_wrapper.h index d3ebc56cb297..d928b2a692ae 100644 --- a/ydb/services/persqueue_v1/ut/functions_executor_wrapper.h +++ b/ydb/services/persqueue_v1/ut/functions_executor_wrapper.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make index 53a2de055807..3adda358bc58 100644 --- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make +++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make @@ -1,5 +1,9 @@ UNITTEST_FOR(ydb/services/persqueue_v1) +ADDINCL( + ydb/public/sdk/cpp +) + FORK_SUBTESTS() IF (WITH_VALGRIND) @@ -26,9 +30,9 @@ PEERDIR( ydb/library/persqueue/tests ydb/core/testlib/default ydb/public/api/grpc - ydb/public/sdk/cpp/client/resources - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/resources + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/table ydb/services/persqueue_v1 ) diff --git a/ydb/services/persqueue_v1/ut/persqueue_common_tests.h b/ydb/services/persqueue_v1/ut/persqueue_common_tests.h index 81ccd2a91259..3de5694cadbb 100644 --- a/ydb/services/persqueue_v1/ut/persqueue_common_tests.h +++ b/ydb/services/persqueue_v1/ut/persqueue_common_tests.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include @@ -24,7 +24,7 @@ #include #include -#include +#include namespace NKikimr::NPersQueueTests { @@ -94,7 +94,7 @@ class TCommonTests { TPersQueueV1TestServer server = CreateServer(); const int iterations = 10; - TVector>> permissions; + std::vector>> permissions; for (int i = 0; i != iterations; ++i) { permissions.push_back({GenerateValidToken(i), {"ydb.generic.write"}}); } @@ -176,7 +176,7 @@ class TCommonTests { TPersQueueV1TestServer server = CreateServer(); SET_LOCALS; const int iterations = 3; - TVector>> permissions; + std::vector>> permissions; for (int i = 0; i != iterations; ++i) { permissions.push_back({GenerateValidToken(i), {"ydb.generic.write"}}); } diff --git a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h index e3f473ed280c..a8007eea8063 100644 --- a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h +++ b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h @@ -7,16 +7,16 @@ #include -#include -#include -#include -#include +#include +#include +#include +#include #include namespace NKikimr::NPersQueueTests { -static void ModifyTopicACL(NYdb::TDriver* driver, const TString& topic, const TVector>>& acl) { +static void ModifyTopicACL(NYdb::TDriver* driver, const TString& topic, const std::vector>>& acl) { NYdb::NScheme::TSchemeClient schemeClient(*driver); auto modifyPermissionsSettings = NYdb::NScheme::TModifyPermissionsSettings(); @@ -114,7 +114,7 @@ static void ModifyTopicACL(NYdb::TDriver* driver, const TString& topic, const TV Cerr << "=== PersQueueClient" << Endl; NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << Server->GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << Server->GrpcPort).SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())).SetDatabase("/Root"); YdbDriver.reset(new NYdb::TDriver(driverCfg)); PersQueueClient = MakeHolder(*YdbDriver); @@ -176,11 +176,11 @@ static void ModifyTopicACL(NYdb::TDriver* driver, const TString& topic, const TV return TenantMode; } - void ModifyTopicACL(const TString& topic, const TVector>>& acl) { + void ModifyTopicACL(const TString& topic, const std::vector>>& acl) { ::ModifyTopicACL(YdbDriver.get(), topic, acl); } - void ModifyTopicACLAndWait(const TString& topic, const TVector>>& acl) { + void ModifyTopicACLAndWait(const TString& topic, const std::vector>>& acl) { auto driver = YdbDriver.get(); auto schemeClient = NYdb::NScheme::TSchemeClient(*driver); auto desc = schemeClient.DescribePath(topic).ExtractValueSync(); diff --git a/ydb/services/persqueue_v1/ut/pq_data_writer.h b/ydb/services/persqueue_v1/ut/pq_data_writer.h index 8671f0883490..f26cda532b03 100644 --- a/ydb/services/persqueue_v1/ut/pq_data_writer.h +++ b/ydb/services/persqueue_v1/ut/pq_data_writer.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/services/persqueue_v1/ut/topic_service_ut.cpp b/ydb/services/persqueue_v1/ut/topic_service_ut.cpp index 66c14c30fa78..4df8f2fc7556 100644 --- a/ydb/services/persqueue_v1/ut/topic_service_ut.cpp +++ b/ydb/services/persqueue_v1/ut/topic_service_ut.cpp @@ -1,9 +1,9 @@ #include -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -153,7 +153,7 @@ class TUpdateOffsetsInTransactionFixture : public NUnitTest::TBaseFixture { server->AnnoyingClient->CreateTopicNoLegacy(VALID_TOPIC_PATH, partsCount, true, true, - Nothing(), + std::nullopt, {"c0nsumer", "consumer-1", "consumer-2"}); NACLib::TDiffACL acl; @@ -184,7 +184,7 @@ class TUpdateOffsetsInTransactionFixture : public NUnitTest::TBaseFixture { Ydb::Topic::UpdateOffsetsInTransactionResponse response; grpc::Status status = stub->UpdateOffsetsInTransaction(&rcontext, - CreateRequest(session->GetId(), tx->GetId(), + CreateRequest(TString{session->GetId()}, TString{tx->GetId()}, consumer, topics), &response); UNIT_ASSERT(status.ok()); diff --git a/ydb/services/persqueue_v1/ut/ya.make b/ydb/services/persqueue_v1/ut/ya.make index ccd28c0883e9..5306994a6ee3 100644 --- a/ydb/services/persqueue_v1/ut/ya.make +++ b/ydb/services/persqueue_v1/ut/ya.make @@ -1,5 +1,13 @@ UNITTEST_FOR(ydb/services/persqueue_v1) +ADDINCL( + ydb/public/sdk/cpp +) + +CFLAGS( + -DYDB_SDK_USE_STD_STRING +) + FORK_SUBTESTS() IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) @@ -43,11 +51,11 @@ PEERDIR( ydb/library/persqueue/tests ydb/library/persqueue/topic_parser ydb/public/api/grpc - ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils - ydb/public/sdk/cpp/client/ydb_persqueue_public - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_topic - ydb/public/sdk/cpp/client/ydb_proto + ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils + ydb/public/sdk/cpp/src/client/persqueue_public + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/topic + ydb/public/sdk/cpp/src/client/proto ydb/services/persqueue_v1 ) diff --git a/ydb/services/persqueue_v1/ya.make b/ydb/services/persqueue_v1/ya.make index ca923cb01a39..65038f5d6852 100644 --- a/ydb/services/persqueue_v1/ya.make +++ b/ydb/services/persqueue_v1/ya.make @@ -26,7 +26,7 @@ PEERDIR( ydb/core/protos ydb/core/ydb_convert ydb/library/aclib - ydb/library/persqueue/obfuscate + ydb/public/sdk/cpp/src/library/persqueue/obfuscate # ydb/library/persqueue/tests ydb/library/persqueue/topic_parser ydb/public/api/grpc diff --git a/ydb/services/rate_limiter/rate_limiter_ut.cpp b/ydb/services/rate_limiter/rate_limiter_ut.cpp index b62f711cfd48..22e68d0ccab0 100644 --- a/ydb/services/rate_limiter/rate_limiter_ut.cpp +++ b/ydb/services/rate_limiter/rate_limiter_ut.cpp @@ -1,13 +1,13 @@ #include -#include -#include +#include +#include -#include +#include #include -#include +#include #include #include @@ -113,16 +113,16 @@ class TTestSetupAcquireActor : public TTestSetup { } if (Settings.IsUsedAmount_) { - request.set_used(Settings.Amount_.GetRef()); + request.set_used(Settings.Amount_.value()); } else { - request.set_required(Settings.Amount_.GetRef()); + request.set_required(Settings.Amount_.value()); } auto id = SelfId(); auto cb = [this, id](Ydb::RateLimiter::AcquireResourceResponse resp) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(resp.operation().issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(resp.operation().issues(), opIssues); NYdb::TStatus status(static_cast(resp.operation().status()), std::move(opIssues)); Promise.SetValue(status); Send(id, new TEvents::TEvPoisonPill); @@ -277,7 +277,7 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) { const auto listResultFuture = setup.RateLimiterClient.ListResources(TTestSetup::CoordinationNodePath, "", TListResourcesSettings().Recursive(true)); ASSERT_STATUS_SUCCESS(listResultFuture); const auto listResult = listResultFuture.GetValueSync(); - TVector paths = listResult.GetResourcePaths(); + auto paths = listResult.GetResourcePaths(); std::sort(paths.begin(), paths.end()); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 5); UNIT_ASSERT_VALUES_EQUAL(paths[0], "parent1"); @@ -292,7 +292,7 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) { const auto listResultFuture = setup.RateLimiterClient.ListResources(TTestSetup::CoordinationNodePath, "", TListResourcesSettings().Recursive(false)); ASSERT_STATUS_SUCCESS(listResultFuture); const auto listResult = listResultFuture.GetValueSync(); - TVector paths = listResult.GetResourcePaths(); + auto paths = listResult.GetResourcePaths(); std::sort(paths.begin(), paths.end()); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2); UNIT_ASSERT_VALUES_EQUAL(paths[0], "parent1"); @@ -304,7 +304,7 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) { const auto listResultFuture = setup.RateLimiterClient.ListResources(TTestSetup::CoordinationNodePath, "parent1", TListResourcesSettings().Recursive()); ASSERT_STATUS_SUCCESS(listResultFuture); const auto listResult = listResultFuture.GetValueSync(); - TVector paths = listResult.GetResourcePaths(); + auto paths = listResult.GetResourcePaths(); std::sort(paths.begin(), paths.end()); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 3); UNIT_ASSERT_VALUES_EQUAL(paths[0], "parent1"); diff --git a/ydb/services/rate_limiter/ut/ya.make b/ydb/services/rate_limiter/ut/ya.make index 5025ae8092bc..e0977f571b62 100644 --- a/ydb/services/rate_limiter/ut/ya.make +++ b/ydb/services/rate_limiter/ut/ya.make @@ -8,8 +8,8 @@ SRCS( PEERDIR( ydb/core/testlib/default - ydb/public/sdk/cpp/client/ydb_coordination - ydb/public/sdk/cpp/client/ydb_rate_limiter + ydb/public/sdk/cpp/src/client/coordination + ydb/public/sdk/cpp/src/client/rate_limiter ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make index 39732a69bd9c..592213ad5de2 100644 --- a/ydb/services/ydb/backup_ut/ya.make +++ b/ydb/services/ydb/backup_ut/ya.make @@ -17,12 +17,12 @@ PEERDIR( ydb/core/testlib/default ydb/core/wrappers/ut_helpers ydb/public/lib/ydb_cli/dump - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_import - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_value + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/import + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/value ydb/library/backup contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ) diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp index d7be9f1084e2..f797e11b2f57 100644 --- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp +++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp @@ -4,11 +4,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include diff --git a/ydb/services/ydb/sdk_sessions_pool_ut/sdk_sessions_pool_ut.cpp b/ydb/services/ydb/sdk_sessions_pool_ut/sdk_sessions_pool_ut.cpp index 97ae2d6aa3c5..3dc29eb7a5f9 100644 --- a/ydb/services/ydb/sdk_sessions_pool_ut/sdk_sessions_pool_ut.cpp +++ b/ydb/services/ydb/sdk_sessions_pool_ut/sdk_sessions_pool_ut.cpp @@ -1,6 +1,6 @@ #include "ydb_common_ut.h" -#include +#include #include #include diff --git a/ydb/services/ydb/sdk_sessions_pool_ut/ya.make b/ydb/services/ydb/sdk_sessions_pool_ut/ya.make index 0e977b03c3df..ae03a20c02e0 100644 --- a/ydb/services/ydb/sdk_sessions_pool_ut/ya.make +++ b/ydb/services/ydb/sdk_sessions_pool_ut/ya.make @@ -15,10 +15,10 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/testlib/default ydb/core/testlib - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/ydb/sdk_sessions_ut/sdk_sessions_ut.cpp b/ydb/services/ydb/sdk_sessions_ut/sdk_sessions_ut.cpp index 3cb04441ae9f..3707859a02c3 100644 --- a/ydb/services/ydb/sdk_sessions_ut/sdk_sessions_ut.cpp +++ b/ydb/services/ydb/sdk_sessions_ut/sdk_sessions_ut.cpp @@ -1,6 +1,6 @@ #include "ydb_common_ut.h" -#include -#include +#include +#include #include @@ -69,7 +69,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { NYdb::NTable::TTableClient client(driver); int count = 10; - THashSet sids; + THashSet sids; while (count--) { auto sessionResponse = client.GetSession().ExtractValueSync(); UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); @@ -650,7 +650,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { TKikimrWithGrpcAndRootSchema server(appConfig); - TVector sessionIds; + std::vector sessionIds; int iterations = 50; while (iterations--) { @@ -669,7 +669,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { for (const auto& sessionId : sessionIds) { grpc::ClientContext context; Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); + request.set_session_id(TStringType{sessionId}); Ydb::Table::KeepAliveResponse response; auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); @@ -684,7 +684,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { TKikimrWithGrpcAndRootSchema server(appConfig); - TVector sessionIds; + std::vector sessionIds; int iterations = 100; while (iterations--) { @@ -727,7 +727,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { for (const auto& sessionId : sessionIds) { grpc::ClientContext context; Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); + request.set_session_id(TStringType{sessionId}); Ydb::Table::KeepAliveResponse response; auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); @@ -742,7 +742,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { TKikimrWithGrpcAndRootSchema server(appConfig); - TVector sessionIds; + std::vector sessionIds; int iterations = 100; while (iterations--) { @@ -773,7 +773,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { for (const auto& sessionId : sessionIds) { grpc::ClientContext context; Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); + request.set_session_id(TStringType{sessionId}); Ydb::Table::KeepAliveResponse response; auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); @@ -788,7 +788,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { TKikimrWithGrpcAndRootSchema server(appConfig); - TVector sessionIds; + std::vector sessionIds; int iterations = 100; while (iterations--) { @@ -819,7 +819,7 @@ Y_UNIT_TEST_SUITE(YdbSdkSessions) { for (const auto& sessionId : sessionIds) { grpc::ClientContext context; Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); + request.set_session_id(TStringType{sessionId}); Ydb::Table::KeepAliveResponse response; auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); diff --git a/ydb/services/ydb/sdk_sessions_ut/ya.make b/ydb/services/ydb/sdk_sessions_ut/ya.make index 886c0116ea08..c62a0941b933 100644 --- a/ydb/services/ydb/sdk_sessions_ut/ya.make +++ b/ydb/services/ydb/sdk_sessions_ut/ya.make @@ -15,10 +15,10 @@ SRCS( ) PEERDIR( - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/core/testlib/default ydb/core/testlib - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ydb/public/lib/ut_helpers ) diff --git a/ydb/services/ydb/table_split_ut/ya.make b/ydb/services/ydb/table_split_ut/ya.make index 738e5b7651f7..d1238863833e 100644 --- a/ydb/services/ydb/table_split_ut/ya.make +++ b/ydb/services/ydb/table_split_ut/ya.make @@ -17,7 +17,7 @@ SRCS( PEERDIR( contrib/libs/apache/arrow library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/svnversion ydb/core/kqp/ut/common @@ -29,13 +29,13 @@ PEERDIR( ydb/public/lib/experimental ydb/public/lib/json_value ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_coordination - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_extension - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_monitoring + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/coordination + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/extension_common + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/monitoring ydb/services/ydb ) diff --git a/ydb/services/ydb/ut/ya.make b/ydb/services/ydb/ut/ya.make index ce58d8659001..6c51d5f62ea2 100644 --- a/ydb/services/ydb/ut/ya.make +++ b/ydb/services/ydb/ut/ya.make @@ -33,7 +33,7 @@ SRCS( PEERDIR( contrib/libs/apache/arrow library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client library/cpp/regex/pcre library/cpp/svnversion ydb/core/kqp/ut/common @@ -50,13 +50,13 @@ PEERDIR( ydb/public/lib/yson_value ydb/public/lib/ut_helpers ydb/public/lib/ydb_cli/commands - ydb/public/sdk/cpp/client/draft - ydb/public/sdk/cpp/client/ydb_coordination - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_extension - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_monitoring + ydb/public/sdk/cpp/src/client/draft + ydb/public/sdk/cpp/src/client/coordination + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/extension_common + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/monitoring ydb/services/ydb ) diff --git a/ydb/services/ydb/ydb_bulk_upsert_olap_ut.cpp b/ydb/services/ydb/ydb_bulk_upsert_olap_ut.cpp index c292b29b052c..2acd54a739e7 100644 --- a/ydb/services/ydb/ydb_bulk_upsert_olap_ut.cpp +++ b/ydb/services/ydb/ydb_bulk_upsert_olap_ut.cpp @@ -1,7 +1,7 @@ #include "ydb_common_ut.h" -#include -#include +#include +#include #include #include @@ -14,6 +14,14 @@ using namespace NYdb; namespace { +template +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return ToString(opt.value()); + } + return "(NULL)"; +} + std::vector ScanQuerySelectSimple( NYdb::NTable::TTableClient client, const TString& tablePath, const std::vector>& ydbSchema = TTestOlap::PublicSchema()) @@ -55,11 +63,11 @@ std::vector ScanQuerySelectSimple( if (colName == "timestamp") { ss << parser.ColumnParser(colName).GetTimestamp() << ","; } else { - ss << parser.ColumnParser(colName).GetOptionalTimestamp() << ","; + ss << OptionalToString(parser.ColumnParser(colName).GetOptionalTimestamp()) << ","; } break; case NYdb::EPrimitiveType::Datetime: - ss << parser.ColumnParser(colName).GetOptionalDatetime() << ","; + ss << OptionalToString(parser.ColumnParser(colName).GetOptionalDatetime()) << ","; break; case NYdb::EPrimitiveType::String: { auto& col = parser.ColumnParser(colName); @@ -78,7 +86,7 @@ std::vector ScanQuerySelectSimple( } break; case NYdb::EPrimitiveType::Int32: - ss << parser.ColumnParser(colName).GetOptionalInt32() << ","; + ss << OptionalToString(parser.ColumnParser(colName).GetOptionalInt32()) << ","; break; case NYdb::EPrimitiveType::JsonDocument: ss << parser.ColumnParser(colName).GetOptionalJsonDocument() << ","; @@ -457,7 +465,7 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsertOlap) { auto rows = ScanQuerySelect(client, tablePath, schema); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(rows[0], - "123123bs,testd,subscr,2020-01-17T22:58:50.000000Z,1973-11-27T01:52:03.000000Z,(empty maybe),http,ru,AsiaNovo,hello,{}"); + "123123bs,testd,subscr,2020-01-17T22:58:50.000000Z,1973-11-27T01:52:03.000000Z,(NULL),http,ru,AsiaNovo,hello,{}"); result = session.DropTable(tablePath).GetValueSync(); UNIT_ASSERT_EQUAL(result.IsTransportError(), false); diff --git a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp index a38c9caac229..096b447983b3 100644 --- a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp +++ b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp @@ -1,7 +1,7 @@ #include "ydb_common_ut.h" -#include -#include +#include +#include #include #include @@ -1276,7 +1276,7 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { auto status = db.RetryOperationSync([&failInjector](NYdb::NTable::TTableClient& db) { EStatus injected = failInjector.GetInjectedStatus(); if (injected != EStatus::SUCCESS) { - return NYdb::NTable::TBulkUpsertResult(TStatus(injected, NYql::TIssues())); + return NYdb::NTable::TBulkUpsertResult(TStatus(injected, NYdb::NIssue::TIssues())); } NYdb::TValueBuilder rows; @@ -1327,7 +1327,7 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { [&failInjector](NYdb::NTable::TTableClient& db) { EStatus injected = failInjector.GetInjectedStatus(); if (injected != EStatus::SUCCESS) { - return NThreading::MakeFuture(NYdb::NTable::TBulkUpsertResult(TStatus(injected, NYql::TIssues()))); + return NThreading::MakeFuture(NYdb::NTable::TBulkUpsertResult(TStatus(injected, NYdb::NIssue::TIssues()))); } NYdb::TValueBuilder rows; diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index c8d5b4f67c91..1352f62a458a 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/services/ydb/ydb_coordination_ut.cpp b/ydb/services/ydb/ydb_coordination_ut.cpp index 36a7c9752ab8..d8353d3f71b0 100644 --- a/ydb/services/ydb/ydb_coordination_ut.cpp +++ b/ydb/services/ydb/ydb_coordination_ut.cpp @@ -1,7 +1,7 @@ #include "ydb_common_ut.h" -#include -#include +#include +#include #include #include @@ -141,7 +141,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::UNAUTHORIZED, TStatusDescription(result)); bool issueFound = false; for (auto& issue : result.GetIssues()) { - NYql::WalkThroughIssues(issue, false, [&issueFound](const NYql::TIssue& issue, ui16){ + NYdb::NIssue::WalkThroughIssues(issue, false, [&issueFound](const NYdb::NIssue::TIssue& issue, ui16){ if (issue.GetCode() == NKikimrIssues::TIssuesIds::ACCESS_DENIED) { issueFound = true; } @@ -181,7 +181,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { auto result = ExpectSuccess(context.Client.DescribeNode("/Root/node2")); UNIT_ASSERT_VALUES_EQUAL( - result.GetSelfCheckPeriod(), + result.GetSelfCheckPeriod().value(), TDuration::MilliSeconds(1234)); } @@ -198,7 +198,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { auto desc1 = ExpectSuccess(context.Client.DescribeNode("/Root/node1")); UNIT_ASSERT_VALUES_EQUAL( - desc1.GetSelfCheckPeriod(), + desc1.GetSelfCheckPeriod().value(), TDuration::MilliSeconds(1234)); ExpectSuccess( @@ -210,10 +210,10 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { auto desc2 = ExpectSuccess(context.Client.DescribeNode("/Root/node1")); UNIT_ASSERT_VALUES_EQUAL( - desc2.GetSelfCheckPeriod(), + desc2.GetSelfCheckPeriod().value(), TDuration::MilliSeconds(1234)); UNIT_ASSERT_VALUES_EQUAL( - desc2.GetSessionGracePeriod(), + desc2.GetSessionGracePeriod().value(), TDuration::MilliSeconds(5678)); ExpectSuccess( @@ -225,10 +225,10 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { auto desc3 = ExpectSuccess(context.Client.DescribeNode("/Root/node1")); UNIT_ASSERT_VALUES_EQUAL( - desc3.GetSelfCheckPeriod(), + desc3.GetSelfCheckPeriod().value(), TDuration::MilliSeconds(2345)); UNIT_ASSERT_VALUES_EQUAL( - desc2.GetSessionGracePeriod(), + desc2.GetSessionGracePeriod().value(), TDuration::MilliSeconds(5678)); ExpectSuccess( @@ -677,7 +677,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClientAuth) { for (int idx = 2; idx <= 4; ++idx) { TString path = TStringBuilder() << "/Root/node" << idx; - TVector permissions(allPermissions.begin(), allPermissions.begin() + (idx - 1)); + std::vector permissions(allPermissions.begin(), allPermissions.begin() + (idx - 1)); ExpectSuccess(context.SchemeClient.ModifyPermissions(path, TModifyPermissionsSettings() .AddGrantPermissions(TPermissions( diff --git a/ydb/services/ydb/ydb_import_ut.cpp b/ydb/services/ydb/ydb_import_ut.cpp index 0009d21f2aea..eb0f3daa85cf 100644 --- a/ydb/services/ydb/ydb_import_ut.cpp +++ b/ydb/services/ydb/ydb_import_ut.cpp @@ -1,8 +1,8 @@ #include "ydb_common_ut.h" -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_index_table_ut.cpp b/ydb/services/ydb/ydb_index_table_ut.cpp index cf353f0e461c..057497f8485a 100644 --- a/ydb/services/ydb/ydb_index_table_ut.cpp +++ b/ydb/services/ydb/ydb_index_table_ut.cpp @@ -1,6 +1,6 @@ -#include -#include -#include +#include +#include +#include #include #include @@ -33,7 +33,7 @@ void CreateTestTableWithIndex(NYdb::NTable::TTableClient& client) { .AddNullableColumn("Timestamp", EPrimitiveType::Int64) .AddNullableColumn("Data", EPrimitiveType::String) .SetPrimaryKeyColumns({"NameHash", "Name"}) - .AddSecondaryIndex("TimestampIndex",TVector({"Timestamp", "Name", "Version"})); + .AddSecondaryIndex("TimestampIndex", {"Timestamp", "Name", "Version"}); auto tableSettings = NYdb::NTable::TCreateTableSettings().PartitioningPolicy( NYdb::NTable::TPartitioningPolicy().UniformPartitions(SHARD_COUNT)); diff --git a/ydb/services/ydb/ydb_ldap_login_ut.cpp b/ydb/services/ydb/ydb_ldap_login_ut.cpp index 206f24dad19a..2d058adf3436 100644 --- a/ydb/services/ydb/ydb_ldap_login_ut.cpp +++ b/ydb/services/ydb/ydb_ldap_login_ut.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/services/ydb/ydb_login_ut.cpp b/ydb/services/ydb/ydb_login_ut.cpp index 26350905af41..87f8031373b1 100644 --- a/ydb/services/ydb/ydb_login_ut.cpp +++ b/ydb/services/ydb/ydb_login_ut.cpp @@ -1,8 +1,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_logstore_ut.cpp b/ydb/services/ydb/ydb_logstore_ut.cpp index 057d70d47b6c..d9167be32284 100644 --- a/ydb/services/ydb/ydb_logstore_ut.cpp +++ b/ydb/services/ydb/ydb_logstore_ut.cpp @@ -1,7 +1,7 @@ #include "ydb_common_ut.h" -#include -#include +#include +#include #include #include @@ -96,7 +96,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { const auto& schema = descr.GetSchemaPresets().begin()->second; UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns().size(), 10); - UNIT_ASSERT(schema.GetColumns()[0].ToString().StartsWith("{ name: \"timestamp\", type:")); + UNIT_ASSERT(schema.GetColumns()[0].ToString().starts_with("{ name: \"timestamp\", type:")); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[1].ToString(), "{ name: \"resource_type\", type: Utf8 }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[4].ToString(), "{ name: \"level\", type: Int32? }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetPrimaryKeyColumns(), @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { UNIT_ASSERT_VALUES_EQUAL(descr.GetShardsCount(), 4); const auto& schema = descr.GetSchema(); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns().size(), 10); - UNIT_ASSERT(schema.GetColumns()[0].ToString().StartsWith("{ name: \"timestamp\", type:")); + UNIT_ASSERT(schema.GetColumns()[0].ToString().starts_with("{ name: \"timestamp\", type:")); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[1].ToString(), "{ name: \"resource_type\", type: Utf8 }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[4].ToString(), "{ name: \"level\", type: Int32? }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetPrimaryKeyColumns(), @@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { UNIT_ASSERT_VALUES_EQUAL(descr.GetShardsCount(), 4); const auto& schema = descr.GetSchema(); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns().size(), 10); - UNIT_ASSERT(schema.GetColumns()[0].ToString().StartsWith("{ name: \"timestamp\", type:")); + UNIT_ASSERT(schema.GetColumns()[0].ToString().starts_with("{ name: \"timestamp\", type:")); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[1].ToString(), "{ name: \"resource_type\", type: Utf8 }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetColumns()[4].ToString(), "{ name: \"level\", type: Int32? }"); UNIT_ASSERT_VALUES_EQUAL(schema.GetPrimaryKeyColumns(), diff --git a/ydb/services/ydb/ydb_monitoring_ut.cpp b/ydb/services/ydb/ydb_monitoring_ut.cpp index b6827d9d23cf..ef88f3c91ca0 100644 --- a/ydb/services/ydb/ydb_monitoring_ut.cpp +++ b/ydb/services/ydb/ydb_monitoring_ut.cpp @@ -6,9 +6,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_object_storage_ut.cpp b/ydb/services/ydb/ydb_object_storage_ut.cpp index 3115d008b4bb..426e0f0b3e95 100644 --- a/ydb/services/ydb/ydb_object_storage_ut.cpp +++ b/ydb/services/ydb/ydb_object_storage_ut.cpp @@ -1,9 +1,9 @@ #include "ydb_common_ut.h" #include -#include -#include -#include +#include +#include +#include using namespace NYdb; @@ -92,9 +92,9 @@ Y_UNIT_TEST_SUITE(YdbS3Internal) { UNIT_ASSERT_VALUES_EQUAL(res.GetContents().RowsCount(), 1); TResultSetParser parser(res.GetContents()); UNIT_ASSERT(parser.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Name").GetOptionalUtf8().GetRef(), "bucket50"); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Path").GetOptionalUtf8().GetRef(), "/home/.bashrc"); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Timestamp").GetOptionalUint64().GetRef(), 10); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Name").GetOptionalUtf8().value(), "bucket50"); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Path").GetOptionalUtf8().value(), "/home/.bashrc"); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("Timestamp").GetOptionalUint64().value(), 10); } } diff --git a/ydb/services/ydb/ydb_olapstore_ut.cpp b/ydb/services/ydb/ydb_olapstore_ut.cpp index d0ae7e72a3dc..e48cc2d82cb6 100644 --- a/ydb/services/ydb/ydb_olapstore_ut.cpp +++ b/ydb/services/ydb/ydb_olapstore_ut.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_register_node_ut.cpp b/ydb/services/ydb/ydb_register_node_ut.cpp index 3b0a9f807385..d686972d88ce 100644 --- a/ydb/services/ydb/ydb_register_node_ut.cpp +++ b/ydb/services/ydb/ydb_register_node_ut.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include @@ -27,12 +27,12 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include diff --git a/ydb/services/ydb/ydb_scripting_ut.cpp b/ydb/services/ydb/ydb_scripting_ut.cpp index e90bf3904381..7344debf7683 100644 --- a/ydb/services/ydb/ydb_scripting_ut.cpp +++ b/ydb/services/ydb/ydb_scripting_ut.cpp @@ -1,8 +1,8 @@ #include "ydb_common_ut.h" #include -#include -#include +#include +#include #include #include @@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(YdbScripting) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT(result.IsSuccess()); - TVector resultSets = result.GetResultSets(); + auto resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); TResultSetParser rsParser(resultSets[0]); @@ -101,7 +101,7 @@ Y_UNIT_TEST_SUITE(YdbScripting) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT(result.IsSuccess()); - TVector resultSets = result.GetResultSets(); + auto resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 3); UNIT_ASSERT_EQUAL(resultSets[0].RowsCount(), 2); diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp index 67b68961b4fc..78c9e2213ead 100644 --- a/ydb/services/ydb/ydb_stats_ut.cpp +++ b/ydb/services/ydb/ydb_stats_ut.cpp @@ -1,8 +1,8 @@ #include "ydb_common_ut.h" -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_table_split_ut.cpp b/ydb/services/ydb/ydb_table_split_ut.cpp index c32e21577fb4..6d883a4ef596 100644 --- a/ydb/services/ydb/ydb_table_split_ut.cpp +++ b/ydb/services/ydb/ydb_table_split_ut.cpp @@ -1,7 +1,7 @@ -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index e09991685bb1..3e63ede0758c 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -1,15 +1,15 @@ #include "ydb_common_ut.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -19,7 +19,7 @@ #include #include -#include +#include #include @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector resultSets = result.GetResultSets(); + auto resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 4); auto columnMeta = resultSets[0].GetColumnsMeta(); @@ -307,7 +307,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector resultSets = result.GetResultSets(); + auto resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); @@ -342,7 +342,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector resultSets = result.GetResultSets(); + std::vector resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); @@ -379,7 +379,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector resultSets = result.GetResultSets(); + std::vector resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); @@ -495,7 +495,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - TVector resultSets = result.GetResultSets(); + std::vector resultSets = result.GetResultSets(); UNIT_ASSERT_VALUES_EQUAL(resultSets.size(), 1); UNIT_ASSERT_VALUES_EQUAL(resultSets[0].ColumnsCount(), 9); UNIT_ASSERT_VALUES_EQUAL(resultSets[0].GetColumnsMeta().size(), 9); @@ -618,7 +618,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector resultSets = result.GetResultSets(); + auto resultSets = result.GetResultSets(); UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); @@ -712,13 +712,13 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { auto status = scheme.ModifyPermissions("Root/Test", NYdb::NScheme::TModifyPermissionsSettings() .AddGrantPermissions( - NYdb::NScheme::TPermissions("pupkin@builtin", TVector{"ydb.tables.modify"}) + NYdb::NScheme::TPermissions("pupkin@builtin", {"ydb.tables.modify"}) ) .AddSetPermissions( - NYdb::NScheme::TPermissions("root@builtin", TVector{"ydb.tables.modify"}) //This permission should be ignored - last set win + NYdb::NScheme::TPermissions("root@builtin", {"ydb.tables.modify"}) //This permission should be ignored - last set win ) .AddSetPermissions( - NYdb::NScheme::TPermissions("root@builtin", TVector{"ydb.tables.read"}) + NYdb::NScheme::TPermissions("root@builtin", {"ydb.tables.read"}) ) ).ExtractValueSync(); UNIT_ASSERT_EQUAL(status.IsTransportError(), false); @@ -844,7 +844,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { auto status = scheme.ModifyPermissions("/Root", NYdb::NScheme::TModifyPermissionsSettings() .AddGrantPermissions( - NYdb::NScheme::TPermissions("test_user@builtin", TVector{"ydb.database.connect"}) + NYdb::NScheme::TPermissions("test_user@builtin", {"ydb.database.connect"}) ) ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); @@ -1663,7 +1663,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TValueBuilder valueFrom; valueFrom.BeginTuple() .AddElement() - .OptionalUint64(Nothing()) + .OptionalUint64(std::nullopt) .EndTuple(); auto settings = TReadTableSettings() @@ -1812,7 +1812,7 @@ R"___(
: Error: Transaction not found: , code: 2015 .AddNullableColumn("Key", EPrimitiveType::Uint32) .AddNullableColumn("Fk", EPrimitiveType::Uint64) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "Fk"}); + tableBuilder.SetPrimaryKeyColumns({"Key", "Fk"}); TCreateTableSettings createTableSettings = TCreateTableSettings() @@ -1962,7 +1962,7 @@ R"___(
: Error: Transaction not found: , code: 2015 .AddNullableColumn("Key", EPrimitiveType::Uint32) .AddNullableColumn("Key2", EPrimitiveType::Uint32) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "Key2"}); + tableBuilder.SetPrimaryKeyColumns({"Key", "Key2"}); TCreateTableSettings createTableSettings = TCreateTableSettings() @@ -2003,10 +2003,10 @@ R"___(
: Error: Transaction not found: , code: 2015 for (const auto& range : describeResult.GetTableDescription().GetKeyRanges()) { TReadTableSettings readTableSettings; if (auto from = range.From()) { - readTableSettings.From(from.GetRef()); + readTableSettings.From(from.value()); } if (auto to = range.To()) { - readTableSettings.To(to.GetRef()); + readTableSettings.To(to.value()); } if (rowLimit) { readTableSettings.RowLimit(1); @@ -2120,7 +2120,7 @@ R"___(
: Error: Transaction not found: , code: 2015 .AddNullableColumn("Key", EPrimitiveType::Uint32) .AddNullableColumn("Key2", EPrimitiveType::Uint32) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector{"Key", "Key2"}); + tableBuilder.SetPrimaryKeyColumns({"Key", "Key2"}); TCreateTableSettings createTableSettings = TCreateTableSettings() @@ -2275,7 +2275,7 @@ R"___(
: Error: Transaction not found: , code: 2015 driver.Stop(true); } - void CheckRetryResult(const TStatus& status, const TVector& resultSets, bool expectSuccess) + void CheckRetryResult(const TStatus& status, const std::vector& resultSets, bool expectSuccess) { if (expectSuccess) { UNIT_ASSERT_C(status.IsSuccess(), status); @@ -2291,7 +2291,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TRetryOperationSettings settings = TRetryOperationSettings()) { size_t retryNumber = 0; - TVector resultSets; + std::vector resultSets; auto operation = [&retryNumber, &resultSets, &retriableStatuses] (TSession session) -> TAsyncStatus { // iterate over all providen statuses and return TStatus to emulate error if (retryNumber < retriableStatuses.size()) { @@ -2322,7 +2322,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TRetryOperationSettings settings = TRetryOperationSettings()) { size_t retryNumber = 0; - TVector resultSets; + std::vector resultSets; auto operation = [&retryNumber, &resultSets, &retriableStatuses] (TSession session) -> TStatus { // iterate over all providen statuses and return TStatus to emulate error if (retryNumber < retriableStatuses.size()) { @@ -2579,7 +2579,7 @@ R"___(
: Error: Transaction not found: , code: 2015 { auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("SomeName", TVector())}); + .AppendAddIndexes({TIndexDescription("SomeName", {})}); auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); @@ -2638,9 +2638,9 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_DOUBLES_EQUAL(meta.Progress, 100, 0.001); UNIT_ASSERT_VALUES_EQUAL(meta.Path, "/Root/Test"); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexName(), "NewIndex"); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns()[0], "Value"); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption->GetIndexName(), "NewIndex"); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption->GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption->GetIndexColumns()[0], "Value"); auto result2 = operationClient.Get(result.GetList()[0].Id()).GetValueSync(); @@ -2649,9 +2649,9 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_DOUBLES_EQUAL(result2.Metadata().Progress, 100, 0.001); UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Path, "/Root/Test"); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexName(), "NewIndex"); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns()[0], "Value"); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption->GetIndexName(), "NewIndex"); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption->GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption->GetIndexColumns()[0], "Value"); { // Cancel already finished operation do nothing @@ -2872,7 +2872,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); if (!returnStats) { - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), false); } else { // Cerr << "\nQUERY: " << query << "\nSTATS:\n" << result.GetStats()->ToString() << Endl; auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); @@ -2902,7 +2902,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); if (!returnStats) { - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), false); } else { // Cerr << "\nQUERY: " << query << "\nSTATS:\n" << result.GetStats()->ToString() << Endl; auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); @@ -2923,7 +2923,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); if (!returnStats) { - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), false); } else { // Cerr << "\nQUERY: " << query << "\nSTATS:\n" << result.GetStats()->ToString() << Endl; auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); @@ -2944,7 +2944,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); if (!returnStats) { - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), false); } else { // Cerr << "\nQUERY: " << query << "\nSTATS:\n" << result.GetStats()->ToString() << Endl; auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); @@ -2965,7 +2965,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); if (!returnStats) { - UNIT_ASSERT_VALUES_EQUAL(result.GetStats().Defined(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStats().has_value(), false); } else { // Cerr << "\nQUERY: " << query << "\nSTATS:\n" << result.GetStats()->ToString() << Endl; auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); @@ -3427,7 +3427,7 @@ R"___(
: Error: Transaction not found: , code: 2015 class TPrintableIssues { public: - TPrintableIssues(const NYql::TIssues& issues) + TPrintableIssues(const NYdb::NIssue::TIssues& issues) : Issues(issues) { } @@ -3437,7 +3437,7 @@ R"___(
: Error: Transaction not found: , code: 2015 } private: - const NYql::TIssues& Issues; + const NYdb::NIssue::TIssues& Issues; }; } @@ -3636,7 +3636,7 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_EQUAL(families.size(), 2); UNIT_ASSERT_VALUES_EQUAL(families[0].GetName(), "default"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetName(), "alt"); - UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression(), EColumnFamilyCompression::None); + UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression().value(), EColumnFamilyCompression::None); } { @@ -3666,7 +3666,7 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_EQUAL(families.size(), 2); UNIT_ASSERT_VALUES_EQUAL(families[0].GetName(), "default"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetName(), "alt"); - UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression().value(), EColumnFamilyCompression::LZ4); } for (int tableIdx = 1; tableIdx <= 4; ++tableIdx) { @@ -3792,7 +3792,7 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_VALUES_EQUAL(families[0].GetName(), "default"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetName(), "alt"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetData(), "hdd"); - UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression().value(), EColumnFamilyCompression::LZ4); } } @@ -3854,14 +3854,14 @@ R"___(
: Error: Transaction not found: , code: 2015 UNIT_ASSERT_VALUES_EQUAL(columns[1].Family, "alt"); const auto& settings = res.GetTableDescription().GetStorageSettings(); UNIT_ASSERT_VALUES_EQUAL(settings.GetExternal(), "hdd"); - UNIT_ASSERT_VALUES_EQUAL(settings.GetStoreExternalBlobs(), true); + UNIT_ASSERT_VALUES_EQUAL(settings.GetStoreExternalBlobs().value(), true); const auto& families = res.GetTableDescription().GetColumnFamilies(); UNIT_ASSERT_EQUAL(families.size(), 2); UNIT_ASSERT_VALUES_EQUAL(families[0].GetName(), "default"); UNIT_ASSERT_VALUES_EQUAL(families[0].GetData(), "ssd"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetName(), "alt"); UNIT_ASSERT_VALUES_EQUAL(families[1].GetData(), "hdd"); - UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression(), EColumnFamilyCompression::LZ4); + UNIT_ASSERT_VALUES_EQUAL(families[1].GetCompression().value(), EColumnFamilyCompression::LZ4); } } @@ -4121,10 +4121,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } @@ -4199,10 +4199,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 3); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } @@ -4242,10 +4242,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 100); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 2); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 50); @@ -4266,8 +4266,8 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 50); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 100); @@ -4286,8 +4286,8 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); } { auto settings = NYdb::NTable::TAlterTableSettings() @@ -4303,8 +4303,8 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), true); } } @@ -4339,10 +4339,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), true); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); } { @@ -4359,10 +4359,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), true); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } @@ -4380,8 +4380,8 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); } } @@ -4411,10 +4411,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); } } @@ -4447,10 +4447,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), true); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } @@ -4485,10 +4485,10 @@ R"___(
: Error: Transaction not found: , code: 2015 .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().value(), true); + UNIT_ASSERT(partSettings.GetPartitioningBySize().has_value()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().value(), false); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); } } @@ -4521,8 +4521,8 @@ R"___(
: Error: Transaction not found: , code: 2015 TDescribeTableResult describeResult = session.DescribeTable(tableName) .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), true); + UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().has_value()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().value(), true); } { auto settings = NYdb::NTable::TAlterTableSettings() @@ -4535,8 +4535,8 @@ R"___(
: Error: Transaction not found: , code: 2015 TDescribeTableResult describeResult = session.DescribeTable(tableName) .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), false); + UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().has_value()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().value(), false); } } @@ -4568,7 +4568,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TDescribeTableResult describeResult = session.DescribeTable(tableName) .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().has_value()); UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() == TReadReplicasSettings::EMode::AnyAz); UNIT_ASSERT_VALUES_EQUAL( @@ -4586,7 +4586,7 @@ R"___(
: Error: Transaction not found: , code: 2015 TDescribeTableResult describeResult = session.DescribeTable(tableName) .GetValueSync(); UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().has_value()); UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() == TReadReplicasSettings::EMode::PerAz); UNIT_ASSERT_VALUES_EQUAL( diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index edcaac71d532..3b17744f0a4d 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -23,19 +23,18 @@ #include #include -#include +#include #include #include -#include -#include +#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include @@ -868,7 +867,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { auto families = desc.GetColumnFamilies(); UNIT_ASSERT_VALUES_EQUAL(families.size(), 1u); auto family = families.at(0); - UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), true); + UNIT_ASSERT_VALUES_EQUAL(*family.GetKeepInMemory(), true); } { @@ -888,7 +887,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { auto family = families.at(0); // Note: server cannot currently distinguish between implicitly // unset and explicitly disabled, so it returns the former. - UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), Nothing()); + UNIT_ASSERT(!family.GetKeepInMemory()); } { @@ -906,7 +905,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { auto families = desc.GetColumnFamilies(); UNIT_ASSERT_VALUES_EQUAL(families.size(), 1u); auto family = families.at(0); - UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), true); + UNIT_ASSERT_VALUES_EQUAL(*family.GetKeepInMemory(), true); } } } @@ -928,7 +927,7 @@ static TString CreateSession(std::shared_ptr channel) { return result.session_id(); } -void IncorrectConnectionStringPending(const TString& incorrectLocation) { +void IncorrectConnectionStringPending(const std::string& incorrectLocation) { auto connection = NYdb::TDriver(incorrectLocation); auto client = NYdb::NTable::TTableClient(connection); auto session = client.CreateSession().ExtractValueSync().GetSession(); @@ -942,7 +941,7 @@ Y_UNIT_TEST_SUITE(GrpcConnectionStringParserTest) { bool done = false; { - TString location = TStringBuilder() << "localhost:" << grpc; + std::string location = TStringBuilder() << "localhost:" << grpc; // by default, location won't have database path auto connection = NYdb::TDriver(location); @@ -1307,9 +1306,9 @@ Y_UNIT_TEST_SUITE(TGRpcYdbTest) { UNIT_ASSERT(status.ok()); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - UNIT_ASSERT(issues.ToString().Contains("invalid or unset index type")); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(deferred.issues(), issues); + UNIT_ASSERT(issues.ToString().contains("invalid or unset index type")); } } @@ -4537,7 +4536,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { auto parser = TValueParser(val); parser.OpenTuple(); UNIT_ASSERT(parser.TryNextElement()); - return parser.GetOptionalUint64().GetRef(); + return parser.GetOptionalUint64().value(); }; int n = 0; diff --git a/ydb/services/ymq/ya.make b/ydb/services/ymq/ya.make index 4d9c71fd367b..208d938a912f 100644 --- a/ydb/services/ymq/ya.make +++ b/ydb/services/ymq/ya.make @@ -14,8 +14,8 @@ PEERDIR( ydb/core/mind ydb/public/api/grpc ydb/public/api/grpc/draft - ydb/public/lib/operation_id - ydb/public/sdk/cpp/client/resources + ydb/public/sdk/cpp/src/library/operation_id + ydb/public/sdk/cpp/src/client/resources ydb/services/lib/actors ydb/services/lib/sharding ydb/services/persqueue_v1 diff --git a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_internal_ut.cpp b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_internal_ut.cpp index eda4d4767ac9..7c1b94ee9619 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_internal_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_internal_ut.cpp @@ -1524,7 +1524,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_LE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL more than current time"); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -1537,7 +1537,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_LE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL more than current time"); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -1550,7 +1550,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); while (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_LE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL more than current time"); } } @@ -1642,7 +1642,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); + UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); } @@ -1657,7 +1657,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); + UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); } @@ -1671,7 +1671,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); while (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); } } @@ -1742,7 +1742,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -1755,7 +1755,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -1768,7 +1768,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); while (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); } } @@ -1830,7 +1830,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); + UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); } @@ -1846,7 +1846,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 2); while (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); + UNIT_ASSERT_VALUES_EQUAL_C(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), false, *parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp()); } } @@ -1907,7 +1907,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -1921,7 +1921,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 3); while (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); } } @@ -1931,7 +1931,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(result.Defined(), true); NYdb::TResultSetParser parser(*result); while (parser.TryNextRow()) { - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_GE_C(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), TInstant::Now(), "TTL less than current time"); } } @@ -2001,7 +2001,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { NYdb::TResultSetParser parser(*result); if (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(), queryId); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(RESULT_SETS_EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(RESULT_SETS_EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(RESULT_SETS_EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), deadline); if (parser.TryNextRow()) { UNIT_ASSERT("Row count more than 1"); @@ -2016,7 +2016,7 @@ Y_UNIT_TEST_SUITE(TYdbControlPlaneStoragePipeline) { UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 5); while (parser.TryNextRow()) { UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString(), resId); - UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().Defined(), true); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp().has_value(), true); UNIT_ASSERT_VALUES_EQUAL(*parser.ColumnParser(EXPIRE_AT_COLUMN_NAME).GetOptionalTimestamp(), deadline); } } diff --git a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_queries_ut.cpp b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_queries_ut.cpp index 9aa3957114b6..4f5539e612f4 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_queries_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_queries_ut.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include diff --git a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_quotas_ut.cpp b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_quotas_ut.cpp index da6603bdc586..585b4e33a99f 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_quotas_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/ydb_control_plane_storage_quotas_ut.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h index 7620c35a1578..684147f48f06 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h +++ b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/tests/fq/pq_async_io/mock_pq_gateway.cpp b/ydb/tests/fq/pq_async_io/mock_pq_gateway.cpp index ff4dba4de4f3..c3d9f24e6cd0 100644 --- a/ydb/tests/fq/pq_async_io/mock_pq_gateway.cpp +++ b/ydb/tests/fq/pq_async_io/mock_pq_gateway.cpp @@ -28,23 +28,23 @@ class TMockTopicReadSession : public NYdb::NTopic::IReadSession { }, ThreadPool); } - TVector GetEvents(bool block, TMaybe maxEventsCount, size_t /*maxByteSize*/) override { - TVector res; - for (auto event = Queue->Pop(block); !event.Empty() && res.size() <= maxEventsCount.GetOrElse(std::numeric_limits::max()); event = Queue->Pop(/*block=*/ false)) { + std::vector GetEvents(bool block, std::optional maxEventsCount, size_t /*maxByteSize*/) override { + std::vector res; + for (auto event = Queue->Pop(block); event.has_value() && res.size() <= maxEventsCount.value_or(std::numeric_limits::max()); event = Queue->Pop(/*block=*/ false)) { res.push_back(*event); } return res; } - TVector GetEvents(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { + std::vector GetEvents(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { return GetEvents(settings.Block_, settings.MaxEventsCount_, settings.MaxByteSize_); } - TMaybe GetEvent(bool block, size_t /*maxByteSize*/) override { + std::optional GetEvent(bool block, size_t /*maxByteSize*/) override { return Queue->Pop(block); } - TMaybe GetEvent(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { + std::optional GetEvent(const NYdb::NTopic::TReadSessionGetEventSettings& settings) override { return GetEvent(settings.Block_, settings.MaxByteSize_); } @@ -56,7 +56,7 @@ class TMockTopicReadSession : public NYdb::NTopic::IReadSession { } NYdb::NTopic::TReaderCounters::TPtr GetCounters() const override {return nullptr;} - TString GetSessionId() const override {return "fake";} + std::string GetSessionId() const override {return "fake";} private: TThreadPool ThreadPool; NYdb::NTopic::TPartitionSession::TPtr Session; @@ -88,7 +88,7 @@ class TMockPqGateway : public IMockPqGateway { std::shared_ptr CreateReadSession(const NYdb::NTopic::TReadSessionSettings& settings) override { Y_ENSURE(!settings.Topics_.empty()); - TString topic = settings.Topics_.front().Path_; + auto topic = TString{settings.Topics_.front().Path_}; Self->Runtime.Send(new NActors::IEventHandle(Self->Notifier, NActors::TActorId(), new NYql::NDq::TEvMockPqEvents::TEvCreateSession())); return std::make_shared(MakeIntrusive(), Self->GetEventQueue(topic)); } diff --git a/ydb/tests/fq/pq_async_io/mock_pq_gateway.h b/ydb/tests/fq/pq_async_io/mock_pq_gateway.h index b6a05e170b7e..13b74271ac47 100644 --- a/ydb/tests/fq/pq_async_io/mock_pq_gateway.h +++ b/ydb/tests/fq/pq_async_io/mock_pq_gateway.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/tests/fq/pq_async_io/ut/ya.make b/ydb/tests/fq/pq_async_io/ut/ya.make index 4adeace2f464..fa4e04e20807 100644 --- a/ydb/tests/fq/pq_async_io/ut/ya.make +++ b/ydb/tests/fq/pq_async_io/ut/ya.make @@ -19,8 +19,8 @@ PEERDIR( ydb/library/yql/providers/pq/gateway/native yql/essentials/public/udf/service/exception_policy yql/essentials/sql - ydb/public/sdk/cpp/client/ydb_datastreams - ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/src/client/datastreams + ydb/public/sdk/cpp/src/client/persqueue_public ydb/tests/fq/pq_async_io ) diff --git a/ydb/tests/fq/pq_async_io/ut_helpers.cpp b/ydb/tests/fq/pq_async_io/ut_helpers.cpp index e011029479ca..2a8311e78152 100644 --- a/ydb/tests/fq/pq_async_io/ut_helpers.cpp +++ b/ydb/tests/fq/pq_async_io/ut_helpers.cpp @@ -173,7 +173,7 @@ void PQWrite( NYdb::TDriverConfig cfg; cfg.SetEndpoint(endpoint); cfg.SetDatabase(GetDefaultPqDatabase()); - cfg.SetLog(CreateLogBackend("cerr")); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(cfg); NYdb::NTopic::TTopicClient client(driver); NYdb::NTopic::TWriteSessionSettings sessionSettings; @@ -200,12 +200,12 @@ std::vector PQReadUntil( NYdb::TDriverConfig cfg; cfg.SetEndpoint(endpoint); cfg.SetDatabase(GetDefaultPqDatabase()); - cfg.SetLog(CreateLogBackend("cerr")); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(cfg); NYdb::NTopic::TTopicClient client(driver); NYdb::NTopic::TReadSessionSettings sessionSettings; sessionSettings - .AppendTopics(topic) + .AppendTopics(std::string{topic}) .ConsumerName(DefaultPqConsumer); auto promise = NThreading::NewPromise(); @@ -233,7 +233,7 @@ void PQCreateStream(const TString& streamName) NYdb::TDriverConfig cfg; cfg.SetEndpoint(GetDefaultPqEndpoint()); cfg.SetDatabase(GetDefaultPqDatabase()); - cfg.SetLog(CreateLogBackend("cerr")); + cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(cfg); NYdb::NDataStreams::V1::TDataStreamsClient client = NYdb::NDataStreams::V1::TDataStreamsClient( diff --git a/ydb/tests/fq/pq_async_io/ut_helpers.h b/ydb/tests/fq/pq_async_io/ut_helpers.h index b685412b5d01..5d39b6024b87 100644 --- a/ydb/tests/fq/pq_async_io/ut_helpers.h +++ b/ydb/tests/fq/pq_async_io/ut_helpers.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include @@ -32,7 +32,7 @@ TString GetDefaultPqDatabase(); struct TPqIoTestFixture : public NUnitTest::TBaseFixture { std::unique_ptr CaSetup = std::make_unique(); - NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr"))); + NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig().SetLog(std::unique_ptr(CreateLogBackend("cerr").Release()))); TPqIoTestFixture(); ~TPqIoTestFixture(); diff --git a/ydb/tests/fq/pq_async_io/ya.make b/ydb/tests/fq/pq_async_io/ya.make index 75efb71a2b4f..96e779b2539a 100644 --- a/ydb/tests/fq/pq_async_io/ya.make +++ b/ydb/tests/fq/pq_async_io/ya.make @@ -9,7 +9,7 @@ PEERDIR( yql/essentials/minikql/computation/llvm14 ydb/library/yql/providers/common/ut_helpers ydb/library/yql/providers/pq/gateway/dummy - ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/src/client/topic ) YQL_LAST_ABI_VERSION() diff --git a/ydb/tests/functional/backup/backup_ut.cpp b/ydb/tests/functional/backup/backup_ut.cpp index 23f4c17906dd..2dd54926f72e 100644 --- a/ydb/tests/functional/backup/backup_ut.cpp +++ b/ydb/tests/functional/backup/backup_ut.cpp @@ -3,12 +3,12 @@ #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -149,7 +149,7 @@ Y_UNIT_TEST_SUITE(Backup) auto res = s.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - auto yson = NYdb::FormatResultSetYson(res.GetResultSet(0)); + TString yson = NYdb::FormatResultSetYson(res.GetResultSet(0)); const TString& expected = "[[[1u];[\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea01\"];\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea02\"]]"; CompareYson(expected, yson); diff --git a/ydb/tests/functional/backup/s3_path_style/s3_path_style_backup_ut.cpp b/ydb/tests/functional/backup/s3_path_style/s3_path_style_backup_ut.cpp index 22a158f10904..d375318fe583 100644 --- a/ydb/tests/functional/backup/s3_path_style/s3_path_style_backup_ut.cpp +++ b/ydb/tests/functional/backup/s3_path_style/s3_path_style_backup_ut.cpp @@ -1,8 +1,8 @@ -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include diff --git a/ydb/tests/functional/backup/s3_path_style/ya.make b/ydb/tests/functional/backup/s3_path_style/ya.make index c0ad2c2130f9..d5eb293b81fb 100644 --- a/ydb/tests/functional/backup/s3_path_style/ya.make +++ b/ydb/tests/functional/backup/s3_path_style/ya.make @@ -7,10 +7,10 @@ ENV(YDB_ERASURE=block_4-2) PEERDIR( ydb/library/testlib/s3_recipe_helper - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/draft ) SRCS( diff --git a/ydb/tests/functional/backup/ya.make b/ydb/tests/functional/backup/ya.make index 7312186dfd0f..9b833e89ee96 100644 --- a/ydb/tests/functional/backup/ya.make +++ b/ydb/tests/functional/backup/ya.make @@ -8,10 +8,10 @@ PEERDIR( library/cpp/threading/local_executor library/cpp/yson ydb/library/testlib/s3_recipe_helper - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/export + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/operation + ydb/public/sdk/cpp/src/client/draft ydb/public/lib/yson_value ) diff --git a/ydb/tests/functional/kqp/kqp_indexes/main.cpp b/ydb/tests/functional/kqp/kqp_indexes/main.cpp index 0a8e0e4cb165..ba719b909363 100644 --- a/ydb/tests/functional/kqp/kqp_indexes/main.cpp +++ b/ydb/tests/functional/kqp/kqp_indexes/main.cpp @@ -1,9 +1,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include diff --git a/ydb/tests/functional/kqp/kqp_indexes/ya.make b/ydb/tests/functional/kqp/kqp_indexes/ya.make index 2ea2df0c7cd3..a67766feecfe 100644 --- a/ydb/tests/functional/kqp/kqp_indexes/ya.make +++ b/ydb/tests/functional/kqp/kqp_indexes/ya.make @@ -6,8 +6,8 @@ ENV(YDB_ERASURE=block_4-2) PEERDIR( library/cpp/threading/local_executor - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/draft ) SRCS( diff --git a/ydb/tests/functional/kqp/kqp_query_session/main.cpp b/ydb/tests/functional/kqp/kqp_query_session/main.cpp index ca647fe8b1b6..2d53f718a204 100644 --- a/ydb/tests/functional/kqp/kqp_query_session/main.cpp +++ b/ydb/tests/functional/kqp/kqp_query_session/main.cpp @@ -2,12 +2,12 @@ #include #include -#include -#include -#include +#include +#include +#include #include -#include +#include #include diff --git a/ydb/tests/functional/kqp/kqp_query_session/ya.make b/ydb/tests/functional/kqp/kqp_query_session/ya.make index 18435e38c5d3..792a744c517a 100644 --- a/ydb/tests/functional/kqp/kqp_query_session/ya.make +++ b/ydb/tests/functional/kqp/kqp_query_session/ya.make @@ -9,8 +9,8 @@ ENV(USE_YDB_TRUNK_RECIPE_TOOLS=true) PEERDIR( library/cpp/threading/local_executor ydb/public/lib/ut_helpers - ydb/public/sdk/cpp/client/ydb_discovery - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/discovery + ydb/public/sdk/cpp/src/client/draft ) SRCS( diff --git a/ydb/tests/functional/kqp/kqp_query_svc/main.cpp b/ydb/tests/functional/kqp/kqp_query_svc/main.cpp index b6ca0dcd800d..b5462c3a5f7c 100644 --- a/ydb/tests/functional/kqp/kqp_query_svc/main.cpp +++ b/ydb/tests/functional/kqp/kqp_query_svc/main.cpp @@ -2,15 +2,15 @@ #include #include -#include -#include +#include +#include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include using namespace NYdbGrpc; using namespace NYdb; diff --git a/ydb/tests/functional/kqp/kqp_query_svc/ya.make b/ydb/tests/functional/kqp/kqp_query_svc/ya.make index 564d2ca6619c..8b1b9aa5e173 100644 --- a/ydb/tests/functional/kqp/kqp_query_svc/ya.make +++ b/ydb/tests/functional/kqp/kqp_query_svc/ya.make @@ -7,9 +7,9 @@ ENV(YDB_ERASURE=block_4-2) PEERDIR( library/cpp/threading/local_executor ydb/public/lib/ut_helpers - ydb/public/sdk/cpp/client/ydb_discovery - ydb/public/sdk/cpp/client/ydb_query - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/discovery + ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/draft ) SRCS( diff --git a/ydb/tests/functional/replication/main.cpp b/ydb/tests/functional/replication/main.cpp index f6dda69bbbb9..4783743344dc 100644 --- a/ydb/tests/functional/replication/main.cpp +++ b/ydb/tests/functional/replication/main.cpp @@ -1,10 +1,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/ydb/tests/functional/replication/ya.make b/ydb/tests/functional/replication/ya.make index 86c0b3963b57..8b9f1c960948 100644 --- a/ydb/tests/functional/replication/ya.make +++ b/ydb/tests/functional/replication/ya.make @@ -6,9 +6,9 @@ ENV(YDB_ERASURE=block_4-2) PEERDIR( library/cpp/threading/local_executor - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_proto - ydb/public/sdk/cpp/client/draft + ydb/public/sdk/cpp/src/client/table + ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/draft ) SRCS( diff --git a/ydb/tests/functional/sdk/cpp/sdk_credprovider/dummy_provider_ut.cpp b/ydb/tests/functional/sdk/cpp/sdk_credprovider/dummy_provider_ut.cpp index fd93c157d057..787ad50a44f5 100644 --- a/ydb/tests/functional/sdk/cpp/sdk_credprovider/dummy_provider_ut.cpp +++ b/ydb/tests/functional/sdk/cpp/sdk_credprovider/dummy_provider_ut.cpp @@ -3,15 +3,15 @@ #include -#include -#include +#include +#include #define INCLUDE_YDB_INTERNAL_H /// !!!! JUST FOR UT, DO NOT COPY-PASTE !!! /// -#include -#include -#include +#include +#include +#include #undef INCLUDE_YDB_INTERNAL_H using namespace NYdb; @@ -22,7 +22,7 @@ class TExampleDummyProviderFactory : public ICredentialsProviderFactory { class TExampleDummyProvider : public ICredentialsProvider { private: TPeriodicCb CreatePingPongTask(std::weak_ptr facility) { - auto periodicCb = [this, facility](NYql::TIssues&&, EStatus status) { + auto periodicCb = [this, facility](NYdb::NIssue::TIssues&&, EStatus status) { if (status != EStatus::SUCCESS) { return false; } @@ -66,7 +66,7 @@ class TExampleDummyProviderFactory : public ICredentialsProviderFactory { strong->AddPeriodicTask(CreatePingPongTask(facility), TDuration::Seconds(1)); } - TString GetAuthInfo() const override { + std::string GetAuthInfo() const override { return ""; } diff --git a/ydb/tests/functional/sdk/cpp/sdk_credprovider/ya.make b/ydb/tests/functional/sdk/cpp/sdk_credprovider/ya.make index 2a52ee3c1fad..c2a963b92d4f 100644 --- a/ydb/tests/functional/sdk/cpp/sdk_credprovider/ya.make +++ b/ydb/tests/functional/sdk/cpp/sdk_credprovider/ya.make @@ -1,12 +1,16 @@ UNITTEST() +ADDINCL( + ydb/public/sdk/cpp +) + SRCS( dummy_provider_ut.cpp ) PEERDIR( - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ydb/public/api/grpc ) diff --git a/ydb/tests/tools/idx_test/main.cpp b/ydb/tests/tools/idx_test/main.cpp index 8d7f28521be6..d83813de52e8 100644 --- a/ydb/tests/tools/idx_test/main.cpp +++ b/ydb/tests/tools/idx_test/main.cpp @@ -4,10 +4,11 @@ #include #include +#include #include -#include -#include +#include +#include using namespace NYdb; using namespace NYdb::NTable; @@ -15,28 +16,6 @@ using namespace NYdb::NScheme; using namespace NIdxTest; using namespace NLastGetopt; -class TYdbErrorException : public yexception { - mutable TString StrBuf_; -public: - TYdbErrorException(const NYdb::TStatus& status) - : Status(status) {} - - NYdb::TStatus Status; - const char* what() const noexcept override { - TStringStream ss; - ss << "Status: " << Status.GetStatus() << "\n"; - ss << "Issues: " << Status.GetIssues().ToString() << "\n"; - StrBuf_ = ss.Str(); - return StrBuf_.c_str(); - } -}; - -static void ThrowOnError(const TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status) << status; - } -} - static TMap Cmds { {"upsert", IWorkLoader::LC_UPSERT}, {"insert", IWorkLoader::LC_INSERT}, @@ -75,7 +54,7 @@ static ui32 ParseCmd(int argc, char** argv) { static void ExecuteDDL(TTableClient client, const TString& sql) { Cerr << "Try to execute DDL query: " << sql << Endl; - ThrowOnError(client.RetryOperationSync([sql](TSession session) { + NStatusHelpers::ThrowOnError(client.RetryOperationSync([sql](TSession session) { return session.ExecuteSchemeQuery(sql).GetValueSync(); })); } @@ -91,7 +70,7 @@ static void CreatePath(TSchemeClient scheme, const TString& database, const TStr if (pos == prefix.npos) break; curPath += prefix.substr(prevPos, pos - prevPos); - ThrowOnError(scheme.MakeDirectory(curPath, TMakeDirectorySettings() + NStatusHelpers::ThrowOnError(scheme.MakeDirectory(curPath, TMakeDirectorySettings() .ClientTimeout(TDuration::Seconds(10))).GetValueSync()); } } @@ -108,7 +87,7 @@ static void ExecuteCreateUniformTable(TTableClient client, const TString& tableP auto desc = builder.Build(); - ThrowOnError(client.RetryOperationSync([tablePath, desc, shardsCount](TSession session) { + NStatusHelpers::ThrowOnError(client.RetryOperationSync([tablePath, desc, shardsCount](TSession session) { auto d = desc; return session.CreateTable(tablePath, std::move(d), @@ -243,9 +222,9 @@ int main(int argc, char** argv) { driver.Stop(true); Cerr << "Test failed: " << ex.what() << Endl; Cerr << "..." << Endl; - auto ydbEx = dynamic_cast(&ex); + auto ydbEx = dynamic_cast(&ex); if (ydbEx) { - Cerr << "Ydb error: " << ydbEx->Status.GetStatus() << " " << ydbEx->Status.GetIssues().ToString() << Endl; + Cerr << *ydbEx << Endl; } return 1; } diff --git a/ydb/tests/tools/idx_test/ya.make b/ydb/tests/tools/idx_test/ya.make index b3dbed04354f..6d0370cb4fad 100644 --- a/ydb/tests/tools/idx_test/ya.make +++ b/ydb/tests/tools/idx_test/ya.make @@ -9,8 +9,8 @@ PEERDIR( library/cpp/resource ydb/public/lib/idx_test ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_scheme - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/scheme + ydb/public/sdk/cpp/src/client/table ) RESOURCE( diff --git a/ydb/tests/tools/kqprun/src/ydb_setup.cpp b/ydb/tests/tools/kqprun/src/ydb_setup.cpp index 856ab445ed7d..2ff276fdd8f7 100644 --- a/ydb/tests/tools/kqprun/src/ydb_setup.cpp +++ b/ydb/tests/tools/kqprun/src/ydb_setup.cpp @@ -25,7 +25,7 @@ class TStaticCredentialsProvider : public NYdb::ICredentialsProvider { : YqlToken_(yqlToken) {} - TString GetAuthInfo() const override { + std::string GetAuthInfo() const override { return YqlToken_; } @@ -34,7 +34,7 @@ class TStaticCredentialsProvider : public NYdb::ICredentialsProvider { } private: - TString YqlToken_; + std::string YqlToken_; }; class TStaticCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { diff --git a/ydb/tests/tools/kqprun/src/ydb_setup.h b/ydb/tests/tools/kqprun/src/ydb_setup.h index 393873b511bf..fcb4249aa10c 100644 --- a/ydb/tests/tools/kqprun/src/ydb_setup.h +++ b/ydb/tests/tools/kqprun/src/ydb_setup.h @@ -3,7 +3,7 @@ #include "common.h" #include "actors.h" -#include +#include namespace NKqpRun { diff --git a/ydb/tests/tools/pq_read/main.cpp b/ydb/tests/tools/pq_read/main.cpp index cfcf4f5e3b6c..4283362646e6 100644 --- a/ydb/tests/tools/pq_read/main.cpp +++ b/ydb/tests/tools/pq_read/main.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -52,7 +52,7 @@ int main(int argc, const char* argv[]) { .SetNetworkThreadsNum(2) .SetEndpoint(opts.Endpoint) .SetDatabase(opts.Database) - .SetLog(CreateLogBackend("cerr")); + .SetLog(std::unique_ptr(CreateLogBackend("cerr").Release())); NYdb::TDriver driver(driverConfig); NYdb::NPersQueue::TPersQueueClient persqueueClient(driver); @@ -61,7 +61,7 @@ int main(int argc, const char* argv[]) { settings .DisableClusterDiscovery(opts.DisableClusterDiscovery) .ConsumerName(opts.ConsumerName) - .AppendTopics(opts.TopicPath); + .AppendTopics(std::string{opts.TopicPath}); std::shared_ptr readSession; size_t messagesReceived = 0; diff --git a/ydb/tests/tools/pq_read/ya.make b/ydb/tests/tools/pq_read/ya.make index 3fbeb140c4e3..9a258a1fd99a 100644 --- a/ydb/tests/tools/pq_read/ya.make +++ b/ydb/tests/tools/pq_read/ya.make @@ -8,7 +8,7 @@ PEERDIR( library/cpp/colorizer library/cpp/getopt library/cpp/threading/future - ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/src/client/persqueue_public ) END() diff --git a/ydb/tools/query_replay/common_deps.inc b/ydb/tools/query_replay/common_deps.inc index 63fe41f127d1..80bf37166ac6 100644 --- a/ydb/tools/query_replay/common_deps.inc +++ b/ydb/tools/query_replay/common_deps.inc @@ -10,7 +10,7 @@ SET(YDB_REPLAY_PEERDIRS ydb/library/actors/core ydb/library/actors/interconnect library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/grpc/server library/cpp/regex/pcre ydb/core/actorlib_impl @@ -53,7 +53,7 @@ SET(YDB_REPLAY_PEERDIRS ydb/public/api/protos ydb/public/lib/base ydb/public/lib/deprecated/kicli - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ydb/services/cms ydb/services/datastreams ydb/services/discovery diff --git a/ydb/tools/query_replay/query_proccessor.cpp b/ydb/tools/query_replay/query_proccessor.cpp index 7d3d272ec171..3258d60923b7 100644 --- a/ydb/tools/query_replay/query_proccessor.cpp +++ b/ydb/tools/query_replay/query_proccessor.cpp @@ -97,7 +97,7 @@ class TQueryProcessorActor: public TActorBootstrapped { if (col.Name == "_logfeller_timestamp") continue; - TString value = parser.ColumnParser(col.Name).GetOptionalString().GetRef(); + TString value = parser.ColumnParser(col.Name).GetOptionalString().value(); json.InsertValue(col.Name, NJson::TJsonValue(std::move(value))); } diff --git a/ydb/tools/query_replay/query_replay.cpp b/ydb/tools/query_replay/query_replay.cpp index 89d586482661..72e1e0bf40f2 100644 --- a/ydb/tools/query_replay/query_replay.cpp +++ b/ydb/tools/query_replay/query_replay.cpp @@ -1,8 +1,8 @@ #include "query_replay.h" -#include -#include -#include +#include +#include +#include #include #include @@ -187,11 +187,11 @@ class TQueryReplayActorRunner: public TActorBootstrapped -#include -#include +#include +#include +#include #include #include diff --git a/ydb/tools/query_replay_yt/common_deps.inc b/ydb/tools/query_replay_yt/common_deps.inc index 8d602b2b487a..4d1502903b04 100644 --- a/ydb/tools/query_replay_yt/common_deps.inc +++ b/ydb/tools/query_replay_yt/common_deps.inc @@ -9,7 +9,7 @@ SET(YDB_REPLAY_PEERDIRS ydb/library/actors/core ydb/library/actors/interconnect library/cpp/getopt - ydb/library/grpc/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/library/grpc/server library/cpp/regex/pcre ydb/core/actorlib_impl @@ -52,7 +52,7 @@ SET(YDB_REPLAY_PEERDIRS ydb/public/api/protos ydb/public/lib/base ydb/public/lib/deprecated/kicli - ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/src/client/table ydb/services/cms ydb/services/datastreams ydb/services/discovery