[Chore](build) use file(GLOB_RECURSE xxx CONFIGURE_DEPENDS) to replac…

…e set cpp (apache#20461)

use file(GLOB_RECURSE xxx CONFIGURE_DEPENDS) to replace set cpp

.clang-format-ignore

@@ -9,7 +9,3 @@ be/src/util/sse2neon.h
 be/src/util/mustache/mustache.h
 be/src/util/mustache/mustache.cc
 be/src/util/utf8_check.cpp
-be/src/util/cityhash102/city.h
-be/src/util/cityhash102/city.cc
-be/src/util/cityhash102/citycrc.h
-be/src/util/cityhash102/config.h

.licenserc.yaml

@@ -66,7 +66,6 @@ header:
     - "be/src/util/sse2neo.h"
     - "be/src/util/sse2neon.h"
     - "be/src/util/utf8_check.cpp"
-    - "be/src/util/cityhash102"
     - "be/src/pch/*"
     - "build-support/run_clang_format.py"
     - "regression-test/data"

be/src/agent/CMakeLists.txt

@@ -21,14 +21,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/agent")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/agent")
 
-set(AGENT_SOURCES
-    agent_server.cpp
-    heartbeat_server.cpp
-    task_worker_pool.cpp
-    utils.cpp
-    topic_subscriber.cpp
-    user_resource_listener.cpp
-)
+file(GLOB_RECURSE AGENT_SOURCES CONFIGURE_DEPENDS *.cpp)
 
 if (OS_MACOSX)
     list(REMOVE_ITEM AGENT_SOURCES user_resource_listener.cpp)

be/src/common/CMakeLists.txt

@@ -18,15 +18,8 @@
 # where to put generated libraries
 set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/common")
 
-add_library(Common STATIC
-  daemon.cpp
-  status.cpp
-  resource_tls.cpp
-  logconfig.cpp
-  config.cpp
-  exception.cpp
-  version_internal.cpp
-)
+file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp)
+add_library(Common STATIC ${SRC_FILES})
 
 pch_reuse(Common)
 

be/src/exec/CMakeLists.txt

@@ -21,39 +21,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exec")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exec")
 
-set(EXEC_FILES
-    data_sink.cpp
-    decompressor.cpp
-    exec_node.cpp
-    text_converter.cpp
-    olap_common.cpp
-    tablet_info.cpp
-    es/es_scan_reader.cpp
-    es/es_scroll_query.cpp
-    es/es_scroll_parser.cpp
-    schema_scanner.cpp
-    schema_scanner/schema_tables_scanner.cpp
-    schema_scanner/schema_dummy_scanner.cpp
-    schema_scanner/schema_schemata_scanner.cpp
-    schema_scanner/schema_variables_scanner.cpp
-    schema_scanner/schema_columns_scanner.cpp
-    schema_scanner/schema_charsets_scanner.cpp
-    schema_scanner/schema_collations_scanner.cpp
-    schema_scanner/schema_helper.cpp
-    schema_scanner/schema_views_scanner.cpp
-    schema_scanner/schema_statistics_scanner.cpp
-    schema_scanner/schema_table_privileges_scanner.cpp
-    schema_scanner/schema_schema_privileges_scanner.cpp
-    schema_scanner/schema_user_privileges_scanner.cpp
-    schema_scanner/schema_files_scanner.cpp
-    schema_scanner/schema_partitions_scanner.cpp
-    schema_scanner/schema_rowsets_scanner.cpp
-    scan_node.cpp
-    odbc_connector.cpp
-    table_connector.cpp
-    schema_scanner.cpp
-    rowid_fetcher.cpp
-)
+file(GLOB_RECURSE EXEC_FILES CONFIGURE_DEPENDS *.cpp)
 
 if (WITH_LZO)
     set(EXEC_FILES ${EXEC_FILES}

be/src/exprs/CMakeLists.txt

@@ -21,14 +21,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exprs")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exprs")
 
-add_library(Exprs
-  block_bloom_filter_avx_impl.cc
-  block_bloom_filter_impl.cc
-  runtime_filter.cpp
-  runtime_filter_rpc.cpp
-  math_functions.cpp
-  string_functions.cpp
-  json_functions.cpp
-)
+file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp *.cc)
+add_library(Exprs STATIC ${SRC_FILES})
 
 pch_reuse(Exprs)

be/src/http/CMakeLists.txt

@@ -21,42 +21,9 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/http")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/http")
 
-add_library(Webserver STATIC
-  http_headers.cpp
-  http_method.cpp
-  http_request.cpp
-  http_channel.cpp
-  http_status.cpp
-  http_parser.cpp
-  http_handler_with_auth.cpp
-  web_page_handler.cpp
-  default_path_handlers.cpp
-  utils.cpp
-  ev_http_server.cpp
-  http_client.cpp
-  action/download_action.cpp
-  action/download_binlog_action.cpp
-  action/pad_rowset_action.cpp
-  action/health_action.cpp
-  action/tablet_migration_action.cpp
-  action/tablets_info_action.cpp
-  action/tablets_distribution_action.cpp
-  action/checksum_action.cpp
-  action/snapshot_action.cpp
-  action/reload_tablet_action.cpp
-  action/restore_tablet_action.cpp
-  action/pprof_actions.cpp
-  action/metrics_action.cpp
-  action/stream_load.cpp
-  action/stream_load_2pc.cpp
-  action/meta_action.cpp
-  action/compaction_action.cpp
-  action/config_action.cpp
-  action/check_rpc_channel_action.cpp
-  action/reset_rpc_channel_action.cpp
-  action/check_tablet_segment_action.cpp
-  action/version_action.cpp
-  action/jeprofile_actions.cpp
-  action/file_cache_action.cpp)
+file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS
+    *.cpp
+)
+add_library(Webserver STATIC ${SRC_FILES})
 
 pch_reuse(Webserver)

be/src/io/CMakeLists.txt

@@ -21,46 +21,8 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
 
-set(IO_FILES
-    file_factory.cpp
-    hdfs_builder.cpp
-    fs/file_reader_options.cpp
-    fs/file_reader.cpp
-    fs/file_system.cpp
-    fs/remote_file_system.cpp
-    fs/local_file_system.cpp
-    fs/local_file_reader.cpp
-    fs/local_file_writer.cpp
-    fs/s3_file_system.cpp
-    fs/s3_file_reader.cpp
-    fs/s3_file_writer.cpp
-    fs/s3_file_write_bufferpool.cpp
-    fs/hdfs_file_system.cpp
-    fs/hdfs_file_reader.cpp
-    fs/hdfs_file_writer.cpp
-    fs/broker_file_system.cpp
-    fs/broker_file_reader.cpp
-    fs/broker_file_writer.cpp
-    fs/buffered_reader.cpp
-    fs/stream_load_pipe.cpp
-    fs/multi_table_pipe.cpp
-    fs/err_utils.cpp
-    fs/fs_utils.cpp
-    cache/dummy_file_cache.cpp
-    cache/file_cache.cpp
-    cache/file_cache_manager.cpp
-    cache/sub_file_cache.cpp
-    cache/whole_file_cache.cpp
-    cache/block/block_file_segment.cpp
-    cache/block/block_file_cache.cpp
-    cache/block/block_file_cache_profile.cpp
-    cache/block/block_file_cache_factory.cpp
-    cache/block/block_lru_file_cache.cpp
-    cache/block/cached_remote_file_reader.cpp
-)
+file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp)
 
-add_library(IO STATIC
-    ${IO_FILES}
-)
+add_library(IO STATIC ${IO_FILES})
 
 pch_reuse(IO)

be/src/olap/CMakeLists.txt

@@ -23,100 +23,8 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/olap")
 
 add_subdirectory(rowset)
 
-add_library(Olap STATIC
-    base_compaction.cpp
-    base_tablet.cpp
-    binlog_config.cpp
-    bloom_filter.hpp
-    block_column_predicate.cpp
-    cold_data_compaction.cpp
-    compaction.cpp
-    compaction_permit_limiter.cpp   
-    cumulative_compaction.cpp
-    cumulative_compaction_policy.cpp
-    delete_bitmap_calculator.cpp
-    delete_handler.cpp
-    delta_writer.cpp
-    hll.cpp
-    inverted_index_parser.cpp
-    itoken_extractor.cpp
-    like_column_predicate.cpp
-    key_coder.cpp
-    lru_cache.cpp
-    match_predicate.cpp
-    memtable.cpp
-    memtable_flush_executor.cpp
-    merger.cpp
-    null_predicate.cpp
-    olap_meta.cpp
-    olap_server.cpp
-    schema_cache.cpp
-    options.cpp
-    page_cache.cpp
-    push_handler.cpp
-    reader.cpp
-    row_cursor.cpp
-    version_graph.cpp
-    schema.cpp
-    schema_change.cpp
-    storage_engine.cpp
-    data_dir.cpp
-    short_key_index.cpp
-    single_replica_compaction.cpp
-    snapshot_manager.cpp
-    tablet.cpp
-    tablet_manager.cpp
-    tablet_meta.cpp
-    tablet_meta_manager.cpp
-    tablet_schema.cpp
-    tablet_schema_cache.cpp
-    txn_manager.cpp
-    types.cpp 
-    utils.cpp
-    wrapper_field.cpp
-    primary_key_index.cpp
-    rowset/segment_v2/bitmap_index_reader.cpp
-    rowset/segment_v2/bitmap_index_writer.cpp
-    rowset/segment_v2/inverted_index_reader.cpp
-    rowset/segment_v2/inverted_index_writer.cpp
-    rowset/segment_v2/inverted_index_cache.cpp
-    rowset/segment_v2/inverted_index_desc.cpp
-    rowset/segment_v2/inverted_index_compound_directory.cpp
-    rowset/segment_v2/inverted_index_compound_reader.cpp
-    rowset/segment_v2/inverted_index_compaction.cpp
-    rowset/segment_v2/bitshuffle_page.cpp
-    rowset/segment_v2/bitshuffle_wrapper.cpp
-    rowset/segment_v2/column_reader.cpp
-    rowset/segment_v2/column_writer.cpp
-    rowset/segment_v2/encoding_info.cpp
-    rowset/segment_v2/index_page.cpp
-    rowset/segment_v2/indexed_column_reader.cpp
-    rowset/segment_v2/indexed_column_writer.cpp
-    rowset/segment_v2/ordinal_page_index.cpp
-    rowset/segment_v2/page_io.cpp
-    rowset/segment_v2/binary_dict_page.cpp
-    rowset/segment_v2/binary_prefix_page.cpp
-    rowset/segment_v2/segment.cpp
-    rowset/segment_v2/segment_iterator.cpp
-    rowset/segment_v2/empty_segment_iterator.cpp
-    rowset/segment_v2/segment_writer.cpp
-    rowset/segment_v2/block_split_bloom_filter.cpp
-    rowset/segment_v2/ngram_bloom_filter.cpp
-    rowset/segment_v2/bloom_filter_index_reader.cpp
-    rowset/segment_v2/bloom_filter_index_writer.cpp
-    rowset/segment_v2/bloom_filter.cpp
-    rowset/segment_v2/zone_map_index.cpp
-    task/engine_batch_load_task.cpp
-    task/engine_checksum_task.cpp
-    task/engine_clone_task.cpp
-    task/engine_storage_migration_task.cpp
-    task/engine_publish_version_task.cpp
-    task/engine_alter_tablet_task.cpp
-    task/engine_index_change_task.cpp
-    task/index_builder.cpp
-    segment_loader.cpp
-    storage_policy.cpp
-)
+file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp)
+add_library(Olap STATIC ${SRC_FILES})
 
 if (NOT USE_MEM_TRACKER)
     target_compile_options(Olap PRIVATE -Wno-unused-lambda-capture)

be/src/olap/rowset/segment_v2/ngram_bloom_filter.cpp

@@ -20,8 +20,8 @@
 #include <gen_cpp/segment_v2.pb.h>
 #include <glog/logging.h>
 
+#include "gutil/hash/city.h"
 #include "gutil/strings/substitute.h"
-#include "util/cityhash102/city.h"
 
 namespace doris {
 namespace segment_v2 {
@@ -54,8 +54,8 @@ Status NGramBloomFilter::init(const char* buf, uint32_t size, HashStrategyPB str
 }
 
 void NGramBloomFilter::add_bytes(const char* data, uint32_t len) {
-    size_t hash1 = CityHash_v1_0_2::CityHash64WithSeed(data, len, 0);
-    size_t hash2 = CityHash_v1_0_2::CityHash64WithSeed(data, len, SEED_GEN);
+    size_t hash1 = util_hash::CityHash64WithSeed(data, len, 0);
+    size_t hash2 = util_hash::CityHash64WithSeed(data, len, SEED_GEN);
 
     for (size_t i = 0; i < HASH_FUNCTIONS; ++i) {
         size_t pos = (hash1 + i * hash2 + i * i) % (8 * _size);

be/src/pipeline/CMakeLists.txt

@@ -21,51 +21,10 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/pipeline")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/pipeline")
 
-set(PIPELINE_FILES
-        pipeline.cpp
-        pipeline_fragment_context.cpp
-        pipeline_task.cpp
-        task_queue.cpp
-        task_scheduler.cpp
-        exec/operator.cpp
-        exec/scan_operator.cpp
-        exec/schema_scan_operator.cpp
-        exec/datagen_operator.cpp
-        exec/empty_set_operator.cpp
-        exec/exchange_source_operator.cpp
-        exec/exchange_sink_operator.cpp
-        exec/exchange_sink_buffer.cpp
-        exec/result_sink_operator.cpp
-        exec/result_file_sink_operator.cpp
-        exec/aggregation_sink_operator.cpp
-        exec/aggregation_source_operator.cpp
-        exec/hashjoin_build_sink.cpp
-        exec/hashjoin_probe_operator.cpp
-        exec/analytic_sink_operator.cpp
-        exec/analytic_source_operator.cpp
-        exec/streaming_aggregation_source_operator.cpp
-        exec/streaming_aggregation_sink_operator.cpp
-        exec/sort_source_operator.cpp
-        exec/sort_sink_operator.cpp
-        exec/repeat_operator.cpp
-        exec/table_function_operator.cpp
-        exec/nested_loop_join_build_operator.cpp
-        exec/nested_loop_join_probe_operator.cpp
-        exec/set_sink_operator.cpp
-        exec/set_source_operator.cpp
-        exec/set_probe_sink_operator.cpp
-        exec/union_sink_operator.cpp
-        exec/union_source_operator.cpp
-        exec/data_queue.cpp
-        exec/select_operator.cpp
-        exec/empty_source_operator.cpp
-        exec/multi_cast_data_streamer.cpp
-        exec/multi_cast_data_stream_source.cpp)
+file(GLOB_RECURSE PIPELINE_FILES CONFIGURE_DEPENDS *.cpp)
 
-if (WITH_MYSQL)
-    set(PIPELINE_FILES
-            ${PIPELINE_FILES}
-            exec/mysql_scan_operator.cpp)
+if (NOT WITH_MYSQL)
+    list(REMOVE_ITEM PIPELINE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/exec/mysql_scan_operator.cpp)
 endif ()
 
 add_library(Pipeline STATIC