From 4ac8999e9c03ada28ecdc1ffa8cf6ca5ac56edb2 Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 17 Sep 2024 07:48:41 -0700 Subject: [PATCH] Configurable long timeout --- .../common/server/properties/Config.java | 7 +++++++ .../server/properties/DefaultConfigImpl.java | 5 +++++ .../server/properties/UserMetadata.java | 1 + .../metadata/mysql/MySqlTagService.java | 9 ++++++--- .../mysql/MySqlUserMetadataConfig.java | 20 ++++++++++++++++++- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java index c9669e737..50321a0bc 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java @@ -587,6 +587,13 @@ public interface Config { */ int getMetadataQueryTimeout(); + /** + * Long metadata query timeout in seconds, for longer running queries. + * + * @return Long metadata query timeout in seconds + */ + int getLongMetadataQueryTimeout(); + /** * Whether to check the existence of the iceberg metadata location before updating the table. * diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java index 5f223ed7a..f1c9e5d75 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java @@ -664,6 +664,11 @@ public int getMetadataQueryTimeout() { return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds(); } + @Override + public int getLongMetadataQueryTimeout() { + return this.metacatProperties.getUsermetadata().getLongQueryTimeoutInSeconds(); + } + @Override public boolean isIcebergPreviousMetadataLocationCheckEnabled() { return this.metacatProperties.getHive().getIceberg().isIcebergPreviousMetadataLocationCheckEnabled(); diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java index 6c6a13927..7af4b595a 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java @@ -31,6 +31,7 @@ public class UserMetadata { @NonNull private Config config = new Config(); private int queryTimeoutInSeconds = 60; + private int longQueryTimeoutInSeconds = 120; /** * config related properties. diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java index f8d039f34..92083829f 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java @@ -99,7 +99,8 @@ public class MySqlTagService implements TagService { private final LookupService lookupService; private final MetacatJson metacatJson; private final UserMetadataService userMetadataService; - private JdbcTemplate jdbcTemplate; + private final JdbcTemplate jdbcTemplate; + private final JdbcTemplate jdbcTemplateLongTimeout; /** * Constructor. @@ -113,12 +114,14 @@ public class MySqlTagService implements TagService { public MySqlTagService( final Config config, final JdbcTemplate jdbcTemplate, + final JdbcTemplate jdbcTemplateLongTimeout, final LookupService lookupService, final MetacatJson metacatJson, final UserMetadataService userMetadataService ) { this.config = Preconditions.checkNotNull(config, "config is required"); this.jdbcTemplate = jdbcTemplate; + this.jdbcTemplateLongTimeout = jdbcTemplateLongTimeout; this.lookupService = Preconditions.checkNotNull(lookupService, "lookupService is required"); this.metacatJson = Preconditions.checkNotNull(metacatJson, "metacatJson is required"); this.userMetadataService = Preconditions.checkNotNull(userMetadataService, "userMetadataService is required"); @@ -320,7 +323,7 @@ public Set getTags() { * @return list of qualified names of the items */ @Override - @Transactional(readOnly = true, timeout = 120) + @Transactional(readOnly = true) public List list( @Nullable final Set includeTags, @Nullable final Set excludeTags, @@ -477,7 +480,7 @@ private List queryTaggedItems(final String name, new SqlParameterValue(Types.INTEGER, type == null ? 1 : 0), new SqlParameterValue(Types.VARCHAR, type == null ? ".*" : type.getRegexValue()) ).collect(Collectors.toList())); - return jdbcTemplate.query(query, + return jdbcTemplateLongTimeout.query(query, sqlParams.toArray(), (rs, rowNum) -> rs.getString("name")); } diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java index a195d403c..045a89924 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java @@ -29,6 +29,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; @@ -101,12 +102,13 @@ public LookupService lookupService( @Bean public TagService tagService( @Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate, + @Qualifier("metadataJdbcTemplateLongTimeout") final JdbcTemplate jdbcTemplateLongTimeout, final Config config, final MetacatJson metacatJson, final LookupService lookupService, final UserMetadataService userMetadataService ) { - return new MySqlTagService(config, jdbcTemplate, lookupService, metacatJson, userMetadataService); + return new MySqlTagService(config, jdbcTemplate, jdbcTemplateLongTimeout, lookupService, metacatJson, userMetadataService); } /** @@ -159,6 +161,7 @@ public DataSourceTransactionManager metadataTxManager( * @return metadata JDBC template */ @Bean + @Primary public JdbcTemplate metadataJdbcTemplate( @Qualifier("metadataDataSource") final DataSource mySqlDataSource, final Config config) { @@ -167,4 +170,19 @@ public JdbcTemplate metadataJdbcTemplate( return result; } + /** + * mySql metadata JDBC template for longer running queries. + * + * @param mySqlDataSource metadata data source + * @param config System config to use + * @return metadata JDBC template + */ + @Bean + public JdbcTemplate metadataJdbcTemplateLongTimeout( + @Qualifier("metadataDataSource") final DataSource mySqlDataSource, + final Config config) { + final JdbcTemplate result = new JdbcTemplate(mySqlDataSource); + result.setQueryTimeout(config.getLongMetadataQueryTimeout()); + return result; + } }