From 3dd4eb4923e8a087a39f045f4e1f94158ef2ea3d Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 17 Sep 2024 11:12:09 -0700 Subject: [PATCH] Configurable long timeout for metadata queries (#609) Co-authored-by: Bryan Keller --- .../metacat/common/server/properties/Config.java | 7 +++++++ .../server/properties/DefaultConfigImpl.java | 5 +++++ .../common/server/properties/UserMetadata.java | 1 + .../metacat/metadata/mysql/MySqlServiceUtil.java | 14 ++++++++++++++ .../metacat/metadata/mysql/MySqlTagService.java | 9 ++++++--- .../metadata/mysql/MySqlUserMetadataConfig.java | 6 ++---- 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java index c9669e737..1f7941ed1 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java @@ -587,6 +587,13 @@ public interface Config { */ int getMetadataQueryTimeout(); + /** + * Metadata query timeout in seconds, for longer running queries. + * + * @return Metadata query timeout in seconds for longer running queries + */ + int getLongMetadataQueryTimeout(); + /** * Whether to check the existence of the iceberg metadata location before updating the table. * diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java index 5f223ed7a..f1c9e5d75 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java @@ -664,6 +664,11 @@ public int getMetadataQueryTimeout() { return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds(); } + @Override + public int getLongMetadataQueryTimeout() { + return this.metacatProperties.getUsermetadata().getLongQueryTimeoutInSeconds(); + } + @Override public boolean isIcebergPreviousMetadataLocationCheckEnabled() { return this.metacatProperties.getHive().getIceberg().isIcebergPreviousMetadataLocationCheckEnabled(); diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java index 6c6a13927..7af4b595a 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java @@ -31,6 +31,7 @@ public class UserMetadata { @NonNull private Config config = new Config(); private int queryTimeoutInSeconds = 60; + private int longQueryTimeoutInSeconds = 120; /** * config related properties. diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlServiceUtil.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlServiceUtil.java index d4c460f7b..3601bca5b 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlServiceUtil.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlServiceUtil.java @@ -19,6 +19,7 @@ import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; +import javax.sql.DataSource; import java.io.InputStream; import java.net.URL; import java.nio.file.FileSystems; @@ -81,6 +82,19 @@ public static void loadMySqlDataSource(final DataSourceManager dataSourceManager } dataSourceManager.load(UserMetadataService.NAME_DATASOURCE, connectionProperties); } + + /** + * Create a JDBC template with a query timeout. + * + * @param dataSource data source + * @param timeoutSec query timeout, in sec + * @return the JDBC template + */ + public static JdbcTemplate createJdbcTemplate(final DataSource dataSource, final int timeoutSec) { + final JdbcTemplate result = new JdbcTemplate(dataSource); + result.setQueryTimeout(timeoutSec); + return result; + } } diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java index f8d039f34..3b6acd445 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java @@ -99,7 +99,8 @@ public class MySqlTagService implements TagService { private final LookupService lookupService; private final MetacatJson metacatJson; private final UserMetadataService userMetadataService; - private JdbcTemplate jdbcTemplate; + private final JdbcTemplate jdbcTemplate; + private final JdbcTemplate jdbcTemplateLongTimeout; /** * Constructor. @@ -119,6 +120,8 @@ public MySqlTagService( ) { this.config = Preconditions.checkNotNull(config, "config is required"); this.jdbcTemplate = jdbcTemplate; + this.jdbcTemplateLongTimeout = MySqlServiceUtil.createJdbcTemplate( + jdbcTemplate.getDataSource(), config.getLongMetadataQueryTimeout()); this.lookupService = Preconditions.checkNotNull(lookupService, "lookupService is required"); this.metacatJson = Preconditions.checkNotNull(metacatJson, "metacatJson is required"); this.userMetadataService = Preconditions.checkNotNull(userMetadataService, "userMetadataService is required"); @@ -320,7 +323,7 @@ public Set getTags() { * @return list of qualified names of the items */ @Override - @Transactional(readOnly = true, timeout = 120) + @Transactional(readOnly = true) public List list( @Nullable final Set includeTags, @Nullable final Set excludeTags, @@ -477,7 +480,7 @@ private List queryTaggedItems(final String name, new SqlParameterValue(Types.INTEGER, type == null ? 1 : 0), new SqlParameterValue(Types.VARCHAR, type == null ? ".*" : type.getRegexValue()) ).collect(Collectors.toList())); - return jdbcTemplate.query(query, + return jdbcTemplateLongTimeout.query(query, sqlParams.toArray(), (rs, rowNum) -> rs.getString("name")); } diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java index a195d403c..aba328caa 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java @@ -162,9 +162,7 @@ public DataSourceTransactionManager metadataTxManager( public JdbcTemplate metadataJdbcTemplate( @Qualifier("metadataDataSource") final DataSource mySqlDataSource, final Config config) { - final JdbcTemplate result = new JdbcTemplate(mySqlDataSource); - result.setQueryTimeout(config.getMetadataQueryTimeout()); - return result; + return MySqlServiceUtil.createJdbcTemplate( + mySqlDataSource, config.getMetadataQueryTimeout()); } - }