diff --git a/cli/pom.xml b/cli/pom.xml index b24ca6cc8261..485b9a87020e 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -127,6 +127,30 @@ hbase-common test + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + + + org.apache.hive + hive-common + ${project.version} + tests + test + + + org.apache.tez + tez-mapreduce + test + + + org.apache.tez + tez-dag + test + org.mockito mockito-core diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 93e06208e1ee..92be54934aeb 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -740,7 +740,7 @@ public int run(String[] args) throws Exception { logInitDetailMessage = e.getMessage(); } - CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); + CliSessionState ss = new CliSessionState(getConf()); ss.in = System.in; try { ss.out = @@ -815,6 +815,10 @@ public Map getHiveVariable() { } } + protected HiveConf getConf() { + return new HiveConf(SessionState.class); + } + /** * Execute the cli work * @param ss CliSessionState of the CLI driver diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index 37448fee2138..df0cdc52291c 100644 --- a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.common.io.SessionStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.IDriver; @@ -57,6 +58,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.junit.Test; + import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; @@ -130,7 +132,8 @@ public void testThatCliDriverDoesNotStripComments() throws Exception { SessionStream err = new SessionStream(dataErr); System.setErr(err); - CliSessionState ss = new CliSessionState(new HiveConf()); + HiveConf hiveConf = getHiveConf(); + CliSessionState ss = new CliSessionState(hiveConf); ss.out = out; ss.err = err; @@ -226,7 +229,7 @@ public void testRun() throws Exception { File historyFile = new File(historyDirectory + File.separator + ".hivehistory"); historyFile.delete(); } - HiveConf configuration = new HiveConf(); + HiveConf configuration = getHiveConf(); configuration.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true); PrintStream oldOut = System.out; ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); @@ -239,7 +242,7 @@ public void testRun() throws Exception { String[] args = {}; try { - new FakeCliDriver().run(args); + new FakeCliDriver(configuration).run(args); assertTrue(dataOut.toString(), dataOut.toString().contains("test message")); assertTrue(dataErr.toString(), dataErr.toString().contains("Hive history file=")); assertTrue(dataErr.toString(), dataErr.toString().contains("File: fakeFile is not a file.")); @@ -260,7 +263,7 @@ public void testRun() throws Exception { @Test public void testQuit() throws Exception { - CliSessionState ss = new CliSessionState(new HiveConf()); + CliSessionState ss = new CliSessionState(getHiveConf()); ss.err = new SessionStream(System.err); ss.out = new SessionStream(System.out); @@ -290,15 +293,15 @@ public void testQuit() throws Exception { @Test public void testProcessSelectDatabase() throws Exception { - CliSessionState sessinState = new CliSessionState(new HiveConf()); - CliSessionState.start(sessinState); + CliSessionState sessionState = new CliSessionState(getHiveConf()); + CliSessionState.start(sessionState); ByteArrayOutputStream data = new ByteArrayOutputStream(); - sessinState.err = new SessionStream(data); - sessinState.database = "database"; + sessionState.err = new SessionStream(data); + sessionState.database = "database"; CliDriver driver = new CliDriver(); try { - driver.processSelectDatabase(sessinState); + driver.processSelectDatabase(sessionState); fail("shuld be exit"); } catch (ExitException e) { e.printStackTrace(); @@ -325,7 +328,7 @@ public void testprocessInitFiles() throws Exception { FileUtils.write(homeFile, "-- init hive file for test "); setEnv("HIVE_HOME", homeFile.getParentFile().getParentFile().getAbsolutePath()); setEnv("HIVE_CONF_DIR", homeFile.getParentFile().getAbsolutePath()); - CliSessionState sessionState = new CliSessionState(new HiveConf()); + CliSessionState sessionState = new CliSessionState(getHiveConf()); ByteArrayOutputStream data = new ByteArrayOutputStream(); @@ -418,11 +421,20 @@ private static void setEnv(String key, String value) throws Exception { private static class FakeCliDriver extends CliDriver { + private HiveConf conf; + + public FakeCliDriver(HiveConf configuration) { + this.conf = configuration; + } + @Override protected void setupConsoleReader() throws IOException { reader = new FakeConsoleReader(); } + protected HiveConf getConf() { + return conf; + } } private static class FakeConsoleReader extends ConsoleReader { @@ -514,4 +526,8 @@ public int getStatus() { return status; } } + + private HiveConf getHiveConf() { + return new HiveConfForTest(this.getClass()); + } } diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestCliSessionState.java b/cli/src/test/org/apache/hadoop/hive/cli/TestCliSessionState.java index c159142954f5..3e9a39a637b4 100644 --- a/cli/src/test/org/apache/hadoop/hive/cli/TestCliSessionState.java +++ b/cli/src/test/org/apache/hadoop/hive/cli/TestCliSessionState.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Test; @@ -33,8 +33,8 @@ public class TestCliSessionState { * test default db name */ @Test - public void testgetDbName() throws Exception { - SessionState.start(new HiveConf()); + public void testGetDbName() throws Exception { + SessionState.start(new HiveConfForTest(getClass())); assertEquals(Warehouse.DEFAULT_DATABASE_NAME, SessionState.get().getCurrentDatabase()); } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 173d8efcdf5f..880d553845f6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4582,10 +4582,9 @@ public static enum ConfVars { HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false, "Whether to show the unquoted partition names in query results."), - HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet(true, "mr", "tez"), - "Chooses execution engine. Options are: mr (Map reduce, default), tez. While MR\n" + - "remains the default engine for historical reasons, it is itself a historical engine\n" + - "and is deprecated in Hive 2 line. It may be removed without further warning."), + HIVE_EXECUTION_ENGINE("hive.execution.engine", "tez", new StringSet(true, "tez", "mr"), + "Chooses execution engine. Options are: 'tez' (Tez, default), 'mr' (MapReduce, deprecated). "+ + "MR is a historical engine and is deprecated in Hive 2 line. It may be removed without further warning."), HIVE_EXECUTION_MODE("hive.execution.mode", "container", new StringSet("container", "llap"), "Chooses whether query fragments will run in container or in llap"), diff --git a/common/src/test/org/apache/hadoop/hive/conf/HiveConfForTest.java b/common/src/test/org/apache/hadoop/hive/conf/HiveConfForTest.java new file mode 100644 index 000000000000..65e23173dea1 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/conf/HiveConfForTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.conf; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; + +/** + * This class is mostly used in unit test environment to prevent polluting the + * original HiveConf, and also to provide a way to set some default values for execution. + */ +public class HiveConfForTest extends HiveConf { + private static final Logger LOG = LoggerFactory.getLogger(HiveConfForTest.class.getName()); + private String testDataDir; + private Map overlay = new HashMap<>(); + + public HiveConfForTest(Class cls) { + super(cls); + init(cls); + } + + public HiveConfForTest(Configuration configuration, Class cls) { + super(configuration, cls); + init(cls); + } + + private void init(Class cls) { + initDataDir(cls); + LOG.info("Using test data dir (class: {}): {}", cls.getName(), testDataDir); + // HIVE_USER_INSTALL_DIR is required in DagUtils, let's set one + setValue(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR.varname, testDataDir); + // to avoid the overhead of starting a tez session when creating a new SessionState + // many unit tests don't need actual tez execution, and this can save a lot of time + setValue(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, "false"); + // to avoid the overhead of using a real yarn cluster for unit tests. + setValue("tez.local.mode", "true"); + // to avoid the overhead of starting an RPC server of the local DAGAppMaster + setValue("tez.local.mode.without.network", "true"); + // tests might assume this is set + setValue("hive.in.tez.test", "true"); + // to prevent polluting the git tree with local tez cache folders (like tez-local-cache136810858646778831) + setValue("tez.local.cache.root.folder", System.getProperty("build.dir")); + // prevent RecoveryService from starting, which is not needed in unit tests + setValue("tez.dag.recovery.enabled", "false"); + } + + public void setValue(String name, String value) { + overlay.put(name, value); + super.set(name, value); + } + + /** + * Get a unique directory for storing test data, which is based on the class name of the caller unit test + * and build.dir environment variable (which is set by the surefire plugin, look for root pom.xml). + * @param cls + * @return + */ + private void initDataDir(Class cls) { + testDataDir = new File( + String.format("%s/%s-%d", System.getProperty("build.dir"), cls.getCanonicalName(), System.currentTimeMillis())) + .getPath(); + } + + public String getTestDataDir() { + return testDataDir; + } + + /** + * Get all overlay options as a query string. + * This can be used in jdbc connection related tests where we want to pass + * only the specific values which were set in this class. + */ + public String getOverlayOptionsAsQueryString() { + return Joiner.on(";").withKeyValueSeparator("=").join(overlay); + } +} diff --git a/data/conf/iceberg/llap/hive-site.xml b/data/conf/iceberg/llap/hive-site.xml index 7a99c50c8c2b..a4bdb6266b56 100644 --- a/data/conf/iceberg/llap/hive-site.xml +++ b/data/conf/iceberg/llap/hive-site.xml @@ -222,12 +222,6 @@ Use column stats to annotate stats for physical optimization phase - - hive.execution.engine - tez - Whether to use MR or Tez - - tez.am.node-blacklisting.enabled false diff --git a/data/conf/iceberg/tez/hive-site.xml b/data/conf/iceberg/tez/hive-site.xml index 248adad2d83c..19984c7ca509 100644 --- a/data/conf/iceberg/tez/hive-site.xml +++ b/data/conf/iceberg/tez/hive-site.xml @@ -219,12 +219,6 @@ Use column stats to annotate stats for physical optimization phase - - hive.execution.engine - tez - Whether to use MR or Tez - - tez.am.node-blacklisting.enabled false diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 4811dca19d81..4f5aedca77d2 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -228,12 +228,6 @@ The default storatge that stores temporary hive statistics. Currently, fs type is supported - - hive.execution.engine - tez - Whether to use MR or Tez - - tez.am.node-blacklisting.enabled false diff --git a/data/conf/mr/hive-site.xml b/data/conf/mr/hive-site.xml new file mode 100644 index 000000000000..003a661d9b6f --- /dev/null +++ b/data/conf/mr/hive-site.xml @@ -0,0 +1,363 @@ + + + + + + + + hive.metastore.client.cache.enabled + true + This property enables a Caffeiene Cache for Metastore client + + + + hive.in.test + true + Internal marker for test. Used for masking env-dependent values + + + + + mapreduce.jobtracker.staging.root.dir + ${test.tmp.dir}/cli/mapred/staging + + + + + hadoop.tmp.dir + ${test.tmp.dir}/hadoop-tmp + A base for other temporary directories. + + + + hive.execution.engine + mr + + + + hive.exec.scratchdir + ${test.tmp.dir}/scratchdir + Scratch space for Hive jobs + + + + hive.exec.local.scratchdir + ${test.tmp.dir}/localscratchdir/ + Local scratch space for Hive jobs + + + + datanucleus.schema.autoCreateAll + true + + + + datanucleus.connectionPool.maxPoolSize + 4 + + + + hive.metastore.schema.verification + false + + + + javax.jdo.option.ConnectionURL + jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true + + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + + javax.jdo.option.ConnectionUserName + APP + + + + javax.jdo.option.ConnectionPassword + mine + + + + + hive.metastore.warehouse.dir + ${test.warehouse.dir} + + + + + test.log.dir + ${test.tmp.dir}/log/ + + + + + test.data.files + ${hive.root}/data/files + + + + + test.data.scripts + ${hive.root}/data/scripts + + + + + hive.jar.path + ${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar + + + + + hive.metastore.rawstore.impl + org.apache.hadoop.hive.metastore.ObjectStore + Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database + + + + hive.querylog.location + ${test.tmp.dir}/tmp + Location of the structured hive logs + + + + hive.exec.pre.hooks + org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables, org.apache.hadoop.hive.ql.hooks.MaterializedViewRegistryPropertiesHook + Pre Execute Hook for Tests + + + + hive.exec.post.hooks + org.apache.hadoop.hive.ql.hooks.PostExecutePrinter + Post Execute Hook for Tests + + + + hive.support.concurrency + true + Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks. + + + + hive.unlock.numretries + 2 + The number of times you want to retry to do one unlock + + + + hive.lock.sleep.between.retries + 2 + The sleep time (in seconds) between various retries + + + + + fs.pfile.impl + org.apache.hadoop.fs.ProxyLocalFileSystem + A proxy for local file system used for cross file system testing + + + + hive.exec.mode.local.auto + false + + Let hive determine whether to run in local mode automatically + Disabling this for tests so that minimr is not affected + + + + + hive.auto.convert.join + false + Whether Hive enable the optimization about converting common join into mapjoin based on the input file size + + + + hive.ignore.mapjoin.hint + false + Whether Hive ignores the mapjoin hint + + + + hive.input.format + org.apache.hadoop.hive.ql.io.CombineHiveInputFormat + The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. + + + + hive.default.rcfile.serde + org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + The default SerDe hive will use for the rcfile format + + + + hive.conf.restricted.list + dummy.config.value + Using dummy config value above because you cannot override config with empty value + + + + hive.exec.submit.local.task.via.child + false + + + + + hive.dummyparam.test.server.specific.config.override + from.hive-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hivesite + from.hive-site.xml + Using dummy param to test server specific configuration + + + + test.var.hiveconf.property + ${hive.exec.default.partition.name} + Test hiveconf property substitution + + + + test.property1 + value1 + Test property defined in hive-site.xml only + + + + hive.test.dummystats.aggregator + value2 + + + + hive.fetch.task.conversion + minimal + + + + hive.users.in.admin.role + hive_admin_user + + + + hive.security.authorization.manager + org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest + The Hive client authorization manager class name. + + + + hive.llap.cache.allow.synthetic.fileid + true + + + + hive.llap.io.use.lrfu + true + + + + hive.llap.io.allocator.direct + false + + + + hive.stats.column.autogather + true + + + + hive.materializedview.rewriting + true + + + + hive.stats.fetch.bitvector + true + + + + + yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage + 99 + + + + hive.query.results.cache.enabled + false + + + + hive.query.reexecution.stats.persist.scope + query + + + + hive.scheduled.queries.executor.enabled + false + + + + hive.strict.timestamp.conversion + false + + + + hive.cbo.fallback.strategy + TEST + + + + iceberg.hive.keep.stats + true + + We want we keep the stats in Hive sessions. + + + + + hive.server2.operation.log.purgePolicy.timeToLive + 5s + + + + hive.txn.xlock.ctas + false + + + + + hive.server2.thrift.resultset.max.fetch.size + 1000000 + + + + hive.server2.webui.max.threads + 4 + + + + hive.async.cleanup.service.thread.count + 4 + + diff --git a/data/conf/perf/tpcds30tb/tez/hive-site.xml b/data/conf/perf/tpcds30tb/tez/hive-site.xml index b806e2b912b4..6b3d7d78b064 100644 --- a/data/conf/perf/tpcds30tb/tez/hive-site.xml +++ b/data/conf/perf/tpcds30tb/tez/hive-site.xml @@ -137,10 +137,6 @@ hive.convert.join.bucket.mapjoin.tez false - - hive.execution.engine - tez - hive.vectorized.execution.mapjoin.minmax.enabled true diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index e797e544daea..6c969616a720 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -220,12 +220,6 @@ Use column stats to annotate stats for physical optimization phase - - hive.execution.engine - tez - Whether to use MR or Tez - - tez.am.node-blacklisting.enabled false diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml index 756dd45203f2..f9609c1a2da9 100644 --- a/hbase-handler/pom.xml +++ b/hbase-handler/pom.xml @@ -34,6 +34,13 @@ hive-common ${project.version} + + org.apache.hive + hive-common + ${project.version} + tests + test + org.apache.hive hive-exec @@ -250,6 +257,16 @@ + + org.apache.tez + tez-mapreduce + test + + + org.apache.tez + tez-dag + test + org.mockito mockito-core diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java index 16bdd8cf3d27..010be8de5f9f 100644 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; @@ -52,7 +53,7 @@ public class TestHBaseQueries { * databases, etc.), otherwise those will be visible for subsequent test methods too. */ public TestHBaseQueries() throws Exception { - baseConf = new HiveConf(HBaseConfiguration.create(), TestHBaseQueries.class); + baseConf = new HiveConfForTest(HBaseConfiguration.create(), TestHBaseQueries.class); baseConf.set(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER.varname, SQLStdHiveAuthorizerFactory.class.getName()); // set up Zookeeper diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java index 8282c68ff5f5..48015b4ba530 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; @@ -54,7 +55,7 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC */ @Test public void authEnabledV1Auth() throws Exception { - HiveConf hcatConf = new HiveConf(this.getClass()); + HiveConf hcatConf = new HiveConfForTest(this.getClass()); hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); hcatConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, StorageBasedAuthorizationProvider.class.getName()); SessionState.start(hcatConf); @@ -66,7 +67,7 @@ public void authEnabledV1Auth() throws Exception { */ @Test public void authEnabledV2Auth() throws Exception { - HiveConf hcatConf = new HiveConf(this.getClass()); + HiveConf hcatConf = new HiveConfForTest(this.getClass()); hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); hcatConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, DummyV2AuthorizerFactory.class.getName()); SessionState.start(hcatConf); @@ -78,7 +79,7 @@ public void authEnabledV2Auth() throws Exception { */ @Test public void authDisabled() throws Exception { - HiveConf hcatConf = new HiveConf(this.getClass()); + HiveConf hcatConf = new HiveConfForTest(this.getClass()); hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, false); SessionState.start(hcatConf); assertFalse("hcat auth should be disabled", HCatAuthUtil.isAuthorizationEnabled(hcatConf)); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestUseDatabase.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestUseDatabase.java index 73558f92cd71..2f24619acda5 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestUseDatabase.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestUseDatabase.java @@ -40,8 +40,9 @@ public class TestUseDatabase { @Before public void setUp() throws Exception { - HiveConf hcatConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hcatConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hcatConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hcatConf.set(ConfVars.PRE_EXEC_HOOKS.varname, ""); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 25cb75ec41d2..0c110465573b 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -79,6 +79,8 @@ public void setUp() throws Exception { */ protected void setUpHiveConf() { hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); hiveConf.set("mapred.local.dir", workDir + File.separator + this.getClass().getSimpleName() diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java index a787f409eb3f..429ed56f90d1 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java @@ -56,8 +56,10 @@ public class TestPassProperties { private static String[] input; private static HiveConf hiveConf; - public void Initialize() throws Exception { + public void initialize() throws Exception { hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); @@ -82,7 +84,7 @@ public void Initialize() throws Exception { @Test public void testSequenceTableWriteReadMR() throws Exception { - Initialize(); + initialize(); String createTable = "CREATE TABLE bad_props_table(a0 int, a1 String, a2 String) STORED AS SEQUENCEFILE"; driver.run("drop table bad_props_table"); driver.run(createTable); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index d5e5fc311973..ca0cb3add708 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -84,6 +84,8 @@ public void setUp() throws Exception { } HiveConf hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index b16d5c183d50..d9bd7c7e74f0 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -62,9 +62,7 @@ @RunWith(Parameterized.class) public class TestHCatLoaderComplexSchema { - //private static MiniCluster cluster = MiniCluster.buildCluster(); private static IDriver driver; - //private static Properties props; private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderComplexSchema.class); private static final Map> DISABLED_STORAGE_FORMATS = @@ -106,6 +104,8 @@ private void createTable(String tablename, String schema) throws Exception { @BeforeClass public static void setUpBeforeClass() throws Exception { HiveConf hiveConf = new HiveConf(TestHCatLoaderComplexSchema.class); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); hiveConf.set("mapred.local.dir", workDir + File.separator + "TestHCatLoaderComplexSchema" diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 0e5691a66543..6fbe515c35e2 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -159,6 +159,8 @@ public void setup() throws Exception { } HiveConf hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index 3a2b3c15b5fc..ef22b6d14593 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -95,6 +95,8 @@ public void setUp() throws Exception { if (driver == null) { HiveConf hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/hcatalog/pom.xml b/hcatalog/pom.xml index 2328f80ec898..e473dc387475 100644 --- a/hcatalog/pom.xml +++ b/hcatalog/pom.xml @@ -79,6 +79,13 @@ tests test + + org.apache.hive + hive-common + ${project.version} + tests + test + org.apache.pig pig diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml index 4307000600a1..cd5ac68fa33c 100644 --- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml +++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml @@ -31,9 +31,10 @@ thrift://localhost:9933 For Hive CLI to connect to + hive.execution.engine - mr + mr templeton.hive.properties hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java index 0cbd2b12a348..b792dc8bf99a 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java @@ -76,6 +76,8 @@ public static void setUpBeforeClass() throws Exception { TestHCatClient.startMetaStoreServer(); hconf = TestHCatClient.getConf(); + //TODO: HIVE-27998: hcatalog tests on Tez + hconf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hconf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname,""); hconf.set(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); hconf diff --git a/hplsql/src/main/resources/hplsql-site.xml b/hplsql/src/main/resources/hplsql-site.xml index 4cd43eddb87c..8b41d390005e 100644 --- a/hplsql/src/main/resources/hplsql-site.xml +++ b/hplsql/src/main/resources/hplsql-site.xml @@ -12,7 +12,6 @@ hplsql.conn.init.hiveconn - set hive.execution.engine=mr; use default; Statements for execute after connection to the database diff --git a/itests/hcatalog-unit/pom.xml b/itests/hcatalog-unit/pom.xml index e3941f3f92ac..748900958db5 100644 --- a/itests/hcatalog-unit/pom.xml +++ b/itests/hcatalog-unit/pom.xml @@ -334,6 +334,13 @@ tests test + + org.apache.hive + hive-common + ${project.version} + tests + test + diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java index acb37344972c..3aa2f8ca64d2 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java @@ -59,18 +59,17 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { - private static HiveConf hcatConf; + private static HiveConf hcatConf; private static IDriver driver; private final byte[] FAMILY = Bytes.toBytes("testFamily"); private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1"); private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2"); - public void Initialize() throws Exception { - + public void initialize() throws Exception { hcatConf = new HiveConf(this.getClass()); - //hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - // HCatSemanticAnalyzer.class.getName()); + //TODO: HIVE-27998: hcatalog tests on Tez + hcatConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); URI fsuri = getFileSystem().getUri(); Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), getTestDir()); @@ -141,7 +140,7 @@ public static void createTestDataFile(String filename) throws IOException { @Test public void testPigHBaseSchema() throws Exception { - Initialize(); + initialize(); String tableName = newTableName("MyTable"); String databaseName = newTableName("MyDatabase"); @@ -203,7 +202,7 @@ public void testPigHBaseSchema() throws Exception { @Test public void testPigFilterProjection() throws Exception { - Initialize(); + initialize(); String tableName = newTableName("MyTable"); String databaseName = newTableName("MyDatabase"); @@ -281,7 +280,7 @@ public void testPigFilterProjection() throws Exception { @Test public void testPigPopulation() throws Exception { - Initialize(); + initialize(); String tableName = newTableName("MyTable"); String databaseName = newTableName("MyDatabase"); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java index e73a57ea58f5..01d80e18dd3e 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java @@ -65,7 +65,8 @@ public class TestDbNotificationCleanup { @BeforeClass public static void connectToMetastore() throws Exception { conf = new HiveConf(); - + //TODO: HIVE-27998: hcatalog tests on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); MetastoreConf.setVar(conf,MetastoreConf.ConfVars.TRANSACTIONAL_EVENT_LISTENERS, "org.apache.hive.hcatalog.listener.DbNotificationListener"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 74b90c60a6f6..796253ffa391 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -269,6 +269,8 @@ public void onBatchAcidWrite(BatchAcidWriteEvent batchAcidWriteEvent) throws Met @BeforeClass public static void connectToMetastore() throws Exception { HiveConf conf = new HiveConf(); + //TODO: HIVE-27998: hcatalog tests on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, DbNotificationListener.class.getName()); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, MockMetaStoreEventListener.class.getName()); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java index 3b9853684a49..7d8872ba986c 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java @@ -24,6 +24,7 @@ import java.util.Collections; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -33,8 +34,6 @@ import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; -import org.apache.hadoop.hive.ql.DriverFactory; -import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; import org.junit.BeforeClass; @@ -44,16 +43,14 @@ public class TestTransactionalDbNotificationListener { private static IMetaStoreClient msClient; - private static IDriver driver; private int startTime; private long firstEventId; - @SuppressWarnings("rawtypes") @BeforeClass public static void connectToMetastore() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(TestTransactionalDbNotificationListener.class); conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, "org.apache.hive.hcatalog.listener.DbNotificationListener"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -66,8 +63,6 @@ public static void connectToMetastore() throws Exception { TestTxnDbUtil.setConfValues(conf); TestTxnDbUtil.prepDb(conf); msClient = new HiveMetaStoreClient(conf); - driver = DriverFactory.newDriver(conf); - } @Before diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java index eb093e105143..e2f10642fa48 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -70,9 +70,10 @@ public void setup() throws Exception { dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis()); hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-27998: hcatalog tests on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse"); inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data"); - hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml index 59e9927643d6..76b4fc621d3c 100644 --- a/itests/hive-blobstore/src/test/resources/hive-site.xml +++ b/itests/hive-blobstore/src/test/resources/hive-site.xml @@ -51,6 +51,12 @@ --> + + + hive.execution.engine + mr + + hive.exec.scratchdir ${test.tmp.dir}/scratchdir diff --git a/itests/hive-unit-hadoop2/pom.xml b/itests/hive-unit-hadoop2/pom.xml index 5fada3822b7c..16cd23f7339f 100644 --- a/itests/hive-unit-hadoop2/pom.xml +++ b/itests/hive-unit-hadoop2/pom.xml @@ -75,6 +75,12 @@ hive-cli test + + org.apache.hive + hive-common + tests + test + org.apache.hive hive-it-util diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index 6526c8c82835..90ded94f5979 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; @@ -138,7 +139,7 @@ public void postAnalyze(HiveSemanticAnalyzerHookContext context, */ @BeforeClass public static void setUpBeforeClass() throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = new HiveConfForTest(TestHs2Hooks.class); hiveConf.setVar(ConfVars.PRE_EXEC_HOOKS, PreExecHook.class.getName()); hiveConf.setVar(ConfVars.POST_EXEC_HOOKS, diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index a94d1da8cc1c..7aefe896355c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; @@ -41,7 +42,7 @@ public class TestMetaStoreMetrics { @BeforeClass public static void before() throws Exception { - hiveConf = new HiveConf(TestMetaStoreMetrics.class); + hiveConf = new HiveConfForTest(TestMetaStoreMetrics.class); hiveConf.setIntVar(HiveConf.ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 69db7180cfff..89c71bbffd08 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -23,6 +23,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; @@ -62,7 +63,7 @@ public void setUp() throws Exception { ObjectStore.setSchemaVerified(false); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); System.setProperty("hive.support.concurrency", "false"); System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); @@ -90,7 +91,7 @@ public void tearDown() throws Exception { @Test public void testDefaults() { System.clearProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString()); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); } @@ -102,7 +103,7 @@ public void testDefaults() { @Test public void testVersionRestriction () throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); @@ -129,7 +130,7 @@ public void testMetastoreVersion() throws Exception { // let the schema and version be auto created System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION.toString(), "true"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); try { @@ -152,7 +153,7 @@ public void testMetastoreVersion() throws Exception { @Test public void testVersionMatching () throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); try { @@ -164,7 +165,7 @@ public void testVersionMatching () throws Exception { ObjectStore.setSchemaVerified(false); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, true); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); setVersion(hiveConf, metastoreSchemaInfo.getHiveSchemaVersion()); driver = DriverFactory.newDriver(hiveConf); driver.run("show tables"); @@ -177,14 +178,14 @@ public void testVersionMatching () throws Exception { @Test public void testVersionMisMatch () throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); driver.run("show tables"); ObjectStore.setSchemaVerified(false); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); setVersion(hiveConf, "fooVersion"); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); @@ -204,13 +205,13 @@ public void testVersionMisMatch () throws Exception { @Test public void testVersionCompatibility () throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); driver.run("show tables"); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true"); - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(this.getClass()); setVersion(hiveConf, "3.9000.0"); SessionState.start(new CliSessionState(hiveConf)); driver = DriverFactory.newDriver(hiveConf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java index def816c41d66..66aea0ae71b9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -91,7 +92,7 @@ public void setUp() throws Exception { os = new ByteArrayOutputStream(); System.setOut(new PrintStream(os)); - hiveConf = new HiveConf(HiveMetaTool.class); + hiveConf = new HiveConfForTest(HiveMetaTool.class); client = new HiveMetaStoreClient(hiveConf); createDatabase(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java index db34491922a8..0fbf6427338a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; @@ -111,7 +112,11 @@ public String toString() { @Before public void setUp() throws Exception { - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(getClass()); + + hiveConf.set(HiveConf.ConfVars.HIVE_TEZ_CONTAINER_SIZE.varname, "128"); + hiveConf.setBoolean(HiveConf.ConfVars.HIVE_MERGE_TEZ_FILES.varname, false); + hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.METASTORE_WAREHOUSE.varname, TEST_WAREHOUSE_DIR); @@ -208,13 +213,11 @@ public void testMapJoinOnTez() throws Exception { @Ignore("HIVE-19509: Disable tests that are failing continuously") @Test public void testNonStandardConversion01() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - setupTez(confForTez); //CTAS with non-ACID target table runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + " stored as ORC TBLPROPERTIES('transactional'='false') as " + - "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez); + "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", hiveConf); - List rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a, b, INPUT__FILE__NAME", confForTez); + List rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a, b, INPUT__FILE__NAME", hiveConf); String expected0[][] = { {"1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"}, {"3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"}, @@ -230,9 +233,9 @@ public void testNonStandardConversion01() throws Exception { Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected0[i][1])); } //make the table ACID - runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true')", confForTez); + runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true')", hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after ctas:"); for (String s : rs) { LOG.warn(s); @@ -255,9 +258,9 @@ public void testNonStandardConversion01() throws Exception { Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1])); } //perform some Update/Delete - runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set a = 70, b = 80 where a = 7", confForTez); - runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 5", confForTez); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez); + runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set a = 70, b = 80 where a = 7", hiveConf); + runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 5", hiveConf); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after update/delete:"); for (String s : rs) { LOG.warn(s); @@ -290,9 +293,9 @@ public void testNonStandardConversion01() throws Exception { Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]); } //run Minor compaction - runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'minor'", confForTez); + runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'minor'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after compact minor:"); for (String s : rs) { LOG.warn(s); @@ -320,9 +323,9 @@ public void testNonStandardConversion01() throws Exception { Assert.assertNull("at " + i + " " + expectedDelDelta2[i] + " not found on disk", expectedDelDelta2[i]); } //run Major compaction - runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'major'", confForTez); + runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'major'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after compact major:"); for (String s : rs) { LOG.warn(s); @@ -349,17 +352,15 @@ public void testNonStandardConversion01() throws Exception { @Ignore("HIVE-17214")//this consistently works locally but never in ptest.... @Test public void testNonStandardConversion02() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - confForTez.setBoolean("mapred.input.dir.recursive", true); - setupTez(confForTez); + hiveConf.setBoolean("mapred.input.dir.recursive", true); runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + " stored as ORC " + "TBLPROPERTIES('transactional'='false') as " + "select a, b from " + Table.ACIDTBL + " where a <= 3 union all " + "select a, b from " + Table.NONACIDORCTBL + " where a >= 7 " + - "union all select a, b from " + Table.ACIDTBL + " where a = 5", confForTez); + "union all select a, b from " + Table.ACIDTBL + " where a = 5", hiveConf); List rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + - Table.NONACIDNONBUCKET + " order by a, b", confForTez); + Table.NONACIDNONBUCKET + " order by a, b", hiveConf); String expected0[][] = { {"1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"}, {"3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"}, @@ -377,7 +378,7 @@ public void testNonStandardConversion02() throws Exception { FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDNONBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); //ensure there is partition dir - runStatementOnDriver("insert into " + Table.NONACIDPART + " partition (p=1) values (100,110)", confForTez); + runStatementOnDriver("insert into " + Table.NONACIDPART + " partition (p=1) values (100,110)", hiveConf); //creates more files in that partition for(FileStatus stat : status) { int limit = 5; @@ -406,8 +407,8 @@ public void testNonStandardConversion02() throws Exception { 4 directories, 4 files **/ //make the table ACID - runStatementOnDriver("alter table " + Table.NONACIDPART + " SET TBLPROPERTIES ('transactional'='true')", confForTez); - rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", confForTez); + runStatementOnDriver("alter table " + Table.NONACIDPART + " SET TBLPROPERTIES ('transactional'='true')", hiveConf); + rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", hiveConf); LOG.warn("after acid conversion:"); for (String s : rs) { LOG.warn(s); @@ -428,9 +429,9 @@ public void testNonStandardConversion02() throws Exception { } //run Major compaction - runStatementOnDriver("alter table " + Table.NONACIDPART + " partition (p=1) compact 'major'", confForTez); + runStatementOnDriver("alter table " + Table.NONACIDPART + " partition (p=1) compact 'major'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", hiveConf); LOG.warn("after major compaction:"); for (String s : rs) { LOG.warn(s); @@ -454,14 +455,12 @@ public void testNonStandardConversion02() throws Exception { */ @Test public void testCtasTezUnion() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - confForTez.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); - setupTez(confForTez); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); //CTAS with ACID target table List rs0 = runStatementOnDriver("explain create table " + Table.ACIDNOBUCKET + " stored as ORC TBLPROPERTIES('transactional'='true') as " + "(select a, b from " + Table.ACIDTBL + " where a <= 5 order by a desc , b desc limit 3) " + "union all (select a, b from " + Table.NONACIDORCTBL + " where a >= 5 order by a desc, b desc limit 3)", - confForTez); + hiveConf); LOG.warn("explain ctas:");//TezEdgeProperty.EdgeType for (String s : rs0) { LOG.warn(s); @@ -469,8 +468,8 @@ public void testCtasTezUnion() throws Exception { runStatementOnDriver("create table " + Table.ACIDNOBUCKET + " stored as ORC TBLPROPERTIES('transactional'='true') as " + "(select a, b from " + Table.ACIDTBL + " where a <= 5 order by a desc, b desc limit 3) " + "union all (select a, b from " + Table.NONACIDORCTBL + " where a >= 5 order by a desc, b desc limit 3)", - confForTez); - List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez); + hiveConf); + List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after ctas:"); for (String s : rs) { LOG.warn(s); @@ -493,9 +492,9 @@ public void testCtasTezUnion() throws Exception { Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1])); } //perform some Update/Delete - runStatementOnDriver("update " + Table.ACIDNOBUCKET + " set a = 70, b = 80 where a = 7", confForTez); - runStatementOnDriver("delete from " + Table.ACIDNOBUCKET + " where a = 5", confForTez); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez); + runStatementOnDriver("update " + Table.ACIDNOBUCKET + " set a = 70, b = 80 where a = 7", hiveConf); + runStatementOnDriver("delete from " + Table.ACIDNOBUCKET + " where a = 5", hiveConf); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after update/delete:"); for (String s : rs) { LOG.warn(s); @@ -528,9 +527,9 @@ public void testCtasTezUnion() throws Exception { Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]); } //run Minor compaction - runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'minor'", confForTez); + runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'minor'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after compact minor:"); for (String s : rs) { LOG.warn(s); @@ -559,9 +558,9 @@ public void testCtasTezUnion() throws Exception { } runCleaner(hiveConf); //run Major compaction - runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'major'", confForTez); + runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'major'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", hiveConf); LOG.warn("after compact major:"); for (String s : rs) { LOG.warn(s); @@ -583,10 +582,8 @@ public void testCtasTezUnion() throws Exception { @Test public void testInsertWithRemoveUnion() throws Exception { int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}}; - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - setupTez(confForTez); - runStatementOnDriver("drop table if exists T", confForTez); - runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='false')", confForTez); + runStatementOnDriver("drop table if exists T", hiveConf); + runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='false')", hiveConf); /* ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree ~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/ /Users/ekoifman/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/ @@ -600,8 +597,8 @@ public void testInsertWithRemoveUnion() throws Exception { 4 directories, 3 files */ - runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez); - List rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME", confForTez); + runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", hiveConf); + List rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME", hiveConf); LOG.warn(testName.getMethodName() + ": before converting to acid"); for(String s : rs) { LOG.warn(s); @@ -619,14 +616,14 @@ public void testInsertWithRemoveUnion() throws Exception { Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1])); } //make the table ACID - runStatementOnDriver("alter table T SET TBLPROPERTIES ('transactional'='true')", confForTez); - rs = runStatementOnDriver("select a,b from T order by a, b", confForTez); + runStatementOnDriver("alter table T SET TBLPROPERTIES ('transactional'='true')", hiveConf); + rs = runStatementOnDriver("select a,b from T order by a, b", hiveConf); Assert.assertEquals("After to Acid conversion", stringifyValues(values), rs); //run Major compaction - runStatementOnDriver("alter table T compact 'major'", confForTez); + runStatementOnDriver("alter table T compact 'major'", hiveConf); runWorker(hiveConf); - rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID", confForTez); + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID", hiveConf); LOG.warn(testName.getMethodName() + ": after compact major of T:"); for (String s : rs) { LOG.warn(s); @@ -658,10 +655,8 @@ public void testInsertWithRemoveUnion() throws Exception { */ @Test public void testAcidInsertWithRemoveUnion() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - setupTez(confForTez); - runStatementOnDriver("drop table if exists T", confForTez); - runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')", confForTez); + runStatementOnDriver("drop table if exists T", hiveConf); + runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')", hiveConf); /*On Tez, below (T is transactional), we get the following layout ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree ~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/ /Users/ekoifman/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/ @@ -683,8 +678,8 @@ public void testAcidInsertWithRemoveUnion() throws Exception { └── bucket_00000 10 directories, 6 files */ - runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez); - List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", confForTez); + runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", hiveConf); + List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", hiveConf); LOG.warn(testName.getMethodName() + ": reading acid table T"); for(String s : rs) { LOG.warn(s); @@ -705,13 +700,11 @@ public void testAcidInsertWithRemoveUnion() throws Exception { } @Test public void testBucketedAcidInsertWithRemoveUnion() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - setupTez(confForTez); int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}}; - runStatementOnDriver("delete from " + Table.ACIDTBL, confForTez); + runStatementOnDriver("delete from " + Table.ACIDTBL, hiveConf); //make sure both buckets are not empty - runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values), confForTez); - runStatementOnDriver("drop table if exists T", confForTez); + runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values), hiveConf); + runStatementOnDriver("drop table if exists T", hiveConf); /* With bucketed target table Union All is not removed @@ -729,9 +722,9 @@ public void testBucketedAcidInsertWithRemoveUnion() throws Exception { 5 directories, 4 files */ - runStatementOnDriver("create table T (a int, b int) clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES ('transactional'='true')", confForTez); - runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez); - List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", confForTez); + runStatementOnDriver("create table T (a int, b int) clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES ('transactional'='true')", hiveConf); + runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", hiveConf); + List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", hiveConf); LOG.warn(testName.getMethodName() + ": reading bucketed acid table T"); for(String s : rs) { LOG.warn(s); @@ -753,15 +746,12 @@ public void testBucketedAcidInsertWithRemoveUnion() throws Exception { @Test public void testGetSplitsLocks() throws Exception { // Need to test this with LLAP settings, which requires some additional configurations set. - HiveConf modConf = new HiveConf(hiveConf); - setupTez(modConf); - modConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez"); - modConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); - modConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); + hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); + hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); // SessionState/Driver needs to be restarted with the Tez conf settings. - restartSessionAndDriver(modConf); - TxnStore txnHandler = TxnUtils.getTxnStore(modConf); + restartSessionAndDriver(hiveConf); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); try { // Request LLAP splits for a table. @@ -814,15 +804,12 @@ public void testGetSplitsLocks() throws Exception { @Test public void testGetSplitsLocksWithMaterializedView() throws Exception { // Need to test this with LLAP settings, which requires some additional configurations set. - HiveConf modConf = new HiveConf(hiveConf); - setupTez(modConf); - modConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez"); - modConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); - modConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); + hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); + hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); // SessionState/Driver needs to be restarted with the Tez conf settings. - restartSessionAndDriver(modConf); - TxnStore txnHandler = TxnUtils.getTxnStore(modConf); + restartSessionAndDriver(hiveConf); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); String mvName = "mv_acidTbl"; try { runStatementOnDriver("create materialized view " + mvName + " as select a from " + Table.ACIDTBL + " where a > 5"); @@ -865,14 +852,12 @@ public void testGetSplitsLocksWithMaterializedView() throws Exception { public void testCrudMajorCompactionSplitGrouper() throws Exception { String tblName = "test_split_grouper"; // make a clone of existing hive conf - HiveConf confForTez = new HiveConf(hiveConf); - setupTez(confForTez); // one-time setup to make query able to run with Tez - HiveConf.setVar(confForTez, HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); + HiveConf.setVar(hiveConf, HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); runStatementOnDriver("create transactional table " + tblName + " (a int, b int) clustered by (a) into 2 buckets " + "stored as ORC TBLPROPERTIES('bucketing_version'='2', 'transactional'='true'," - + " 'transactional_properties'='default')", confForTez); - runStatementOnDriver("insert into " + tblName + " values(1,2),(1,3),(1,4),(2,2),(2,3),(2,4)", confForTez); - runStatementOnDriver("insert into " + tblName + " values(3,2),(3,3),(3,4),(4,2),(4,3),(4,4)", confForTez); + + " 'transactional_properties'='default')", hiveConf); + runStatementOnDriver("insert into " + tblName + " values(1,2),(1,3),(1,4),(2,2),(2,3),(2,4)", hiveConf); + runStatementOnDriver("insert into " + tblName + " values(3,2),(3,3),(3,4),(4,2),(4,3),(4,4)", hiveConf); runStatementOnDriver("delete from " + tblName + " where b = 2"); List expectedRs = new ArrayList<>(); expectedRs.add("{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t2\t3"); @@ -884,10 +869,10 @@ public void testCrudMajorCompactionSplitGrouper() throws Exception { expectedRs.add("{\"writeid\":2,\"bucketid\":536936448,\"rowid\":1}\t4\t3"); expectedRs.add("{\"writeid\":2,\"bucketid\":536936448,\"rowid\":2}\t4\t4"); List rs = - runStatementOnDriver("select ROW__ID, * from " + tblName + " order by ROW__ID.bucketid, ROW__ID", confForTez); - HiveConf.setVar(confForTez, HiveConf.ConfVars.SPLIT_GROUPING_MODE, "compactor"); + runStatementOnDriver("select ROW__ID, * from " + tblName + " order by ROW__ID.bucketid, ROW__ID", hiveConf); + HiveConf.setVar(hiveConf, HiveConf.ConfVars.SPLIT_GROUPING_MODE, "compactor"); // No order by needed: this should use the compactor split grouping to return the rows in correct order - List rsCompact = runStatementOnDriver("select ROW__ID, * from " + tblName, confForTez); + List rsCompact = runStatementOnDriver("select ROW__ID, * from " + tblName, hiveConf); Assert.assertEquals("normal read", expectedRs, rs); Assert.assertEquals("compacted read", rs, rsCompact); } @@ -944,19 +929,15 @@ private void restartSessionAndDriver(HiveConf conf) throws Exception { // Ideally test like this should be a qfile test. However, the explain output from qfile is always // slightly different depending on where the test is run, specifically due to file size estimation private void testJoin(String engine, String joinType) throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf - HiveConf confForMR = new HiveConf(hiveConf); // make a clone of existing hive conf - - if (engine.equals("tez")) { - setupTez(confForTez); // one-time setup to make query able to run with Tez - } + HiveConf hiveConfMr = new HiveConf(hiveConf); // make a clone of existing hive conf + hiveConfMr.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); if (joinType.equals("MapJoin")) { - setupMapJoin(confForTez); - setupMapJoin(confForMR); + setupMapJoin(hiveConf); + setupMapJoin(hiveConfMr); } - runQueries(engine, joinType, confForTez, confForMR); + runQueries(engine, joinType, hiveConf, hiveConfMr); // Perform compaction. Join result after compaction should still be the same runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); @@ -967,7 +948,7 @@ private void testJoin(String engine, String joinType) throws Exception { Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); runCleaner(hiveConf); - runQueries(engine, joinType, confForTez, confForMR); + runQueries(engine, joinType, hiveConf, hiveConfMr); } private void runQueries(String engine, String joinType, HiveConf confForTez, HiveConf confForMR) throws Exception { @@ -1002,22 +983,6 @@ private void runQueries(String engine, String joinType, HiveConf confForTez, Hiv } } - public static void setupTez(HiveConf conf) { - conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez"); - conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, TEST_DATA_DIR); - conf.set("tez.am.resource.memory.mb", "128"); - conf.set("tez.am.dag.scheduler.class", "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled"); - conf.setBoolean("tez.local.mode", true); - conf.setBoolean("tez.local.mode.without.network", true); - conf.set("fs.defaultFS", "file:///"); - conf.setBoolean("tez.runtime.optimize.local.fetch", true); - conf.set("tez.staging-dir", TEST_DATA_DIR); - conf.setBoolean("tez.ignore.lib.uris", true); - conf.set("hive.tez.container.size", "128"); - conf.setBoolean("hive.merge.tezfiles", false); - conf.setBoolean("hive.in.tez.test", true); - } - private void setupMapJoin(HiveConf conf) { conf.setBoolVar(HiveConf.ConfVars.HIVE_CONVERT_JOIN, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_CONVERT_JOIN_NOCONDITIONALTASK, true); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java index 055688a3cc65..09f9d68a91d8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java @@ -29,24 +29,23 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hive.common.util.RetryTestRunner; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; -import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TestAutoPurgeTables { + private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; private static final String testDbName = "auto_purge_test_db"; //private static final String testTableName = "auto_purge_test_table"; @@ -112,7 +111,7 @@ private static void createTestTable(Statement stmt, String isAutopurge, boolean @BeforeClass public static void setUpBeforeClass() throws Exception { - conf = new HiveConf(TestAutoPurgeTables.class); + conf = new HiveConfForTest(TestAutoPurgeTables.class); // enable trash so it can be tested conf.setFloat("fs.trash.checkpoint.interval", 30); conf.setFloat("fs.trash.interval", 30); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestConstraintsMerge.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestConstraintsMerge.java index 1e1a8d74ae1a..747c5c1dbb0d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestConstraintsMerge.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestConstraintsMerge.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; import org.apache.hadoop.hive.ql.exec.errors.DataConstraintViolationError; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -38,8 +39,6 @@ import static org.apache.hadoop.hive.ql.TestAcidOnTez.TEST_DATA_DIR; import static org.apache.hadoop.hive.ql.TestAcidOnTez.TEST_WAREHOUSE_DIR; import static org.apache.hadoop.hive.ql.TestAcidOnTez.runStatementOnDriver; -import static org.apache.hadoop.hive.ql.TestAcidOnTez.setupTez; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -70,7 +69,7 @@ public String toString() { @Before public void setUp() throws Exception { - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(getClass()); hiveConf.set(ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(ConfVars.METASTORE_WAREHOUSE.varname, TEST_WAREHOUSE_DIR); @@ -134,19 +133,17 @@ public void tearDown() throws Exception { @Test public void testUpdateInMergeViolatesCheckConstraint() throws Exception { - HiveConf confForTez = new HiveConf(hiveConf); - confForTez.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); - setupTez(confForTez); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); runStatementOnDriver("insert into " + Table.TBL_CHECK_MERGE + "(name, age, gpa) values " + - "('student1', 16, 2.0)", confForTez); + "('student1', 16, 2.0)", hiveConf); Throwable error = null; try { runStatementOnDriver("merge into " + Table.TBL_CHECK_MERGE + " using (select age from table_source) source\n" + "on source.age=table_check_merge.age\n" + - "when matched then update set gpa=6", confForTez); + "when matched then update set gpa=6", hiveConf); } catch (Throwable t) { error = t; } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java index f77565a01d77..6963711af021 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.reexec.ReExecDriver; @@ -34,8 +35,7 @@ public class TestCreateUdfEntities { @Before public void setUp() throws Exception { - - HiveConf conf = new HiveConf(IDriver.class); + HiveConf conf = new HiveConfForTest(getClass()); SessionState.start(conf); driver = DriverFactory.newDriver(conf); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index 4792ca8c9486..2817074ed3ed 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -28,9 +28,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -79,7 +79,7 @@ public void setUp() throws Exception { } tests = new JUnit4TestAdapter(this.getClass()).countTestCases(); try { - conf = new HiveConf(ExecDriver.class); + conf = new HiveConfForTest(getClass()); SessionState.start(conf); // Test with remote metastore service diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java index f5cbd1636962..a64bba594f60 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -86,7 +87,7 @@ public String toString() { @Before public void setUp() throws Exception { - hiveConf = new HiveConf(this.getClass()); + hiveConf = new HiveConfForTest(getClass()); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.CREATE_TABLES_AS_ACID, true); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY, true); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index daf864210df3..0371ced62500 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -130,7 +130,7 @@ public void testAlterTablePartitionLocation_alter5() throws Exception { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { - qt[i] = new CheckResults(resDir, logDir, MiniClusterType.NONE, "parta"); + qt[i] = new CheckResults(resDir, logDir, MiniClusterType.TEZ_LOCAL, "parta"); qt[i].postInit(); qt[i].newSession(); qt[i].setInputFile(qfiles[i]); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java index a68f89cf50ac..9e9de7e93f8d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java @@ -33,6 +33,7 @@ import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HMSHandler; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hive.jdbc.miniHS2.MiniHS2; @@ -64,7 +65,7 @@ public class TestMetaStoreLimitPartitionRequest { @BeforeClass public static void beforeTest() throws Exception { Class.forName(MiniHS2.getJdbcDriverName()); - conf = new HiveConf(); + conf = new HiveConfForTest(TestMetaStoreLimitPartitionRequest.class); DriverManager.setLoginTimeout(0); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 6ae8239c667b..2b6a2180218e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.common.io.SessionStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; @@ -46,6 +47,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.tools.LineageInfo; import org.apache.hadoop.mapred.TextInputFormat; + import static org.junit.Assert.fail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -71,7 +73,7 @@ public class TestHiveHistory { @Before public void setUp() { try { - conf = new HiveConf(HiveHistory.class); + conf = getHiveConf(); SessionState.start(conf); fs = FileSystem.get(conf); @@ -136,7 +138,7 @@ public void testSimpleQuery() { LogUtils.initHiveLog4j(); } catch (LogInitializationException e) { } - HiveConf hconf = new HiveConf(SessionState.class); + HiveConf hconf = getHiveConf(); hconf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true); CliSessionState ss = new CliSessionState(hconf); ss.in = System.in; @@ -187,7 +189,7 @@ public void testQueryloglocParentDirNotExist() throws Exception { } try { String actualDir = parentTmpDir + "/test"; - HiveConf conf = new HiveConf(SessionState.class); + HiveConf conf = getHiveConf(); conf.set(HiveConf.ConfVars.HIVE_HISTORY_FILE_LOC.toString(), actualDir); SessionState ss = new CliSessionState(conf); HiveHistory hiveHistory = new HiveHistoryImpl(ss); @@ -209,7 +211,7 @@ public void testQueryloglocParentDirNotExist() throws Exception { */ @Test public void testHiveHistoryConfigEnabled() throws Exception { - HiveConf conf = new HiveConf(SessionState.class); + HiveConf conf = getHiveConf(); conf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true); SessionState ss = new CliSessionState(conf); SessionState.start(ss); @@ -223,7 +225,7 @@ public void testHiveHistoryConfigEnabled() throws Exception { */ @Test public void testHiveHistoryConfigDisabled() throws Exception { - HiveConf conf = new HiveConf(SessionState.class); + HiveConf conf = getHiveConf(); conf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, false); SessionState ss = new CliSessionState(conf); SessionState.start(ss); @@ -236,7 +238,10 @@ public void testHiveHistoryConfigDisabled() throws Exception { } - - - + private HiveConf getHiveConf() { + HiveConf conf = new HiveConfForTest(getClass()); + // to get consistent number of tasks in assertions + conf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); + return conf; + } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestAlterTableMetadata.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestAlterTableMetadata.java index 96aeb0f12c10..fdc51bee9465 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestAlterTableMetadata.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestAlterTableMetadata.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.metadata; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; @@ -35,7 +36,7 @@ public void testAlterTableOwner() throws HiveException, CommandProcessorExceptio * owner metadata of the table in HMS. */ - HiveConf conf = new HiveConf(this.getClass()); + HiveConf conf = new HiveConfForTest(getClass()); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); SessionState.start(conf); IDriver driver = DriverFactory.newDriver(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java index 594c8dcd52fe..9fa9655e0521 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/metadata/TestSemanticAnalyzerHookLoading.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; @@ -38,7 +39,7 @@ public class TestSemanticAnalyzerHookLoading { @Test public void testHookLoading() throws Exception { - HiveConf conf = new HiveConf(this.getClass()); + HiveConf conf = new HiveConfForTest(getClass()); conf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, DummySemanticAnalyzerHook.class.getName()); conf.set(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); SessionState.start(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java index f486fc35a059..0d7866b587f7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.shims.Utils; @@ -54,7 +55,9 @@ public class BaseReplicationAcrossInstances { static void internalBeforeClassSetup(Map overrides, Class clazz) throws Exception { - conf = new HiveConf(clazz); + conf = new HiveConfForTest(BaseReplicationAcrossInstances.class); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hive.repl.cmrootdir", "/tmp/"); conf.set("dfs.namenode.acls.enabled", "true"); @@ -86,7 +89,9 @@ static void internalBeforeClassSetupExclusiveReplica(Map primary throws Exception { // Setup replica HDFS. String replicaBaseDir = Files.createTempDirectory("replica").toFile().getAbsolutePath(); - replicaConf = new HiveConf(clazz); + replicaConf = new HiveConfForTest(clazz); + //TODO: HIVE-28044: Replication tests to run on Tez + replicaConf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); replicaConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, replicaBaseDir); replicaConf.set("dfs.client.use.datanode.hostname", "true"); MiniDFSCluster miniReplicaDFSCluster = @@ -94,7 +99,9 @@ static void internalBeforeClassSetupExclusiveReplica(Map primary // Setup primary HDFS. String primaryBaseDir = Files.createTempDirectory("base").toFile().getAbsolutePath(); - conf = new HiveConf(clazz); + conf = new HiveConfForTest(clazz); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, primaryBaseDir); conf.set("dfs.client.use.datanode.hostname", "true"); MiniDFSCluster miniPrimaryDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java index 90aa944fe4da..ecfaf4777b23 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest; import org.apache.hadoop.hive.metastore.api.OpenTxnRequest; import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse; @@ -41,7 +42,6 @@ import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.Utils; - import org.junit.rules.TestName; import org.junit.After; import org.junit.Assert; @@ -65,7 +65,6 @@ * TestReplicationScenariosAcidTablesBase - base class for replication for ACID tables tests */ public class BaseReplicationScenariosAcidTables { - @Rule public final TestName testName = new TestName(); @@ -81,7 +80,9 @@ public class BaseReplicationScenariosAcidTables { static void internalBeforeClassSetup(Map overrides, Class clazz) throws Exception { - conf = new HiveConf(clazz); + conf = new HiveConfForTest(clazz); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java index 17791f4604ec..041e00940b15 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hive.ql.parse; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -42,6 +42,7 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim; import static org.apache.hadoop.hive.common.repl.ReplConst.SOURCE_OF_REPLICATION; @@ -63,9 +64,6 @@ static class WarehouseInstanceWithMR extends WarehouseInstance { super(logger, cluster, overridesForHiveConf); HadoopShims shims = ShimLoader.getHadoopShims(); mrCluster = shims.getLocalMiniTezCluster(hiveConf, false); - // mrCluster = shims.getMiniMrCluster(hiveConf, 2, - // miniDFSCluster.getFileSystem().getUri().toString(), 1); - mrCluster.setupConfiguration(hiveConf); } @@ -80,7 +78,7 @@ public void close() throws IOException { @BeforeClass public static void classLevelSetup() throws Exception { - Configuration conf = new Configuration(); + HiveConf conf = new HiveConfForTest(TestCopyUtils.class); conf.set("dfs.client.use.datanode.hostname", "true"); UserGroupInformation ugi = Utils.getUGI(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java index 21490e4c3e7d..4376d6a86281 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.ql.parse; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.shims.Utils; import org.junit.AfterClass; import org.junit.Before; @@ -51,7 +51,7 @@ public class TestExportImport { @BeforeClass public static void classLevelSetup() throws Exception { - Configuration conf = new Configuration(); + HiveConf conf = new HiveConfForTest(TestExportImport.class); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); conf.set("hive.repl.include.external.tables", "false"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java index 2704e5a58a7e..090a596fabae 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java @@ -19,14 +19,13 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.events.AlterDatabaseEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.shims.Utils; - import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.TestName; @@ -62,7 +61,7 @@ public class TestMetaStoreEventListenerInRepl { @BeforeClass public static void internalBeforeClassSetup() throws Exception { - TestMetaStoreEventListenerInRepl.conf = new HiveConf(TestMetaStoreEventListenerInRepl.class); + TestMetaStoreEventListenerInRepl.conf = new HiveConfForTest(TestMetaStoreEventListenerInRepl.class); TestMetaStoreEventListenerInRepl.conf.set("dfs.client.use.datanode.hostname", "true"); TestMetaStoreEventListenerInRepl.conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java index 5ab19c0100df..d3c5c3063c96 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState; @@ -61,7 +62,7 @@ public static void beforeClassSetup() throws Exception { System.setProperty("jceks.key.serialFilter", "java.lang.Enum;java.security.KeyRep;" + "java.security.KeyRep$Type;javax.crypto.spec.SecretKeySpec;" + "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;!*"); - conf = new HiveConf(); + conf = new HiveConfForTest(TestParseUtilsStagingDir.class); conf.set("hadoop.security.key.provider.path", "jceks://file" + jksFile); miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java index dc22be2d9331..2e4443fbeab8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java @@ -25,8 +25,6 @@ import org.junit.rules.TestRule; import java.util.ArrayList; -import java.util.List; -import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithReadOnlyHook.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithReadOnlyHook.java index f81f99d2c8ea..6b292ee85601 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithReadOnlyHook.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithReadOnlyHook.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.shims.Utils; @@ -42,7 +43,9 @@ public static void classLevelSetup() throws Exception { overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), GzipJSONMessageEncoder.class.getCanonicalName()); - conf = new HiveConf(TestReplWithReadOnlyHook.class); + conf = new HiveConfForTest(TestReplWithReadOnlyHook.class); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationFilterTransactions.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationFilterTransactions.java index bb64f17c0b83..dd695e609a4d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationFilterTransactions.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationFilterTransactions.java @@ -22,15 +22,11 @@ import java.sql.ResultSet; import java.sql.Statement; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.permission.AclStatus; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -39,7 +35,6 @@ import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; import org.apache.hadoop.hive.ql.parse.repl.PathBuilder; -import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.shims.Utils; import org.junit.After; import org.junit.Assert; @@ -53,9 +48,7 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -72,13 +65,8 @@ public class TestReplicationFilterTransactions { static final private Logger LOG = LoggerFactory.getLogger(TestReplicationFilterTransactions.class); - private final static String tid = - TestReplicationFilterTransactions.class.getCanonicalName().toLowerCase().replace('.','_') + "_" + System.currentTimeMillis(); - private final static String TEST_PATH = - System.getProperty("test.warehouse.dir", "/tmp") + Path.SEPARATOR + tid; - @Rule - public TemporaryFolder tempFolder= new TemporaryFolder(); + public TemporaryFolder tempFolder = new TemporaryFolder(); // Event listener for the primary, mainly counts txn events. // Count totals are saved to static fields so they can be accessed @@ -263,12 +251,15 @@ public void setup() throws Throwable { Map conf = setupConf(miniDFSCluster.getFileSystem().getUri().toString(), PrimaryEventListenerTestImpl.class.getName()); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.put(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); primary = new WarehouseInstance(LOG, miniDFSCluster, conf); conf = setupConf(miniDFSCluster.getFileSystem().getUri().toString(), ReplicaEventListenerTestImpl.class.getName()); conf.put(MetastoreConf.ConfVars.REPLDIR.getHiveName(), primary.repldDir); - + //TODO: HIVE-28044: Replication tests to run on Tez + conf.put(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); replica = new WarehouseInstance(LOG, miniDFSCluster, conf); primaryDbName = testName.getMethodName() + "_" + System.currentTimeMillis(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java index ef83ec6c7928..f20d01badef5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.shims.Utils; @@ -69,7 +70,7 @@ public static void classLevelSetup() throws Exception { static void internalBeforeClassSetup(Map overrides, Class clazz) throws Exception { - HiveConf conf = new HiveConf(clazz); + HiveConf conf = new HiveConfForTest(TestReplicationOfHiveStreaming.class); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java index 92879d5ebba3..86972dd5c820 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java @@ -46,6 +46,7 @@ public class TestReplicationOnHDFSEncryptedZones { private static String jksFile = System.getProperty("java.io.tmpdir") + "/test.jks"; private static String jksFile2 = System.getProperty("java.io.tmpdir") + "/test2.jks"; + @Rule public final TestName testName = new TestName(); @@ -60,7 +61,8 @@ public static void beforeClassSetup() throws Exception { System.setProperty("jceks.key.serialFilter", "java.lang.Enum;java.security.KeyRep;" + "java.security.KeyRep$Type;javax.crypto.spec.SecretKeySpec;" + "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;!*"); - conf = new Configuration(); + conf = getNewConf(); + conf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); conf.set("hadoop.security.key.provider.path", "jceks://file" + jksFile); @@ -99,7 +101,7 @@ public void setup() throws Throwable { @Test public void targetAndSourceHaveDifferentEncryptionZoneKeys() throws Throwable { String replicaBaseDir = Files.createTempDirectory("replica").toFile().getAbsolutePath(); - Configuration replicaConf = new Configuration(); + Configuration replicaConf = getNewConf(); replicaConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, replicaBaseDir); replicaConf.set("dfs.client.use.datanode.hostname", "true"); replicaConf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); @@ -144,7 +146,7 @@ public void targetAndSourceHaveDifferentEncryptionZoneKeys() throws Throwable { @Test public void targetAndSourceHaveSameEncryptionZoneKeys() throws Throwable { String replicaBaseDir = Files.createTempDirectory("replica2").toFile().getAbsolutePath(); - Configuration replicaConf = new Configuration(); + Configuration replicaConf = getNewConf(); replicaConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, replicaBaseDir); replicaConf.set("dfs.client.use.datanode.hostname", "true"); replicaConf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); @@ -191,4 +193,11 @@ public void targetAndSourceHaveSameEncryptionZoneKeys() throws Throwable { .run("select value from encrypted_table") .verifyResults(new String[] { "value1", "value2" }); } + + private static Configuration getNewConf() { + Configuration conf = new Configuration(); + //TODO: HIVE-28044: Replication tests to run on Tez + conf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); + return conf; + } } \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 6ffaecab38d0..068119f11bb0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -193,6 +193,8 @@ public static void setUpBeforeClass() throws Exception { static void internalBeforeClassSetup(Map additionalProperties) throws Exception { hconf = new HiveConf(TestReplicationScenarios.class); + //TODO: HIVE-28044: Replication tests to run on Tez + hconf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); String metastoreUri = System.getProperty("test."+MetastoreConf.ConfVars.THRIFT_URIS.getHiveName()); if (metastoreUri != null) { hconf.set(MetastoreConf.ConfVars.THRIFT_URIS.getHiveName(), metastoreUri); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index dd501a1c5135..28ea5ca9a22a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.repl.ReplConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.ReplChangeManager; import org.apache.hadoop.hive.metastore.Warehouse; @@ -137,7 +138,7 @@ public static void classLevelSetup() throws Exception { static void internalBeforeClassSetup(Map overrides, Class clazz) throws Exception { - conf = new HiveConf(clazz); + conf = new HiveConfForTest(clazz); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("metastore.warehouse.tenant.colocation", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java index 7d6fcb10a68c..60772ca68b3f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreClientWithLocalCache; @@ -72,7 +73,7 @@ public static void classLevelSetup() throws Exception { static void internalBeforeClassSetup(Map overrides, Class clazz) throws Exception { - conf = new HiveConf(clazz); + conf = new HiveConfForTest(TestReplicationScenariosIncrementalLoadAcidTables.class); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java index 58fb5958b734..45ec72ef9501 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -92,7 +93,7 @@ static void internalBeforeClassSetup(Map primaryOverrides, Map replicaOverrides, Class clazz, boolean autogather, AcidTableKind acidTableKind) throws Exception { - conf = new HiveConf(clazz); + conf = new HiveConfForTest(TestStatsReplicationScenarios.class); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTimedOutTxnNotificationLogging.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTimedOutTxnNotificationLogging.java index 130e4908b3c2..62c35c615d35 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTimedOutTxnNotificationLogging.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTimedOutTxnNotificationLogging.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.repl.ReplScope; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetastoreTaskThread; @@ -98,7 +99,7 @@ public void setUp() throws Exception { } private void setConf() { - hiveConf = new HiveConf(); + hiveConf = new HiveConfForTest(getClass()); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); MetastoreConf.setVar(hiveConf, MetastoreConf.ConfVars.WAREHOUSE, "/tmp"); MetastoreConf.setTimeVar(hiveConf, MetastoreConf.ConfVars.TXN_TIMEOUT, 1, TimeUnit.SECONDS); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index 0235aef1d54b..b9fbf61699cd 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; @@ -65,7 +66,7 @@ public void setUp() throws Exception { int port = MetaStoreTestUtils.startMetaStoreWithRetry(); - clientHiveConf = new HiveConf(this.getClass()); + clientHiveConf = new HiveConfForTest(this.getClass()); clientHiveConf.setVar(HiveConf.ConfVars.METASTORE_URIS, "thrift://localhost:" + port); clientHiveConf.setIntVar(HiveConf.ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES, 3); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index 120d967a4754..a664b07755da 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -25,17 +25,21 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.exec.tez.ObjectCache; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertFalse; @@ -62,16 +66,13 @@ protected String getAuthorizationProvider(){ @Before public void setUp() throws Exception { - - - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); int port = MetaStoreTestUtils.startMetaStoreWithRetry(); - clientHiveConf = new HiveConf(this.getClass()); + clientHiveConf = new HiveConfForTest(this.getClass()); // Turn on client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,true); @@ -93,6 +94,9 @@ public void setUp() throws Exception { SessionState.start(new CliSessionState(clientHiveConf)); msc = new HiveMetaStoreClient(clientHiveConf); driver = DriverFactory.newDriver(clientHiveConf); + + // this test involves limit operator which needs an object cache + ObjectCache.setupObjectRegistry(new ObjectRegistryImpl()); } @After diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index 72a953fea425..e71ecde8187a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; @@ -82,7 +83,7 @@ protected String getAuthorizationProvider(){ } protected HiveConf createHiveConf() throws Exception { - return new HiveConf(this.getClass()); + return new HiveConfForTest(getClass()); } protected String getProxyUserName() { @@ -91,9 +92,6 @@ protected String getProxyUserName() { @Before public void setUp() throws Exception { - - - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreClientSideAuthorizationProvider.java index b166df33b8bd..5f69ce86cb4b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreClientSideAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreClientSideAuthorizationProvider.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.TableMeta; @@ -54,7 +55,7 @@ public void setUp() throws Exception { int port = MetaStoreTestUtils.startMetaStoreWithRetry(); - clientHiveConf = new HiveConf(this.getClass()); + clientHiveConf = new HiveConfForTest(this.getClass()); // Turn on client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,true); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 1a5a840c8680..fff8d0b38d7c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; @@ -60,7 +61,7 @@ public static void setUp() throws Exception { int port = MetaStoreTestUtils.startMetaStoreWithRetry(); - clientHiveConf = new HiveConf(); + clientHiveConf = new HiveConfForTest(TestMultiAuthorizationPreEventListener.class); clientHiveConf.setVar(HiveConf.ConfVars.METASTORE_URIS, "thrift://localhost:" + port); clientHiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java index 0d939af9de46..71b25c565649 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java @@ -38,6 +38,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.QueryState; @@ -97,7 +98,7 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC @BeforeClass public static void beforeTest() throws Exception { - conf = new HiveConf(); + conf = new HiveConfForTest(TestHiveAuthorizerCheckInvocation.class); // Turn on mocked authorization conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java index 36ac85b73095..5633ae017548 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.UtilsForTest; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; @@ -113,7 +114,7 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC @BeforeClass public static void beforeTest() throws Exception { - conf = new HiveConf(); + conf = new HiveConfForTest(TestHiveAuthorizerShowFilters.class); // Turn on mocked authorization conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java index 8e09d9697f4a..8557946331c3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.exec.Utilities; import org.junit.AfterClass; import org.junit.Assert; @@ -51,7 +52,7 @@ public class TestClearDanglingScratchDir { @BeforeClass static public void oneTimeSetup() throws Exception { m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build(); - conf = new HiveConf(); + conf = new HiveConfForTest(TestClearDanglingScratchDir.class); conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true"); conf.set(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true"); LoggerFactory.getLogger("SessionState"); @@ -145,7 +146,7 @@ public void testClearDanglingScratchDir() throws Exception { */ @Test public void testLocalDanglingFilesCleaning() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.set("fs.default.name", "file:///"); String tmpDir = System.getProperty("test.tmp.dir"); conf.set("hive.exec.scratchdir", tmpDir + "/hive-27317-hdfsscratchdir"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java index f619e6f7d2a2..8b462462893d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.session; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; @@ -32,7 +33,7 @@ public class TestUdfClassLoaderAcrossSessions { @Test public void testDropDatabaseCascadeDoesNotThrow() throws CommandProcessorException, IOException { - HiveConf conf = new HiveConf(this.getClass()); + HiveConf conf = new HiveConfForTest(this.getClass()); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); SessionState.start(conf); IDriver driver = DriverFactory.newDriver(conf); @@ -48,7 +49,7 @@ public void testDropDatabaseCascadeDoesNotThrow() throws CommandProcessorExcepti @Test public void testDropFunctionDoesNotThrow() throws CommandProcessorException, IOException { - HiveConf conf = new HiveConf(this.getClass()); + HiveConf conf = new HiveConfForTest(this.getClass()); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); SessionState.start(conf); IDriver driver = DriverFactory.newDriver(conf); @@ -65,7 +66,7 @@ public void testDropFunctionDoesNotThrow() throws CommandProcessorException, IOE @Test public void testUseBeforeDropDatabaseCascadeDoesNotThrow() throws CommandProcessorException, IOException { - HiveConf conf = new HiveConf(this.getClass()); + HiveConf conf = new HiveConfForTest(this.getClass()); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); SessionState.start(conf); IDriver driver = DriverFactory.newDriver(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java index d91245414608..b606515391c0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java @@ -81,6 +81,8 @@ public void setup() throws Exception { } HiveConf hiveConf = new HiveConf(this.getClass()); + // this test is mr specific, for Tez-based compaction look at CompactorOnTezTest + hiveConf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); hiveConf.setVar(HiveConf.ConfVars.PRE_EXEC_HOOKS, ""); hiveConf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, ""); hiveConf.setVar(HiveConf.ConfVars.METASTORE_WAREHOUSE, TEST_WAREHOUSE_DIR); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 821f504bac22..1fa698770135 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; @@ -101,6 +102,7 @@ * */ public class TestJdbcDriver2 { + private static final Logger LOG = LoggerFactory.getLogger(TestJdbcDriver2.class); private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; private static final String testDbName = "testjdbcdriver"; @@ -118,7 +120,7 @@ public class TestJdbcDriver2 { private static final String dataTypeTableComment = "Table with many column data types"; private static final String externalTableName = "testjdbcdriverexttbl"; private static final String externalTableComment = "An external table"; - private static HiveConf conf; + private static HiveConfForTest conf; private static String dataFileDir; private static Path dataFilePath; private static int dataFileRowCount; @@ -130,9 +132,13 @@ public class TestJdbcDriver2 { @Rule public ExpectedException thrown = ExpectedException.none(); @Rule public final TestName testName = new TestName(); - private static Connection getConnection(String postfix) throws SQLException { + private static Connection getConnection(String prefix, String postfix) throws SQLException { Connection con1; - con1 = DriverManager.getConnection("jdbc:hive2:///" + postfix, "", ""); + String connString = "jdbc:hive2:///" + prefix + "?" + conf.getOverlayOptionsAsQueryString() + + ";hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;" + postfix; + LOG.info("Connection string: {}", connString); + con1 = DriverManager + .getConnection(connString, "", ""); assertNotNull("Connection is null", con1); assertFalse("Connection should not be closed", con1.isClosed()); return con1; @@ -193,10 +199,9 @@ private static void createTestTables(Statement stmt, String testDbName) throws S @SuppressWarnings("deprecation") @BeforeClass public static void setUpBeforeClass() throws Exception { - conf = new HiveConf(TestJdbcDriver2.class); - HiveConf initConf = new HiveConf(conf); - TestTxnDbUtil.setConfValues(initConf); - TestTxnDbUtil.prepDb(initConf); + conf = new HiveConfForTest(TestJdbcDriver2.class); + TestTxnDbUtil.setConfValues(conf); + TestTxnDbUtil.prepDb(conf); dataFileDir = conf.get("test.data.files").replace('\\', '/') .replace("c:", ""); dataFilePath = new Path(dataFileDir, "kv1.txt"); @@ -211,7 +216,8 @@ public static void setUpBeforeClass() throws Exception { System.setProperty(ConfVars.HIVE_SERVER2_PARALLEL_OPS_IN_SESSION.varname, "false"); System.setProperty(ConfVars.REPL_CM_ENABLED.varname, "true"); System.setProperty(ConfVars.REPL_CM_DIR.varname, "cmroot"); - con = getConnection(defaultDbName + ";create=true"); + + con = getConnection(defaultDbName + ";create=true", ""); Statement stmt = con.createStatement(); assertNotNull("Statement is null", stmt); stmt.execute("set hive.support.concurrency = false"); @@ -224,7 +230,7 @@ public static void setUpBeforeClass() throws Exception { @Test - public void testExceucteUpdateCounts() throws Exception { + public void testExecuteUpdateCounts() throws Exception { Statement stmt = con.createStatement(); stmt.execute("set " + ConfVars.HIVE_SUPPORT_CONCURRENCY.varname + "=true"); stmt.execute("set " + ConfVars.HIVE_TXN_MANAGER.varname + @@ -255,16 +261,16 @@ public void testExceucteUpdateCounts() throws Exception { } @Test - public void testExceucteMergeCounts() throws Exception { - testExceucteMergeCounts(true); + public void testExecuteMergeCounts() throws Exception { + testExecuteMergeCounts(true); } @Test - public void testExceucteMergeCountsNoSplitUpdate() throws Exception { - testExceucteMergeCounts(false); + public void testExecuteMergeCountsNoSplitUpdate() throws Exception { + testExecuteMergeCounts(false); } - private void testExceucteMergeCounts(boolean splitUpdateEarly) throws Exception { + private void testExecuteMergeCounts(boolean splitUpdateEarly) throws Exception { Statement stmt = con.createStatement(); stmt.execute("set " + ConfVars.SPLIT_UPDATE.varname + "=" + splitUpdateEarly); @@ -332,7 +338,7 @@ private void checkBadUrl(String url) throws SQLException { * @throws SQLException */ public void testURLWithFetchSize() throws SQLException { - Connection con = getConnection(testDbName + ";fetchSize=1234"); + Connection con = getConnection(testDbName + ";fetchSize=1234", ""); Statement stmt = con.createStatement(); assertEquals(stmt.getFetchSize(), 1234); stmt.close(); @@ -345,7 +351,7 @@ public void testURLWithFetchSize() throws SQLException { * @throws SQLException */ public void testCreateTableAsExternal() throws SQLException { - Connection con = getConnection(testDbName + ";hiveCreateAsExternalLegacy=true"); + Connection con = getConnection(testDbName + ";hiveCreateAsExternalLegacy=true", ""); Statement stmt = con.createStatement(); ResultSet res = stmt.executeQuery("set hive.create.as.external.legacy"); assertTrue("ResultSet is empty", res.next()); @@ -701,7 +707,7 @@ public void testExecutePreparedStatement() throws Exception { @Test public void testSetOnConnection() throws Exception { - Connection connection = getConnection(testDbName + "?conf1=conf2;conf3=conf4#var1=var2;var3=var4"); + Connection connection = getConnection(testDbName, "conf1=conf2;conf3=conf4#var1=var2;var3=var4"); try { verifyConfValue(connection, "conf1", "conf2"); verifyConfValue(connection, "conf3", "conf4"); @@ -1632,7 +1638,7 @@ public void testClientInfo() throws SQLException { fail(msg); } - Connection conn = getConnection(""); + Connection conn = getConnection("", ""); try { conn.setClientInfo("ApplicationName", "test"); assertEquals("test", conn.getClientInfo("ApplicationName")); @@ -3063,7 +3069,7 @@ public void testAutoCommit() throws Exception { @Test public void setAutoCommitOnClosedConnection() throws Exception { - Connection mycon = getConnection(""); + Connection mycon = getConnection("", ""); try { mycon.setAutoCommit(true); mycon.close(); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHA.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHA.java index 26ea18607557..aafa101a3541 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHA.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHA.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.session.HiveSessionHook; @@ -67,7 +68,7 @@ public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLExcepti @BeforeClass public static void beforeTest() throws Exception { Class.forName(MiniHS2.getJdbcDriverName()); - conf = new HiveConf(); + conf = new HiveConfForTest(TestJdbcWithMiniHA.class); conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); String dataFileDir = conf.get("test.data.files").replace('\\', '/') .replace("c:", ""); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 7f03c0ee0c5f..6f8110aa9d59 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -67,6 +67,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.PersistenceManagerProvider; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -104,7 +105,7 @@ public class TestJdbcWithMiniHS2 { @BeforeClass public static void setupBeforeClass() throws Exception { MiniHS2.cleanupLocalDir(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", ""); kvDataFilePath = new Path(dataFileDir, "kv1.txt"); @@ -202,7 +203,7 @@ private static void restoreMiniHS2AndConnections() throws Exception { } } stopMiniHS2(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); startMiniHS2(conf); openDefaultConnections(); openTestConnections(); @@ -788,7 +789,7 @@ private void unsetSerializeInTasksInConf(Statement stmt) throws SQLException { public void testSessionScratchDirs() throws Exception { // Stop HiveServer2 stopMiniHS2(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); String userName; Path scratchDirPath; // Set a custom prefix for hdfs scratch dir path @@ -856,7 +857,7 @@ public void testSessionScratchDirs() throws Exception { */ @Test public void testUdfWhiteBlackList() throws Exception { - HiveConf testConf = new HiveConf(); + HiveConf testConf = getNewHiveConf(); assertTrue(testConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST).isEmpty()); // verify that udf in default whitelist can be executed Statement stmt = conDefault.createStatement(); @@ -892,6 +893,13 @@ public void testUdfWhiteBlackList() throws Exception { restoreMiniHS2AndConnections(); } + private static HiveConf getNewHiveConf() { + HiveConf conf = new HiveConfForTest(TestJdbcWithMiniHS2.class); + //TODO: HIVE-28284: TestJdbcWithMiniHS2/TestJdbcWithMiniHS2ErasureCoding to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); + return conf; + } + /** Test UDF blacklist * - verify default value * - verify udfs allowed with default blacklist @@ -900,7 +908,7 @@ public void testUdfWhiteBlackList() throws Exception { */ @Test public void testUdfBlackList() throws Exception { - HiveConf testConf = new HiveConf(); + HiveConf testConf = getNewHiveConf(); assertTrue(testConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST).isEmpty()); Statement stmt = conDefault.createStatement(); // verify that udf in default whitelist can be executed @@ -931,7 +939,7 @@ public void testUdfBlackList() throws Exception { public void testUdfBlackListOverride() throws Exception { stopMiniHS2(); // setup whitelist - HiveConf testConf = new HiveConf(); + HiveConf testConf = getNewHiveConf(); Set funcNames = FunctionRegistry.getFunctionNames(); String funcNameStr = ""; @@ -967,7 +975,7 @@ public void testUdfBlackListOverride() throws Exception { public void testRootScratchDir() throws Exception { // Stop HiveServer2 stopMiniHS2(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); String userName; Path scratchDirPath; conf.set("hive.exec.scratchdir", tmpDir + "/hs2"); @@ -1017,7 +1025,7 @@ private void verifyScratchDir(HiveConf conf, FileSystem fs, Path scratchDirPath, public void testHttpHeaderSize() throws Exception { // Stop HiveServer2 stopMiniHS2(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE, 1024); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE, 1024); startMiniHS2(conf, true); @@ -1087,7 +1095,7 @@ public void testHttpHeaderSize() throws Exception { public void testHttpRetryOnServerIdleTimeout() throws Exception { // Stop HiveServer2 stopMiniHS2(); - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); // Set server's idle timeout to a very low value conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, "5000"); startMiniHS2(conf, true); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java index 1923e262fa48..3d5ddaa08c11 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java @@ -31,6 +31,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.processors.ErasureProcessor; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.HadoopShims.HdfsErasureCodingShim; @@ -67,7 +68,9 @@ public class TestJdbcWithMiniHS2ErasureCoding { @BeforeClass public static void beforeTest() throws Exception { Class.forName(MiniHS2.getJdbcDriverName()); - conf = new HiveConf(); + conf = new HiveConfForTest(TestJdbcWithMiniHS2ErasureCoding.class); + //TODO: HIVE-28284: TestJdbcWithMiniHS2/TestJdbcWithMiniHS2ErasureCoding to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); DriverManager.setLoginTimeout(0); miniHS2 = new MiniHS2.Builder() diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestCLIAuthzSessionContext.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestCLIAuthzSessionContext.java index fa359f9f8f3b..04b3b5b5e46b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestCLIAuthzSessionContext.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestCLIAuthzSessionContext.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.common.io.SessionStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; @@ -60,7 +61,7 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC @BeforeClass public static void beforeTest() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(TestCLIAuthzSessionContext.class); conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); conf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER, SessionStateUserAuthenticator.class.getName()); conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthUDFBlacklist.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthUDFBlacklist.java index 8130c51413bf..99df3efc453c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthUDFBlacklist.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthUDFBlacklist.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator; import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory; import org.apache.hive.jdbc.miniHS2.MiniHS2; @@ -64,7 +65,7 @@ public void shutDownHS2() throws Exception { @Test public void testBlackListedUdfUsage() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST, "sqrt"); startHS2(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java index 850fa243d9da..f9e68714331a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.UtilsForTest; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; @@ -37,7 +38,7 @@ public class TestEmbeddedThriftBinaryCLIService extends CLIServiceTest { @BeforeClass public static void setUpBeforeClass() throws Exception { service = new EmbeddedThriftBinaryCLIService(); - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(TestEmbeddedThriftBinaryCLIService.class); conf.setBoolean("datanucleus.schema.autoCreateTables", true); conf.setVar(HiveConf.ConfVars.HIVE_MAPRED_MODE, "nonstrict"); UtilsForTest.expandHiveConfParams(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java index 66325b128c9c..80b64b6861b4 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java @@ -64,6 +64,8 @@ public static void setUpBeforeClass() throws Exception { }; hiveConf = new HiveConf(); hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); + // this test is mr specific + hiveConf.set(ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); miniHS2 = new MiniHS2(hiveConf); confOverlay = new HashMap(); confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java index e2a7d3385fa5..8e0c4c8957fb 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java @@ -24,8 +24,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.log.HushableRandomAccessFileAppender; import org.apache.hadoop.hive.ql.log.LogDivertAppender; import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest; @@ -69,7 +69,7 @@ public class TestOperationLoggingLayout { @BeforeClass public static void setUpBeforeClass() throws Exception { tableName = "TestOperationLoggingLayout_table"; - hiveConf = new HiveConf(); + hiveConf = new HiveConfForTest(TestOperationLoggingLayout.class); hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "execution"); miniHS2 = new MiniHS2(hiveConf); confOverlay = new HashMap(); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java index 86425ad7754a..c37665c1a31b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java @@ -18,6 +18,7 @@ package org.apache.hive.service.cli.session; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.QueryDisplay; import org.apache.hadoop.hive.ql.QueryInfo; import org.apache.hadoop.hive.ql.exec.Task; @@ -49,7 +50,7 @@ public class TestQueryDisplay { @Before public void setup() { - conf = new HiveConf(); + conf = new HiveConfForTest(getClass()); conf.set("hive.support.concurrency", "false"); sessionManager = new SessionManager(null, true); @@ -198,14 +199,13 @@ private void testGraphDDL(HiveSession session, boolean exceedMaxGraphSize) throw } /** - * Test for the HiveConf option HIVE_SERVER2_WEBUI_SHOW_STATS, which is available for MapReduce + * Test for the HiveConf option HIVE_SERVER2_WEBUI_SHOW_STATS, which is available for Tez * jobs only. */ @Test public void checkWebUIShowStats() throws Exception { // WebUI-related boolean confs must be set before build. HIVE_SERVER2_WEBUI_SHOW_STATS depends // on HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT and HIVE_SERVER2_WEBUI_SHOW_GRAPH being set to true. - conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_SHOW_GRAPH, true); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_GRAPH_SIZE, 40); @@ -220,12 +220,12 @@ public void checkWebUIShowStats() throws Exception { OperationHandle opHandleSetup = session.executeStatement("CREATE TABLE statsTable (i int)", null); session.closeOperation(opHandleSetup); - OperationHandle opHandleMrQuery = + OperationHandle opHandleQuery = session.executeStatement("INSERT INTO statsTable VALUES (0)", null); - session.closeOperation(opHandleMrQuery); + session.closeOperation(opHandleQuery); - // INSERT queries include a MapReduce task. - verifyDDLHtml("Counters", opHandleMrQuery.getHandleIdentifier().toString(), true); + // INSERT queries include a Tez task. + verifyDDLHtml("DagId", opHandleQuery.getHandleIdentifier().toString(), true); session.close(); resetConfToDefaults(); @@ -241,7 +241,7 @@ private void resetConfToDefaults() { private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean finished) { Assert.assertEquals(queryInfo.getUserName(), "testuser"); - Assert.assertEquals(queryInfo.getExecutionEngine(), "mr"); + Assert.assertEquals(queryInfo.getExecutionEngine(), "tez"); Assert.assertEquals(queryInfo.getOperationId(), handle); Assert.assertTrue(queryInfo.getBeginTime() > 0 && queryInfo.getBeginTime() <= System.currentTimeMillis()); @@ -300,9 +300,10 @@ private void verifyDDLHtml(String stmt, String opHandle, boolean assertCondition HiveConf hiveConf = sessionManager.getOperationManager().getHiveConf(); new QueryProfileTmpl().render(sw, queryInfo, hiveConf); String html = sw.toString(); - Assert.assertEquals(assertCondition, html.contains(stmt)); + Assert.assertEquals("HTML assert condition hasn't met, statement: " + stmt + ", html: " + html, assertCondition, + html.contains(stmt)); - Assert.assertTrue(html.contains("testuser")); + Assert.assertTrue("HTML doesn't contain and expected string, got: " + html, html.contains("testuser")); } static class MyTask extends Task { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java index 6c77e0fe881e..f71aa4fa8880 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; @@ -193,7 +194,7 @@ public static void setupInternal(boolean zookeeperSSLEnabled) throws Exception { zkCluster = new MiniZooKeeperCluster(zookeeperSSLEnabled); int zkPort = zkCluster.startup(zkDataDir); - miniHS2 = new MiniHS2(new HiveConf()); + miniHS2 = new MiniHS2(new HiveConfForTest(InformationSchemaWithPrivilegeTestBase.class)); confOverlay = new HashMap(); Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); @@ -219,7 +220,7 @@ public static void setupInternal(boolean zookeeperSSLEnabled) throws Exception { if(zookeeperSSLEnabled) { String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty("test.data.files") : - (new HiveConf()).get("test.data.files").replace('\\', '/').replace("c:", ""); + (new HiveConfForTest(InformationSchemaWithPrivilegeTestBase.class)).get("test.data.files").replace('\\', '/').replace("c:", ""); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION.varname, dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname, diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java index 983e8afd6cd9..61fa16a13ea9 100755 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java @@ -49,6 +49,8 @@ public static void setupBeforeClass() throws Exception { MiniHS2.cleanupLocalDir(); try { HiveConf conf = new HiveConf(); + //TODO: HIVE-28296: TestGracefulStopHS2 to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java index ff36e24b2d13..e5e1c10b74a3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.Utils; import org.junit.Assert; @@ -34,7 +35,7 @@ public class TestHS2ClearDanglingScratchDir { @Test public void testScratchDirCleared() throws Exception { MiniDFSCluster m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build(); - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.addResource(m_dfs.getConfiguration(0)); conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true"); conf.set(HiveConf.ConfVars.HIVE_SERVER2_CLEAR_DANGLING_SCRATCH_DIR.toString(), "true"); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java index acf64b0dc910..34848eeae376 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java @@ -26,15 +26,15 @@ */ public class AccumuloQTestUtil extends QTestUtil { - public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, - AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception { + public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, AccumuloTestSetup setup, + String hiveConfDir, String initScript, String cleanupScript) throws Exception { super( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(outDir) .withLogDir(logDir) .withClusterType(miniMr) - .withConfDir(null) + .withConfDir(hiveConfDir) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 5705e5f38b77..8c3ee72c0c10 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -57,7 +57,7 @@ public CliConfig() { setInitScript("q_test_init.sql"); setCleanupScript("q_test_cleanup.sql"); - setHiveConfDir(""); + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -77,7 +77,8 @@ public ParseNegativeConfig() { setInitScript("q_test_init_parse.sql"); setCleanupScript("q_test_cleanup.sql"); - setHiveConfDir("data/conf/perf-reg/"); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -99,7 +100,7 @@ public MinimrCliConfig() { setInitScript("q_test_init_for_minimr.sql"); setCleanupScript("q_test_cleanup.sql"); - setHiveConfDir(""); + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.MR); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -289,13 +290,9 @@ public EncryptedHDFSCliConfig() { setClusterType(MiniClusterType.MR); - setFsType(QTestMiniClusters.FsType.ENCRYPTED_HDFS); // override default FsType.HDFS - if (getClusterType() == MiniClusterType.TEZ) { - setHiveConfDir("data/conf/tez"); - } else { - setHiveConfDir("data/conf"); - } - + setFsType(QTestMiniClusters.FsType.ENCRYPTED_HDFS); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } @@ -314,7 +311,8 @@ public ContribCliConfig() { setInitScript("q_test_init_contrib.sql"); setCleanupScript("q_test_cleanup_contrib.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } @@ -397,7 +395,8 @@ public HBaseCliConfig() { setInitScript("q_test_init_src_with_stats.sql"); setCleanupScript("q_test_cleanup_src.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -417,7 +416,8 @@ public HBaseNegativeCliConfig() { setInitScript("q_test_init_src.sql"); setCleanupScript("q_test_cleanup_src.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -437,7 +437,8 @@ public ContribNegativeCliConfig() { setInitScript("q_test_init.sql"); setCleanupScript("q_test_cleanup.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -460,7 +461,8 @@ public BeeLineConfig() { setInitScript("q_test_init_src.sql"); setCleanupScript("q_test_cleanup_src.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -480,7 +482,8 @@ public AccumuloCliConfig() { setInitScript("q_test_init_src_with_stats.sql"); setCleanupScript("q_test_cleanup_src.sql"); - setHiveConfDir(""); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); setClusterType(MiniClusterType.NONE); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); @@ -562,7 +565,8 @@ private void setHiveConfDir(MiniClusterType clusterType) { setHiveConfDir("data/conf/tez"); break; default: - setHiveConfDir("data/conf"); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + setHiveConfDir("data/conf/mr"); break; } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 1b634f2be135..f950af20abae 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -44,11 +44,12 @@ public CoreAccumuloCliDriver(AbstractCliConfig cliConfig) { @BeforeClass public void beforeClass() throws Exception { MiniClusterType miniMR = cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); qt = new AccumuloQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new AccumuloTestSetup(), - initScript, cleanupScript); + hiveConfDir, initScript, cleanupScript); } @Override diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java index 4196133025c5..9bdd63d675a6 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java @@ -58,6 +58,7 @@ import com.google.common.collect.ObjectArrays; public class CoreBeeLineDriver extends CliAdapter { + private final File hiveRootDirectory = new File(AbstractCliConfig.HIVE_ROOT); private final File queryDirectory; private final File logDirectory; @@ -107,6 +108,8 @@ public CoreBeeLineDriver(AbstractCliConfig testCliConfig) { private static MiniHS2 createMiniServer() throws Exception { HiveConf hiveConf = new HiveConf(); + // TODO: HIVE-28031: Adapt some cli driver tests to Tez where it's applicable + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); // We do not need Zookeeper at the moment hiveConf.set(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index db23405e786d..07ecde9aa168 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -43,11 +43,12 @@ public CoreHBaseCliDriver(AbstractCliConfig testCliConfig) { @BeforeClass public void beforeClass() throws Exception { MiniClusterType miniMR = cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); - qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new HBaseTestSetup(), initScript, - cleanupScript); + qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new HBaseTestSetup(), hiveConfDir, + initScript, cleanupScript); } @Override diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index 0af634307d50..9f2752a30781 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -42,11 +42,12 @@ public CoreHBaseNegativeCliDriver(AbstractCliConfig testCliConfig) { @Override public void beforeClass() throws Exception { MiniClusterType miniMR = cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); - qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new HBaseTestSetup(), initScript, - cleanupScript); + qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new HBaseTestSetup(), hiveConfDir, + initScript, cleanupScript); } @Override diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 94b72f7fe79e..77e86673f0cf 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -38,17 +38,15 @@ public class HBaseQTestUtil extends QTestUtil { /** A handle to this harness's cluster */ private final Connection conn; - public HBaseQTestUtil( - String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup, - String initScript, String cleanupScript) - throws Exception { + public HBaseQTestUtil(String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup, String hiveConfDir, + String initScript, String cleanupScript) throws Exception { super( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(outDir) .withLogDir(logDir) .withClusterType(miniMr) - .withConfDir(null) + .withConfDir(hiveConfDir) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index d8d35bf55625..44a001186651 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -47,6 +47,7 @@ public CoreParseNegative(AbstractCliConfig testCliConfig) { @BeforeClass public void beforeClass() throws Exception { MiniClusterType miniMR = cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); @@ -55,7 +56,7 @@ public void beforeClass() throws Exception { .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) - .withConfDir(null) + .withConfDir(hiveConfDir) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) diff --git a/packaging/src/docker/conf/hive-site.xml b/packaging/src/docker/conf/hive-site.xml index b7eec691f268..53ac86e49a9e 100644 --- a/packaging/src/docker/conf/hive-site.xml +++ b/packaging/src/docker/conf/hive-site.xml @@ -52,10 +52,6 @@ tez.local.mode true - - hive.execution.engine - tez - hive.metastore.warehouse.dir /opt/hive/data/warehouse diff --git a/pom.xml b/pom.xml index d69626e0f289..da80147e90ce 100644 --- a/pom.xml +++ b/pom.xml @@ -1334,11 +1334,36 @@ org.apache.tez tez-mapreduce ${tez.version} + true io.netty netty + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop + hadoop-hdfs + + + org.apache.hadoop + hadoop-yarn-client + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HivePreWarmProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HivePreWarmProcessor.java index b6c0d7f3d875..a9dcfb0845f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HivePreWarmProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HivePreWarmProcessor.java @@ -86,7 +86,7 @@ public void run(Map inputs, ReadaheadPool rpool = ReadaheadPool.getInstance(); ShimLoader.getHadoopShims(); - URL hiveurl = new URL("jar:" + DagUtils.getInstance().getExecJarPathLocal(conf) + "!/"); + URL hiveurl = new URL("jar:file:" + DagUtils.getInstance().getExecJarPathLocal(conf) + "!/"); JarURLConnection hiveconn = (JarURLConnection)hiveurl.openConnection(); JarFile hivejar = hiveconn.getJarFile(); try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index c2a9ae5203b1..245c4251eacb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -283,7 +283,7 @@ protected void openInternal(String[] additionalFilesNotFromConf, } else { this.resources = new HiveResources(createTezDir(sessionId, "resources")); ensureLocalResources(conf, additionalFilesNotFromConf); - LOG.info("Created new resources: " + resources); + LOG.info("Created new resources: " + this.resources); } // unless already installed on all the cluster nodes, we'll have to diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 57f2a0c05f50..2675ae61fa69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -141,6 +141,14 @@ public void setTezCounters(final TezCounters counters) { this.counters = counters; } + /** + * Making TezTask backward compatible with the old MR-based Task API (ExecDriver/MapRedTask) + */ + @Override + public String getExternalHandle() { + return this.jobID; + } + @Override public int execute() { int rc = 1; @@ -256,6 +264,7 @@ public int execute() { LOG.info("HS2 Host: [{}], Query ID: [{}], Dag ID: [{}], DAG Session ID: [{}]", ServerUtils.hostname(), queryId, dagId, this.dagClient.getSessionIdentifierString()); LogUtils.putToMDC(LogUtils.DAGID_KEY, dagId); + this.jobID = dagId; // finally monitor will print progress until the job is done TezJobMonitor monitor = new TezJobMonitor(work.getAllWork(), dagClient, conf, dag, ctx, counters, perfLogger); @@ -268,7 +277,10 @@ public int execute() { try { // fetch the counters Set statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS); - TezCounters dagCounters = dagClient.getDAGStatus(statusGetOpts).getDAGCounters(); + DAGStatus dagStatus = dagClient.getDAGStatus(statusGetOpts); + this.setStatusMessage(dagStatus.getState().name()); + + TezCounters dagCounters = dagStatus.getDAGCounters(); // if initial counters exists, merge it with dag counters to get aggregated view TezCounters mergedCounters = counters == null ? dagCounters : Utils.mergeTezCounters(dagCounters, counters); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/TestDriver.java index c3da12a32ee4..b5c7c3ef454d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestDriver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -34,7 +35,7 @@ public class TestDriver { @Before public void beforeTest() { - conf = new HiveConf(); + conf = new HiveConfForTest(getClass()); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); @@ -62,8 +63,7 @@ public void testDriverContextQueryErrorMessageCompileTime() { @Test public void testDriverContextQueryErrorMessageRuntime() { - conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez"); - conf.setBoolVar(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION, false); + conf.setInt("tez.am.counters.max.keys", 0); SessionState.start(conf); Driver driver = getDriver(); try { @@ -78,7 +78,9 @@ public void testDriverContextQueryErrorMessageRuntime() { Assert.assertEquals(message, driver.driverContext.getQueryErrorMessage()); // sanity check: the message is as expected Assert.assertTrue("Exception message is not as expected, got: " + message, - e.getMessage().startsWith("FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.tez.TezTask")); + e.getMessage().equalsIgnoreCase( + "FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Too many " + + "counters: 1 max=0")); } finally { driver.close(); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index bf8f095f66f9..f32a1d47d161 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -106,6 +106,8 @@ public void setUp() throws Exception { } void initHiveConf() { hiveConf = new HiveConf(this.getClass()); + //TODO: HIVE-28029: Make unit tests based on TxnCommandsBaseForTests run on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); } void setUpInternal() throws Exception { initHiveConf(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/ddl/table/partition/show/TestShowPartitionAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/ddl/table/partition/show/TestShowPartitionAnalyzer.java index e3c64cb12f12..d6a195f10930 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/ddl/table/partition/show/TestShowPartitionAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/ddl/table/partition/show/TestShowPartitionAnalyzer.java @@ -24,12 +24,12 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -56,7 +56,7 @@ public class TestShowPartitionAnalyzer { @Before public void before() throws Exception { - conf = new HiveConf(); + conf = new HiveConfForTest(getClass()); SessionState.start(conf); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java index de59ba5644e4..e1ebc7927add 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; @@ -31,7 +32,7 @@ import static org.mockito.Mockito.*; public class TestContext { - private static HiveConf conf = new HiveConf(); + private static HiveConf conf = new HiveConfForTest(TestContext.class); private Context context; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 00a58c4cea6d..df77800d68eb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -89,6 +89,8 @@ public class TestExecDriver { try { queryState = new QueryState.Builder().withHiveConf(new HiveConf(ExecDriver.class)).build(); conf = queryState.getConf(); + // this test is mr specific + conf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); conf.setBoolVar(HiveConf.ConfVars.SUBMIT_VIA_CHILD, true); conf.setBoolVar(HiveConf.ConfVars.SUBMIT_LOCAL_TASK_VIA_CHILD, true); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java index 099c12bd686c..993da1d2c205 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java @@ -23,6 +23,7 @@ import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.hooks.Entity; @@ -51,15 +52,15 @@ public class TestExplainTask { private static final String BACKUP_ID = "backup-id-mock"; - private static final String AST = "ast-mock"; private PrintStream out; private ExplainTask uut; + private HiveConf hiveConf; private ObjectMapper objectMapper = new ObjectMapper(); @Before public void setUp() { - HiveConf hiveConf = new HiveConf(); + hiveConf = new HiveConfForTest(getClass()); uut = new ExplainTask(); uut.conf = hiveConf; out = mock(PrintStream.class); @@ -322,7 +323,7 @@ public void testCollectAuthRelatedEntitiesJsonShouldMatch() throws Exception { when(qs.getHiveOperation()).thenReturn(HiveOperation.EXPLAIN); uut.queryState = qs; - SessionState.start(new HiveConf(ExplainTask.class)); + SessionState.start(hiveConf); // SessionState.get().setCommandType(HiveOperation.EXPLAIN); HiveAuthenticationProvider authenticationProviderMock = mock(HiveAuthenticationProvider.class); when(authenticationProviderMock.getUserName()).thenReturn("test-user"); @@ -340,7 +341,6 @@ public void testCollectAuthRelatedEntitiesJsonShouldMatch() throws Exception { public void testOutputPlanVectorizationJsonShouldMatch() throws Exception { QueryState qs = mock(QueryState.class); when(qs.getHiveOperation()).thenReturn(HiveOperation.EXPLAIN); - HiveConf hiveConf = new HiveConf(); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); when(qs.getConf()).thenReturn(hiveConf); uut.queryState = qs; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java index dfbb7f36902a..15bf768895ae 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; @@ -96,7 +97,8 @@ public void setUp() { varchar5 = TypeInfoFactory.getPrimitiveTypeInfo("varchar(5)"); char10 = TypeInfoFactory.getPrimitiveTypeInfo("char(10)"); char5 = TypeInfoFactory.getPrimitiveTypeInfo("char(5)"); - SessionState.start(new HiveConf()); + HiveConf conf = new HiveConfForTest(getClass()); + SessionState.start(conf); } private void implicit(TypeInfo a, TypeInfo b, boolean convertible) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java index 160892c7dcb5..eff69cdce059 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.io.ContentSummaryInputFormat; @@ -79,7 +80,7 @@ public class TestGetInputSummary { @Before public void setup() throws Exception { // creates scratch directories needed by the Context object - SessionState.start(new HiveConf()); + SessionState.start(new HiveConfForTest(getClass())); this.jobConf = new JobConf(); this.properties = new Properties(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index dcf0483cf057..2324b3263de2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; @@ -421,7 +422,7 @@ public InputSplit[] getSplits(JobConf job, int splits) throws IOException { @Test public void testFetchOperatorContext() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.set("hive.support.concurrency", "false"); conf.setVar(HiveConf.ConfVars.HIVE_MAPRED_MODE, "nonstrict"); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index 7d47fa34e8c4..b18a3e360683 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; import org.apache.hadoop.hive.ql.exec.tez.TezTask; @@ -132,10 +133,10 @@ public void testSerializeTimestamp() { } @Test - public void testgetDbTableName() throws HiveException{ + public void testGetDbTableName() throws HiveException{ String tablename; String [] dbtab; - SessionState.start(new HiveConf(this.getClass())); + SessionState.start(new HiveConfForTest(getClass())); String curDefaultdb = SessionState.get().getCurrentDatabase(); //test table without db portion @@ -249,7 +250,7 @@ public void testRenameFileExistsHivePath() throws Exception { private List runRemoveTempOrDuplicateFilesTestCase(String executionEngine, boolean dPEnabled) throws Exception { - Configuration hconf = new HiveConf(this.getClass()); + Configuration hconf = new HiveConfForTest(getClass()); // do this to verify that Utilities.removeTempOrDuplicateFiles does not revert to default scheme information hconf.set("fs.defaultFS", "hdfs://should-not-be-used/"); hconf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, executionEngine); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 5bac3bb62c76..5afa13c6da30 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.metrics.common.Metrics; import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.Operator; @@ -170,7 +171,7 @@ public Edge answer(InvocationOnMock invocation) throws Throwable { conf = new JobConf(); appLr = createResource("foo.jar"); - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = new HiveConfForTest(getClass()); hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); @@ -356,7 +357,7 @@ public void testParseRightmostXmx() throws Exception { } @Test - public void tezTask_updates_Metrics() throws IOException { + public void testTezTaskUpdatesMetrics() throws IOException { Metrics mockMetrics = Mockito.mock(Metrics.class); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorUDFUnixTimeStampString.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorUDFUnixTimeStampString.java index 9dd79d44e92a..45637e181b28 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorUDFUnixTimeStampString.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorUDFUnixTimeStampString.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -72,7 +73,7 @@ public static Collection readInputs() throws IOException, CsvException @Test public void testEvaluate() throws HiveException, InterruptedException, CharacterCodingException { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(HiveConf.ConfVars.HIVE_DATETIME_FORMATTER, formatter); conf.setVar(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE, zone); SessionState state = SessionState.start(conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/TestMapJoinOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/TestMapJoinOperator.java index 7e7b356d57bd..d9a9c3fd6aec 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/TestMapJoinOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/TestMapJoinOperator.java @@ -197,7 +197,7 @@ private boolean doTestLong0(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -275,7 +275,7 @@ private boolean doTestLong0_NoRegularKeys(long seed, int rowCount, int hiveConfV VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -354,7 +354,7 @@ public boolean doTestLong1(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -432,7 +432,7 @@ public boolean doTestLong2(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -510,7 +510,7 @@ public boolean doTestLong3(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -589,7 +589,7 @@ public boolean doTestLong3_NoRegularKeys(long seed, int rowCount, int hiveConfVa VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -669,7 +669,7 @@ public boolean doTestLong4(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -746,7 +746,7 @@ public boolean doTestLong5(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -825,7 +825,7 @@ public boolean doTestLong6(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception { - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -927,7 +927,7 @@ public boolean doTestMultiKey0(long seed, int hiveConfVariation, VectorMapJoinVa int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1006,7 +1006,7 @@ public boolean doTestMultiKey1(long seed, int hiveConfVariation, VectorMapJoinVa int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1087,7 +1087,7 @@ public boolean doTestMultiKey2(long seed, int hiveConfVariation, VectorMapJoinVa int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1169,7 +1169,7 @@ public boolean doTestMultiKey3(long seed, int hiveConfVariation, VectorMapJoinVa int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1250,7 +1250,7 @@ public boolean doTestString0(long seed, int hiveConfVariation, int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1330,7 +1330,7 @@ public boolean doTestString1(long seed, int hiveConfVariation, int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -1416,7 +1416,7 @@ public boolean doTestString2(long seed, int hiveConfVariation, int rowCount = 10; - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = getHiveConf(); if (!addNonLongHiveConfVariation(hiveConfVariation, hiveConf)) { return true; @@ -2054,4 +2054,11 @@ private void executeTestImplementation( " variation " + testDesc.vectorMapJoinVariation + option); } } + + private HiveConf getHiveConf() { + HiveConf hiveConf = new HiveConf(); + // TODO: HIVE-28033: TestMapJoinOperator to run on Tez + hiveConf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); + return hiveConf; + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java index 263c68acb839..12c8e0e619d8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.hooks; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -41,7 +42,7 @@ public class TestQueryHooks { @BeforeClass public static void setUpBeforeClass() { - conf = new HiveConf(TestQueryHooks.class); + conf = new HiveConfForTest(TestQueryHooks.class); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java index afdc6844109a..4e7aaa826e2d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java @@ -111,7 +111,6 @@ public void tearDown() throws IOException { @Test public void testCombine() throws Exception { JobConf newJob = new JobConf(job); - FileSystem fs = dataDir1.getFileSystem(newJob); Path dir1_file1 = new Path(dataDir1, "combinefile1_1"); writeTextFile(dir1_file1, @@ -132,6 +131,8 @@ public void testCombine() throws Exception { HiveConf hiveConf = new HiveConf(TestSymlinkTextInputFormat.class); + // TODO: HIVE-28032: TestSymlinkTextInputFormat.testCombine to run on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/DbTxnManagerEndToEndTestBase.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/DbTxnManagerEndToEndTestBase.java index 8de3e2d57a3a..8f62c598932c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/DbTxnManagerEndToEndTestBase.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/DbTxnManagerEndToEndTestBase.java @@ -54,6 +54,8 @@ public abstract class DbTxnManagerEndToEndTestBase { protected TxnStore txnHandler; public DbTxnManagerEndToEndTestBase() { + //TODO: HIVE-28029: Make unit tests based on DbTxnManagerEndToEndTestBase run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index b6f533490ffd..77daeb5cb6b0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -19,6 +19,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.ThreadPool; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; @@ -67,7 +68,7 @@ */ public class TestDbTxnManager { private static final int TEST_TIMED_OUT_TXN_ABORT_BATCH_SIZE = 1000; - private final HiveConf conf = new HiveConf(); + private final HiveConf conf = new HiveConfForTest(getClass()); private HiveTxnManager txnMgr; private AcidHouseKeeperService houseKeeperService = null; private final Context ctx; @@ -76,8 +77,7 @@ public class TestDbTxnManager { HashSet writeEntities; public TestDbTxnManager() throws Exception { - conf - .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); TestTxnDbUtil.setConfValues(conf); SessionState.start(conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java index 04447126eac6..83aa080c3842 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java @@ -24,6 +24,7 @@ import org.junit.Assert; import org.junit.Before; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -50,7 +51,7 @@ @RunWith(MockitoJUnitRunner.class) public class TestDummyTxnManager { - private final HiveConf conf = new HiveConf(); + private final HiveConf conf = new HiveConfForTest(getClass()); private HiveTxnManager txnMgr; private Context ctx; private int nextInput = 1; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index bb7f754fc509..8f03685cf86b 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -90,8 +90,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import org.junit.BeforeClass; import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; /** @@ -105,10 +105,17 @@ public class TestHive { @BeforeClass public static void setUp() throws Exception { - hiveConf = new HiveConf(TestHive.class); + hiveConf = getNewConf(null); hm = setUpImpl(hiveConf); } + private static HiveConf getNewConf(HiveConf oldConf) { + HiveConf conf = oldConf == null ? new HiveConf(TestHive.class) : new HiveConf(oldConf, TestHive.class); + //TODO: HIVE-28289: TestHive/TestHiveRemote to run on Tez + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); + return conf; + } + private static Hive setUpImpl(HiveConf hiveConf) throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); @@ -469,14 +476,16 @@ public void testGetAndDropTables() throws Throwable { @Test public void testWmNamespaceHandling() throws Throwable { - HiveConf hiveConf = new HiveConf(this.getClass()); + HiveConf hiveConf = getNewConf(null); Hive hm = setUpImpl(hiveConf); // TODO: threadlocals... Why is all this Hive client stuff like that?!! final AtomicReference hm2r = new AtomicReference<>(); Thread pointlessThread = new Thread(new Runnable() { @Override public void run() { - HiveConf hiveConf2 = new HiveConf(this.getClass()); + HiveConf hiveConf2 = getNewConf(null); + //TODO: HIVE-28289: TestHive/TestHiveRemote to run on Tez + hiveConf2.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf2.setVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE, "hm2"); try { hm2r.set(setUpImpl(hiveConf2)); @@ -638,8 +647,8 @@ public void testDropPartitionsWithPurge() throws Exception { .put("ds", "20141216") .put("hr", "12") .build(); - - int trashSizeBeforeDrop = getTrashContents().length; + FileStatus[] trashContentsBeforeDrop = getTrashContents(); + int trashSizeBeforeDrop = trashContentsBeforeDrop.length; Table table = createPartitionedTable(dbName, tableName); hm.createPartition(table, partitionSpec); @@ -672,11 +681,12 @@ public void testDropPartitionsWithPurge() throws Exception { .purgeData(false) ); - int trashSizeWithoutPurge = getTrashContents().length; - - assertEquals("After dropPartitions(noPurge), data should've gone to trash!", - trashSizeBeforeDrop, trashSizeWithoutPurge); + FileStatus[] trashContentsWithoutPurge = getTrashContents(); + int trashSizeWithoutPurge = trashContentsWithoutPurge.length; + assertEquals("After dropPartitions(noPurge), data should've gone to trash, contents before drop: " + + Arrays.asList(trashContentsBeforeDrop) + ", contents without purge: " + Arrays.asList(trashContentsWithoutPurge) + + "!", trashSizeBeforeDrop, trashSizeWithoutPurge); } catch (Exception e) { fail("Unexpected exception: " + StringUtils.stringifyException(e)); @@ -973,7 +983,7 @@ public void testHiveRefreshOnConfChange() throws Throwable{ Hive newHiveObj; //if HiveConf has not changed, same object should be returned - HiveConf newHconf = new HiveConf(hiveConf); + HiveConf newHconf = getNewConf(hiveConf); newHiveObj = Hive.get(newHconf); assertTrue(prevHiveObj == newHiveObj); @@ -985,7 +995,7 @@ public void testHiveRefreshOnConfChange() throws Throwable{ prevHiveObj = Hive.get(); prevHiveObj.getDatabaseCurrent(); //change value of a metavar config param in new hive conf - newHconf = new HiveConf(hiveConf); + newHconf = getNewConf(hiveConf); newHconf.setIntVar(ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES, newHconf.getIntVar(ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES) + 1); newHiveObj = Hive.get(newHconf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java index af2920150478..a6188e1ad97e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java @@ -27,11 +27,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.CheckResult; import org.apache.hadoop.hive.metastore.HiveMetaStoreChecker; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -80,16 +80,17 @@ public class TestHiveMetaStoreChecker { @Before public void setUp() throws Exception { hive = Hive.get(); - hive.getConf().set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "15"); - hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "throw"); - msc = new HiveMetaStoreClient(hive.getConf()); - checker = new HiveMetaStoreChecker(msc, hive.getConf()); + HiveConf conf = new HiveConfForTest(hive.getConf(), getClass()); + conf.set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "15"); + conf.set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "throw"); + msc = new HiveMetaStoreClient(conf); + checker = new HiveMetaStoreChecker(msc, conf); - hive.getConf().setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - HiveConf.setBoolVar(hive.getConf(), HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - SessionState ss = SessionState.start(hive.getConf()); - ss.initTxnMgr(hive.getConf()); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + SessionState ss = SessionState.start(conf); + ss.initTxnMgr(conf); partCols = new ArrayList<>(); partCols.add(new FieldSchema(partDateName, serdeConstants.STRING_TYPE_NAME, "")); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java index 175b47c47d83..6d93b1750687 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; @@ -73,21 +74,22 @@ public void setUp() throws Exception { client = new TestHiveMetaStoreClient(new HiveConf(Hive.class)); hive = Hive.get(client); - hive.getConf().set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "15"); - hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "throw"); - msc = new HiveMetaStoreClient(hive.getConf()); + HiveConf conf = new HiveConfForTest(hive.getConf(), getClass()); + conf.set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "15"); + conf.set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "throw"); + msc = new HiveMetaStoreClient(conf); - hive.getConf().setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - HiveConf.setBoolVar(hive.getConf(), HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - hive.getConf().set(ValidTxnList.VALID_TXNS_KEY, "1:"); - hive.getConf().set(ValidWriteIdList.VALID_WRITEIDS_KEY, TABLE_NAME + ":1:"); - hive.getConf().setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.TestTxnManager"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.set(ValidTxnList.VALID_TXNS_KEY, "1:"); + conf.set(ValidWriteIdList.VALID_WRITEIDS_KEY, TABLE_NAME + ":1:"); + conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.TestTxnManager"); // Pick a small number for the batch size to easily test code with multiple batches. - hive.getConf().setIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, 2); - SessionState.start(hive.getConf()); - SessionState.get().initTxnMgr(hive.getConf()); - Context ctx = new Context(hive.getConf()); + conf.setIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, 2); + SessionState.start(conf); + SessionState.get().initTxnMgr(conf); + Context ctx = new Context(conf); SessionState.get().getTxnMgr().openTxn(ctx, USER_NAME); t = new Table(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java index 78ec15969440..b614a70edf3c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.metadata; import java.io.IOException; -import java.net.ServerSocket; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; @@ -44,6 +43,8 @@ public class TestHiveRemote extends TestHive { @BeforeClass public static void setUp() throws Exception { hiveConf = new HiveConf(TestHiveRemote.class); + //TODO: HIVE-28289: TestHive/TestHiveRemote to run on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true); @@ -77,17 +78,4 @@ public void after() throws IOException { @Override public void testDropTableTrash() { } - - /** - * Finds a free port. - * - * @return a free port - * @throws IOException - */ - private String findFreePort() throws IOException { - ServerSocket socket= new ServerSocket(0); - int port = socket.getLocalPort(); - socket.close(); - return String.valueOf(port); - } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java index 6f996a34ede8..2e3ad0e03d58 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java @@ -19,6 +19,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; @@ -66,7 +67,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java index eb96a99de2ee..ca1082773786 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -73,7 +74,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java index 72977c710bdd..1cb3e1d2edd9 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java @@ -19,6 +19,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; @@ -71,7 +72,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAppendPartitionTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAppendPartitionTempTable.java index 6a2e958f07b0..2f9a9a59a273 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAppendPartitionTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAppendPartitionTempTable.java @@ -19,6 +19,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; @@ -62,7 +63,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientDropPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientDropPartitionsTempTable.java index c18dadc3cf2d..eaa20ed14535 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientDropPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientDropPartitionsTempTable.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; @@ -72,7 +73,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java index 6844c5df1129..836483e2a938 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -69,7 +70,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws Exception{ - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientGetPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientGetPartitionsTempTable.java index dee96521c09a..d622e07c9aef 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientGetPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientGetPartitionsTempTable.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.Partition; @@ -75,7 +76,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java index 09d9dc2f5327..006ba6a0584a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientListPartitionsTempTable.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TestMetastoreExpr; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; @@ -94,7 +95,7 @@ public void setUp() throws Exception { } private void initHiveConf() throws HiveException { - conf = Hive.get().getConf(); + conf = new HiveConfForTest(Hive.get().getConf(), getClass()); conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java index a96d93e04291..60e3c47413da 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java @@ -64,6 +64,8 @@ public class TestGenMapRedUtilsCreateConditionalTask { @BeforeClass public static void initializeSessionState() { hiveConf = new HiveConf(); + // GenMapRedUtils is an MR compiler class + hiveConf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); } @Before diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestNullScanTaskDispatcher.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestNullScanTaskDispatcher.java index 07eabd171f55..1370a724818a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestNullScanTaskDispatcher.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestNullScanTaskDispatcher.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -83,7 +84,7 @@ public class TestNullScanTaskDispatcher { @Before public void setup() { - hiveConf = new HiveConf(); + hiveConf = new HiveConfForTest(getClass()); hiveConf.set("fs.mock.impl", MockFileSystem.class.getName()); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_METADATA_ONLY_QUERIES, true); sessionState = SessionState.start(hiveConf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestDMLSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestDMLSemanticAnalyzer.java index 211c80aa662b..6e7814864cac 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestDMLSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestDMLSemanticAnalyzer.java @@ -229,6 +229,8 @@ public void testInsertValuesPartitioned() throws Exception { public void setup() throws Exception { queryState = new QueryState.Builder().build(); conf = queryState.getConf(); + // the test doesn't involve DAG execution, skip TezSessionState initialization + conf.setBoolean(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, false); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java index 888e4efcbcf1..68156a1a4388 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java @@ -22,13 +22,13 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Properties; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -68,8 +68,7 @@ public class TestGenTezWork { @SuppressWarnings("unchecked") @Before public void setUp() throws Exception { - // Init conf - final HiveConf conf = new HiveConf(SemanticAnalyzer.class); + final HiveConf conf = new HiveConfForTest(getClass()); SessionState.start(conf); // Init parse context diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestHiveDecimalParse.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestHiveDecimalParse.java index 7d96e630365f..ebab09e73360 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestHiveDecimalParse.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestHiveDecimalParse.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.QueryPlan; @@ -166,7 +167,7 @@ public void testDecimalType9() throws ParseException { } private Driver createDriver() { - HiveConf conf = new HiveConf(Driver.class); + HiveConf conf = new HiveConfForTest(getClass()); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java index 6f61c33c0323..3aa49b9aab18 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java @@ -44,6 +44,8 @@ public class TestMacroSemanticAnalyzer { public void setup() throws Exception { queryState = new QueryState.Builder().build(); conf = queryState.getConf(); + // the test doesn't involve DAG execution, skip TezSessionState initialization + conf.setBoolean(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, false); SessionState.start(conf); context = new Context(conf); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java index 696311df25d1..5970dc42a0bd 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.TxnType; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -49,7 +50,7 @@ public class TestParseUtils { public TestParseUtils(String query, TxnType txnType) { this.query = query; this.txnType = txnType; - this.conf = new HiveConf(); + this.conf = new HiveConfForTest(getClass()); } @Before diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java index b1c8c0fbde08..ccdc04991815 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Test; import org.junit.experimental.runners.Enclosed; @@ -32,7 +33,7 @@ public class TestReplicationSemanticAnalyzer { private static HiveConf hiveConf = buildHiveConf(); public static HiveConf buildHiveConf() { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(TestReplicationSemanticAnalyzer.class); conf.setVar(HIVE_QUOTEDID_SUPPORT, Quotation.NONE.stringValue()); return conf; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java index d170986e52bc..99e40d950066 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java @@ -98,6 +98,8 @@ public static void reset() { public void setup() throws Exception { queryState = new QueryState.Builder().build(); HiveConf conf = queryState.getConf(); + // the test doesn't involve DAG execution, skip TezSessionState initialization + conf.setBoolean(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, false); conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY, TestHiveAuthorizationTaskFactory.DummyHiveAuthorizationTaskFactoryImpl.class.getName()); conf diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV1.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV1.java index 1fa11fc3361d..25ac38c00df2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV1.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV1.java @@ -42,6 +42,8 @@ public void setup() throws Exception { queryState = new QueryState.Builder().build(); db = Mockito.mock(Hive.class); HiveConf hiveConf = queryState.getConf(); + // the test doesn't involve DAG execution, skip TezSessionState initialization + hiveConf.setBoolean(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, false); table = new Table(DB, TABLE); partition = new Partition(table); hiveConf diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV2.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV2.java index e88ad35c1563..fe805187783a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestPrivilegesV2.java @@ -43,6 +43,8 @@ public void setup() throws Exception { queryState = new QueryState.Builder().build(); //set authorization mode to V2 HiveConf conf = queryState.getConf(); + // the test doesn't involve DAG execution, skip TezSessionState initialization + conf.setBoolean(HiveConf.ConfVars.HIVE_CLI_TEZ_INITIALIZE_SESSION.varname, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, SQLStdHiveAuthorizerFactory.class.getName()); db = Mockito.mock(Hive.class); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestSessionUserName.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestSessionUserName.java index 61d755d2fd40..252f2018fbce 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestSessionUserName.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestSessionUserName.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; @@ -132,7 +133,7 @@ private void setupDataNucleusFreeHive(HiveConf hiveConf) throws MetaException { * that captures the given user name */ private HiveConf getAuthV2HiveConf() { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, HiveAuthorizerStoringUserNameFactory.class.getName()); conf.setVar(HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestDecimalStringValidation.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestDecimalStringValidation.java index 0d295ad64be1..ad57753c774a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestDecimalStringValidation.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestDecimalStringValidation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse.type; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -100,7 +101,7 @@ public static Collection params() throws Exception { @Test public void testValidationDecimalWithCharacterFailsWhenStrictChecksEnabled() { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setBoolVar(HiveConf.ConfVars.HIVE_STRICT_CHECKS_TYPE_SAFETY, true); try { validateCall(conf); @@ -112,7 +113,7 @@ public void testValidationDecimalWithCharacterFailsWhenStrictChecksEnabled() { @Test public void testValidationDecimalWithCharacterSucceedsWhenStrictChecksDisabled() throws SemanticException { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setBoolVar(HiveConf.ConfVars.HIVE_STRICT_CHECKS_TYPE_SAFETY, false); validateCall(conf); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java index 0c960f2b8e13..e8174df185ad 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java @@ -21,6 +21,7 @@ import java.sql.SQLException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Assert; import org.junit.Before; @@ -34,7 +35,7 @@ public class TestCommandProcessorFactory { @Before public void setUp() throws Exception { - conf = new HiveConf(); + conf = new HiveConfForTest(getClass()); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java index cc251789c11d..54da0eb2bcc6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.junit.Before; @@ -44,12 +45,13 @@ public class TestAddResource { private static final String TEST_JAR_DIR = System.getProperty("test.tmp.dir", ".") + File.separator; + private HiveConf conf; private ResourceType t; @Before public void setup() throws IOException { - conf = new HiveConf(); + conf = new HiveConfForTest(getClass()); t = ResourceType.JAR; //Generate test jar files @@ -165,8 +167,6 @@ public void testDuplicateAdds() throws URISyntaxException, IOException { // test when two jars with shared dependencies are added, the classloader contains union of the dependencies @Test public void testUnion() throws URISyntaxException, IOException { - - HiveConf conf = new HiveConf(); SessionState ss = Mockito.spy(SessionState.start(conf).get()); ResourceType t = ResourceType.JAR; String query1 = "testQuery1"; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java index 9f93f096d382..6c8cf2c0b90b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.session; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -46,6 +47,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hive.common.util.HiveTestUtils; import org.junit.After; import org.junit.Assert; @@ -62,6 +64,7 @@ */ @RunWith(value = Parameterized.class) public class TestSessionState { + private final boolean prewarm; private final static String clazzDistFileName = "RefreshedJarClass.jar.V1"; private final static String clazzV2FileName = "RefreshedJarClass.jar.V2"; @@ -85,7 +88,7 @@ public static Collection data() { @Before public void setUp() { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); String tmp = System.getProperty("java.io.tmpdir"); File tmpDir = new File(tmp); if (!tmpDir.exists()) { @@ -133,7 +136,7 @@ public void testgetDbName() throws Exception { SessionState.get().getCurrentDatabase()); //verify that a new sessionstate has default db - SessionState.start(new HiveConf()); + SessionState.start(getNewHiveConf()); assertEquals(Warehouse.DEFAULT_DATABASE_NAME, SessionState.get().getCurrentDatabase()); @@ -166,7 +169,7 @@ public void run() { @Test public void testClassLoaderEquality() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); final SessionState ss1 = new SessionState(conf); RegisterJarRunnable otherThread = new RegisterJarRunnable("./build/contrib/test/test-udfs.jar", ss1); Thread th1 = new Thread(otherThread); @@ -203,7 +206,7 @@ private void generateRefreshJarFiles(String version) throws IOException, Interru @Test public void testReloadAuxJars2() { - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); HiveConf.setVar(conf, ConfVars.HIVE_RELOADABLE_JARS, hiveReloadPath); SessionState ss = new SessionState(conf); SessionState.start(ss); @@ -273,7 +276,7 @@ public void testReflectionCleanup() throws Exception { @Test public void testReloadExistingAuxJars2() { - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); HiveConf.setVar(conf, ConfVars.HIVE_RELOADABLE_JARS, hiveReloadPath); SessionState ss = new SessionState(conf); @@ -319,7 +322,7 @@ public void testReloadExistingAuxJars2() { */ @Test public void testCreatePath() throws Exception { - HiveConf conf = new HiveConf(); + HiveConf conf = getNewHiveConf(); LocalFileSystem localFileSystem = FileSystem.getLocal(conf); Path repeatedCreate = new Path("repeatedCreate"); @@ -360,4 +363,8 @@ public void testCreatePath() throws Exception { assertTrue(e.getMessage().contains("Failed to create directory noPermissions/child")); } } + + private HiveConf getNewHiveConf() { + return new HiveConfForTest(getClass()); + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java index d37b54437832..b97a8fecf625 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.common.repl.ReplConst; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -59,21 +60,18 @@ public class TestStatsUpdaterThread { @SuppressWarnings("unused") static final private Logger LOG = LoggerFactory.getLogger(TestStatsUpdaterThread.class); - private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") + - File.separator + TestStatsUpdaterThread.class.getCanonicalName() - + "-" + System.currentTimeMillis() - ).getPath().replaceAll("\\\\", "/"); - private HiveConf hiveConf; + + private HiveConfForTest hiveConf; private SessionState ss; String getTestDataDir() { - return TEST_DATA_DIR; + return hiveConf.getTestDataDir(); } @SuppressWarnings("deprecation") @Before public void setUp() throws Exception { - this.hiveConf = new HiveConf(TestStatsUpdaterThread.class); + hiveConf = new HiveConfForTest(getClass()); hiveConf.set(HiveConf.ConfVars.PRE_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POST_EXEC_HOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.METASTORE_WAREHOUSE.varname, getTestDataDir()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java b/ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java index 4c522c85a143..4770e5f1b4e3 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hive.ql.tool; - import static org.junit.Assert.fail; import java.util.TreeSet; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.tools.LineageInfo; @@ -40,7 +40,7 @@ public class TestLineageInfo { @Before public void before() { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); SessionState.start(conf); ctx = new Context(conf); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.java index 024bd6c27231..a13dc6c62677 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; @@ -51,6 +52,7 @@ */ @RunWith(Parameterized.class) public class TestGenericUDFDateFormatEvaluate { + private final GenericUDFDateFormat udf = new GenericUDFDateFormat(); private final String value; private final String pattern; @@ -84,7 +86,7 @@ public static Collection readInputs() throws IOException, CsvException @Test public void testEvaluate() throws HiveException, InterruptedException { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(HiveConf.ConfVars.HIVE_DATETIME_FORMATTER, formatter); conf.setVar(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE, zone); SessionState state = SessionState.start(conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.java index 3b6f1eedd13d..8c9dbd2b3026 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; @@ -84,7 +85,7 @@ public static Collection readInputs() throws IOException, CsvException @Test public void testEvaluate() throws HiveException, InterruptedException { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(HiveConf.ConfVars.HIVE_DATETIME_FORMATTER, formatter); conf.setVar(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE, zone); SessionState state = SessionState.start(conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestampEvaluateStringString.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestampEvaluateStringString.java index bda6a171b161..5a5d4cfe0e10 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestampEvaluateStringString.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestampEvaluateStringString.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; @@ -92,7 +93,7 @@ public void testEvaluateUnixTimeStamp() throws HiveException, InterruptedExcepti } private void testEvaluateWithUDF(GenericUDF udfToTest) throws HiveException, InterruptedException { - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setVar(HiveConf.ConfVars.HIVE_DATETIME_FORMATTER, formatter); conf.setVar(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE, zone); conf.setVar(HiveConf.ConfVars.HIVE_DATETIME_RESOLVER_STYLE, resolverStyle); diff --git a/ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java b/ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java index f098a217efaa..aaa63fb094fa 100644 --- a/ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java +++ b/ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java @@ -361,4 +361,4 @@ public HiveTestEnvContext getTestCtx() { return testEnvContext; } -} +} \ No newline at end of file diff --git a/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java b/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java index ac19b19dbb96..8abcdc659aab 100644 --- a/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java +++ b/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java @@ -301,6 +301,8 @@ private void syncThreadStart(final CountDownLatch cdlIn, final CountDownLatch cd @Test public void testExecuteStatementParallel() throws Exception { Map confOverlay = new HashMap(); + //TODO: HIVE-28283: CliServiceTest.testExecuteStatementParallel to run on Tez + confOverlay.put(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "mr"); String tableName = "TEST_EXEC_PARALLEL"; String columnDefinitions = "(ID STRING)"; diff --git a/service/src/test/org/apache/hive/service/cli/operation/TestCommandWithSpace.java b/service/src/test/org/apache/hive/service/cli/operation/TestCommandWithSpace.java index 5c57357c84b4..568a05bf631a 100644 --- a/service/src/test/org/apache/hive/service/cli/operation/TestCommandWithSpace.java +++ b/service/src/test/org/apache/hive/service/cli/operation/TestCommandWithSpace.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.hooks.TestQueryHooks; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.processors.DfsProcessor; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.HiveSQLException; @@ -36,7 +36,7 @@ public class TestCommandWithSpace { @Test public void testCommandWithPrefixSpace() throws IllegalAccessException, ClassNotFoundException, InstantiationException, HiveSQLException { String query = " dfs -ls /"; - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConfForTest(getClass()); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/service/src/test/org/apache/hive/service/cli/operation/TestQueryLifeTimeHooksWithSQLOperation.java b/service/src/test/org/apache/hive/service/cli/operation/TestQueryLifeTimeHooksWithSQLOperation.java index 2fd336128b08..0ac82e8362bc 100644 --- a/service/src/test/org/apache/hive/service/cli/operation/TestQueryLifeTimeHooksWithSQLOperation.java +++ b/service/src/test/org/apache/hive/service/cli/operation/TestQueryLifeTimeHooksWithSQLOperation.java @@ -21,9 +21,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContext; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookWithParseHooks; -import org.apache.hadoop.hive.ql.hooks.TestQueryHooks; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.HandleIdentifier; @@ -48,7 +48,7 @@ public class TestQueryLifeTimeHooksWithSQLOperation { @Test public void testQueryInfoInHookContext() throws IllegalAccessException, ClassNotFoundException, InstantiationException, HiveSQLException { - HiveConf conf = new HiveConf(TestQueryHooks.class); + HiveConf conf = new HiveConfForTest(getClass()); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/service/src/test/org/apache/hive/service/cli/operation/TestQueryShutdownHooks.java b/service/src/test/org/apache/hive/service/cli/operation/TestQueryShutdownHooks.java index 0170c716c890..d501cca96688 100644 --- a/service/src/test/org/apache/hive/service/cli/operation/TestQueryShutdownHooks.java +++ b/service/src/test/org/apache/hive/service/cli/operation/TestQueryShutdownHooks.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.util.ShutdownHookManagerInspector; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.OperationHandle; @@ -53,7 +54,9 @@ public class TestQueryShutdownHooks { public void setUp() throws Exception { service = new EmbeddedThriftBinaryCLIService(); - HiveConf hiveConf = new HiveConf(); + HiveConf hiveConf = new HiveConfForTest(getClass()); + //TODO: HIVE-28298: TestQueryShutdownHooks to run on Tez + hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); @@ -99,7 +102,11 @@ public void testSync() throws Exception { assertEquals("Query should be finished", OperationState.FINISHED, state); } - ShutdownHookManagerInspector.assertShutdownHookCount(shutdownHooksBeforeQueries); + //TODO: HIVE-28298: TestQueryShutdownHooks to run on Tez + // the query starts a Tez session, which involves a TezJobMonitor hook + // int expectedHooks = shutdownHooksBeforeQueries + 1; + int expectedHooks = shutdownHooksBeforeQueries; + ShutdownHookManagerInspector.assertShutdownHookCount(expectedHooks); } @Test diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java index 97ba39f1b8a7..407bcc72bdee 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hive.service.Service; import org.apache.hive.service.cli.OperationHandle; @@ -62,7 +63,7 @@ public static void setUpBeforeClass() throws Exception { // Find a free port port = MetaStoreTestUtils.findFreePort(); hiveServer2 = new HiveServer2(); - hiveConf = new HiveConf(); + hiveConf = new HiveConfForTest(ThriftCLIServiceTest.class); } /** diff --git a/streaming/pom.xml b/streaming/pom.xml index 200d7f44f3a3..399ee9f56c90 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -133,12 +133,29 @@ tests test + + org.apache.hive + hive-common + ${project.version} + tests + test + org.apache.hadoop hadoop-mapreduce-client-jobclient ${hadoop.version} test + + org.apache.tez + tez-mapreduce + test + + + org.apache.tez + tez-dag + test + ${basedir}/src/java diff --git a/streaming/src/test/org/apache/hive/streaming/TestStreamingDynamicPartitioning.java b/streaming/src/test/org/apache/hive/streaming/TestStreamingDynamicPartitioning.java index c8c7a8e26db0..0a21f4613e45 100644 --- a/streaming/src/test/org/apache/hive/streaming/TestStreamingDynamicPartitioning.java +++ b/streaming/src/test/org/apache/hive/streaming/TestStreamingDynamicPartitioning.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -126,7 +127,7 @@ public FileStatus getFileStatus(Path path) throws IOException { private final static String dbName2 = "testing2"; public TestStreamingDynamicPartitioning() throws Exception { - conf = new HiveConf(this.getClass()); + conf = new HiveConfForTest(getClass()); conf.set("fs.raw.impl", RawFileSystem.class.getName()); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,