Port rest integ tests to use task avoidance api (elastic#65011)

This ports the majority of the rest integ tests tasks to use the task avoidance api.

- There are some edge cases left that we need to investigate, but we can do that separately.

build.gradle

@@ -287,9 +287,10 @@ gradle.projectsEvaluated {
       // :test:framework:test cannot run before and after :server:test
       return
     }
-    if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
-      integTest.mustRunAfter test
+    tasks.matching {it.name.equals('integTest')}.configureEach {integTestTask ->
+      integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
     }
+
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
         Project upstreamProject = dep.dependencyProject

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -39,8 +39,8 @@ class DocsTestPlugin implements Plugin<Project> {
 
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.testClusters.integTest.testDistribution = distribution.toUpperCase()
-        project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
+        project.testClusters.matching { it.name.equals("integTest") }.configureEach { testDistribution = distribution.toUpperCase() }
+        project.testClusters.matching { it.name.equals("integTest") }.configureEach { nameCustomization = { it.replace("integTest", "node") } }
         // Docs are published separately so no need to assemble
         project.tasks.named("assemble").configure {enabled = false }
         Map<String, String> commonDefaultSubstitutions = [

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy

@@ -25,6 +25,7 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.JavaBasePlugin
+import org.gradle.api.tasks.TaskProvider
 
 /**
  * Adds support for starting an Elasticsearch cluster before running integration
@@ -47,10 +48,11 @@ class RestTestPlugin implements Plugin<Project> {
         }
         project.getPlugins().apply(RestTestBasePlugin.class);
         project.pluginManager.apply(TestClustersPlugin)
-        RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
-        integTest.description = 'Runs rest tests against an elasticsearch cluster.'
-        integTest.group = JavaBasePlugin.VERIFICATION_GROUP
-        integTest.mustRunAfter(project.tasks.named('precommit'))
+        TaskProvider<RestIntegTestTask> integTest = project.tasks.register('integTest', RestIntegTestTask.class) {
+            it.description = 'Runs rest tests against an elasticsearch cluster.'
+            it.group = JavaBasePlugin.VERIFICATION_GROUP
+            it.mustRunAfter(project.tasks.named('precommit'))
+        }
         project.tasks.named('check').configure { it.dependsOn(integTest) }
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java

@@ -24,12 +24,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 
 import java.util.Map;
 
+import static org.gradle.api.tasks.SourceSet.TEST_SOURCE_SET_NAME;
+
 /**
  * <p>
  * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API
@@ -100,7 +102,7 @@ public void apply(Project project) {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
                 task.coreConfig = testConfig;
-                task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
+                task.sourceSetName = TEST_SOURCE_SET_NAME;
                 if (BuildParams.isInternal()) {
                     // core
                     Dependency restTestdependency = project.getDependencies()
@@ -131,7 +133,7 @@ public void apply(Project project) {
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
                 task.dependsOn(copyRestYamlTestTask);
                 task.coreConfig = specConfig;
-                task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
+                task.sourceSetName = TEST_SOURCE_SET_NAME;
                 if (BuildParams.isInternal()) {
                     Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
@@ -149,12 +151,14 @@ public void apply(Project project) {
                 task.dependsOn(xpackSpecConfig);
             });
 
-        project.afterEvaluate(p -> {
+        project.getPlugins().withType(JavaBasePlugin.class).configureEach(javaBasePlugin -> {
             SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-            SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
-            if (testSourceSet != null) {
-                project.getTasks().named(testSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(copyRestYamlSpecTask));
-            }
+            sourceSets.matching(sourceSet -> sourceSet.getName().equals(TEST_SOURCE_SET_NAME))
+                .configureEach(
+                    testSourceSet -> project.getTasks()
+                        .named(testSourceSet.getProcessResourcesTaskName())
+                        .configure(t -> t.dependsOn(copyRestYamlSpecTask))
+                );
         });
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java

@@ -64,11 +64,15 @@ public void apply(Project project) {
         setupDependencies(project, yamlTestSourceSet);
 
         // setup the copy for the rest resources
-        project.getTasks().withType(CopyRestApiTask.class, copyRestApiTask -> {
-            copyRestApiTask.sourceSetName = SOURCE_SET_NAME;
-            project.getTasks().named(yamlTestSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(copyRestApiTask));
-        });
-        project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { copyRestTestTask.sourceSetName = SOURCE_SET_NAME; });
+        project.getTasks()
+            .withType(CopyRestApiTask.class)
+            .configureEach(copyRestApiTask -> { copyRestApiTask.sourceSetName = SOURCE_SET_NAME; });
+        project.getTasks()
+            .named(yamlTestSourceSet.getProcessResourcesTaskName())
+            .configure(t -> t.dependsOn(project.getTasks().withType(CopyRestApiTask.class)));
+        project.getTasks()
+            .withType(CopyRestTestsTask.class)
+            .configureEach(copyRestTestTask -> copyRestTestTask.sourceSetName = SOURCE_SET_NAME);
 
         // setup IDE
         GradleUtils.setupIdeForTestSourceSet(project, yamlTestSourceSet);

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -44,5 +44,4 @@ default void useCluster(ElasticsearchCluster cluster) {
     }
 
     default void beforeStart() {}
-
 }

buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -200,12 +200,14 @@ public static void extendSourceSet(Project project, String parentSourceSetName,
      * Extends one configuration from another and refreshes the classpath of a provided Test.
      * The Test parameter is only needed for eagerly defined test tasks.
      */
-    public static void extendSourceSet(Project project, String parentSourceSetName, String childSourceSetName, Test test) {
+    public static void extendSourceSet(Project project, String parentSourceSetName, String childSourceSetName, TaskProvider<Test> test) {
         extendSourceSet(project, parentSourceSetName, childSourceSetName);
         if (test != null) {
-            SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-            SourceSet child = sourceSets.getByName(childSourceSetName);
-            test.setClasspath(child.getRuntimeClasspath());
+            test.configure(t -> {
+                SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+                SourceSet child = sourceSets.getByName(childSourceSetName);
+                t.setClasspath(child.getRuntimeClasspath());
+            });
         }
     }
 

docs/build.gradle

@@ -48,7 +48,7 @@ restResources {
   }
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest"}.configureEach {
   if (singleNode().testDistribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
@@ -82,21 +82,21 @@ testClusters.integTest {
 
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
-}
 
-// build the cluster with all plugins
-project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
-  /* Skip repositories. We just aren't going to be able to test them so it
-   * doesn't make sense to waste time installing them.
-   */
-  if (subproj.path.startsWith(':plugins:repository-')) {
-    return
-  }
-  // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported
-  if (subproj.path.startsWith(':plugins:ingest-attachment') && BuildParams.inFipsJvm) {
-    return
+  // build the cluster with all plugins
+  project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
+    /* Skip repositories. We just aren't going to be able to test them so it
+     * doesn't make sense to waste time installing them.
+     */
+    if (subproj.path.startsWith(':plugins:repository-')) {
+      return
+    }
+    // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported
+    if (subproj.path.startsWith(':plugins:ingest-attachment') && BuildParams.inFipsJvm) {
+      return
+    }
+    plugin subproj.path
   }
-  testClusters.integTest.plugin subproj.path
 }
 
 buildRestTests.docs = fileTree(projectDir) {

plugins/repository-hdfs/build.gradle

@@ -99,7 +99,7 @@ tasks.named("integTest").configure {
   dependsOn(project.tasks.named("bundlePlugin"))
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   plugin(project.tasks.bundlePlugin.archiveFile)
 }
 

qa/die-with-dignity/build.gradle

@@ -10,9 +10,9 @@ esplugin {
 }
 
 // let the javaRestTest see the classpath of main
-GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", tasks.named("javaRestTest"))
 
-javaRestTest {
+tasks.named("javaRestTest").configure {
   systemProperty 'tests.security.manager', 'false'
   systemProperty 'tests.system_call_filter', 'false'
   nonInputProperties.systemProperty 'log', "${-> testClusters.javaRestTest.singleNode().getServerLog()}"

qa/logging-config/build.gradle

@@ -22,7 +22,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.standalone-test'
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   /**
    * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch
    * can successfully startup.
@@ -33,7 +33,7 @@ testClusters.integTest {
 integTest {
   nonInputProperties.systemProperty 'tests.logfile',
     "${-> testClusters.integTest.singleNode().getServerLog().absolutePath.replaceAll("_server.json", ".log")}"
-  
+
   nonInputProperties.systemProperty 'tests.jsonLogfile',
     "${-> testClusters.integTest.singleNode().getServerLog()}"
 }

qa/smoke-test-ingest-disabled/build.gradle

@@ -26,6 +26,6 @@ dependencies {
   testImplementation project(':modules:ingest-common')
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   setting 'node.roles', '[data,master,remote_cluster_client]'
 }

qa/smoke-test-multinode/build.gradle

@@ -29,12 +29,12 @@ restResources {
 }
 
 File repo = file("$buildDir/testclusters/repo")
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   numberOfNodes = 2
   setting 'path.repo', repo.absolutePath
 }
 
-integTest {
+tasks.named("integTest").configure {
   doFirst {
     project.delete(repo)
     repo.mkdirs()

qa/smoke-test-plugins/build.gradle

@@ -27,7 +27,7 @@ apply plugin: 'elasticsearch.rest-resources'
 
 int pluginsCount = 0
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   project(':plugins').getChildProjects().each { pluginName, pluginProject ->
     if (BuildParams.inFipsJvm && pluginName == "ingest-attachment") {
       // Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle)

qa/unconfigured-node-name/build.gradle

@@ -23,11 +23,11 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   nameCustomization = { null }
 }
 
-integTest {
+tasks.named("integTest").configure {
   nonInputProperties.systemProperty 'tests.logfile',
     "${-> testClusters.integTest.singleNode().getServerLog()}"
 }

rest-api-spec/build.gradle

@@ -19,5 +19,5 @@ testClusters.all {
   module ':modules:mapper-extras'
 }
 
-test.enabled = false
-jarHell.enabled = false
+tasks.named("test").configure {enabled = false }
+tasks.named("jarHell").configure {enabled = false }

x-pack/docs/build.gradle

@@ -27,7 +27,7 @@ restResources {
   }
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   extraConfigFile 'op-jwks.json', xpackProject('test:idp-fixture').file("oidc/op-jwks.json")
   extraConfigFile 'idp-docs-metadata.xml', xpackProject('test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
   extraConfigFile 'testClient.crt', xpackProject('plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt")

x-pack/plugin/deprecation/qa/rest/build.gradle

@@ -14,7 +14,7 @@ dependencies {
 }
 
 // let the javaRestTest see the classpath of main
-GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", tasks.named("javaRestTest"))
 
 restResources {
   restApi {

x-pack/plugin/ilm/qa/multi-cluster/build.gradle

@@ -11,14 +11,14 @@ dependencies {
 
 File repoDir = file("$buildDir/testclusters/repo")
 
-task 'leader-cluster'(type: RestIntegTestTask) {
+tasks.register('leader-cluster', RestIntegTestTask) {
   mustRunAfter("precommit")
     systemProperty 'tests.target_cluster', 'leader'
     /* To support taking index snapshots, we have to set path.repo setting */
     systemProperty 'tests.path.repo', repoDir.absolutePath
 }
 
-testClusters.'leader-cluster' {
+testClusters.matching { it.name == 'leader-cluster' }.configureEach {
   testDistribution = 'DEFAULT'
   setting 'path.repo', repoDir.absolutePath
   setting 'xpack.ccr.enabled', 'true'
@@ -29,19 +29,19 @@ testClusters.'leader-cluster' {
   setting 'indices.lifecycle.poll_interval', '1000ms'
 }
 
-task 'follow-cluster'(type: RestIntegTestTask) {
-  dependsOn 'leader-cluster'
-    useCluster testClusters.'leader-cluster'
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host',
-      "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
-    nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed',
-      "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}"
-    /* To support taking index snapshots, we have to set path.repo setting */
-    systemProperty 'tests.path.repo', repoDir.absolutePath
+tasks.register('follow-cluster', RestIntegTestTask) {
+  dependsOn tasks.findByName('leader-cluster')
+  useCluster testClusters.'leader-cluster'
+  systemProperty 'tests.target_cluster', 'follow'
+  nonInputProperties.systemProperty 'tests.leader_host',
+    "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
+  nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed',
+    "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}"
+  /* To support taking index snapshots, we have to set path.repo setting */
+  systemProperty 'tests.path.repo', repoDir.absolutePath
 }
 
-testClusters.'follow-cluster' {
+testClusters.matching{ it.name == 'follow-cluster' }.configureEach {
   testDistribution = 'DEFAULT'
   setting 'path.repo', repoDir.absolutePath
   setting 'xpack.ccr.enabled', 'true'
@@ -54,5 +54,6 @@ testClusters.'follow-cluster' {
     { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }
 }
 
-check.dependsOn 'follow-cluster'
-test.enabled = false // no unit tests for this module, only the rest integration test
+tasks.named("check").configure { dependsOn 'follow-cluster' }
+// no unit tests for this module, only the rest integration test
+tasks.named("test").configure { enabled = false }

x-pack/plugin/ilm/qa/multi-node/build.gradle

@@ -7,7 +7,7 @@ dependencies {
 }
 
 // let the javaRestTest see the classpath of main
-GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", tasks.named("javaRestTest"))
 
 
 File repoDir = file("$buildDir/testclusters/repo")

x-pack/plugin/repositories-metering-api/qa/azure/build.gradle

@@ -66,7 +66,7 @@ integTest {
   nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_metering_tests_" + BuildParams.testSeed
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   testDistribution = 'DEFAULT'
   plugin repositoryPlugin.bundlePlugin.archiveFile
 

x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle

@@ -110,7 +110,7 @@ integTest {
   nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_metering" + BuildParams.testSeed
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   testDistribution = 'DEFAULT'
   plugin repositoryPlugin.bundlePlugin.archiveFile
 

x-pack/plugin/repositories-metering-api/qa/s3/build.gradle

@@ -48,7 +48,7 @@ integTest {
   nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + BuildParams.testSeed : 'base_path'
 }
 
-testClusters.integTest {
+testClusters.matching { it.name == "integTest" }.configureEach {
   testDistribution = 'DEFAULT'
   plugin repositoryPlugin.bundlePlugin.archiveFile