Skip to content

Commit

Permalink
Build: Add AntTask to simplify controlling logging when running ant f…
Browse files Browse the repository at this point in the history
…rom gradle

This new task allows setting code, similar to a doLast or doFirst,
except it is specifically geared at running ant (and thus called doAnt).
It adjusts the ant logging while running the ant so that the log
level/behavior can be tweaked, and automatically buffers based on gradle
logging level, and dumps the ant output upon failure.
  • Loading branch information
rjernst committed Dec 18, 2015
1 parent 4ec605e commit 9f1dfdb
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 61 deletions.
11 changes: 0 additions & 11 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,6 @@ subprojects {
}
}
}
// For reasons we don't fully understand yet, external dependencies are not picked up by Ant's optional tasks.
// But you can easily do it in another way.
// Only if your buildscript and Ant's optional task need the same library would you have to define it twice.
// https://docs.gradle.org/current/userguide/organizing_build_logic.html
configurations {
buildTools
}
dependencies {
buildTools 'de.thetaphi:forbiddenapis:2.0'
buildTools 'org.apache.rat:apache-rat:0.11'
}
}

// Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
Expand Down
1 change: 1 addition & 0 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ dependencies {
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
compile 'de.thetaphi:forbiddenapis:2.0'
compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
compile 'org.apache.rat:apache-rat:0.11'
}

processResources {
Expand Down
111 changes: 111 additions & 0 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle

import org.apache.tools.ant.BuildException
import org.apache.tools.ant.BuildListener
import org.apache.tools.ant.BuildLogger
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction

import java.nio.charset.Charset

/**
* A task which will run ant commands.
*
* Logging for the task is customizable for subclasses by overriding makeLogger.
*/
public class AntTask extends DefaultTask {

/**
* A buffer that will contain the output of the ant code run,
* if the output was not already written directly to stdout.
*/
public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()

@TaskAction
final void executeTask() {
// capture the current loggers
List<BuildLogger> savedLoggers = new ArrayList<>();
for (BuildListener l : project.ant.project.getBuildListeners()) {
if (l instanceof BuildLogger) {
savedLoggers.add(l);
}
}
// remove them
for (BuildLogger l : savedLoggers) {
project.ant.project.removeBuildListener(l)
}

final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())
BuildLogger antLogger = makeLogger(stream, outputLevel)

// now run the command with just our logger
project.ant.project.addBuildListener(antLogger)
try {
runAnt(project.ant)
} catch (BuildException e) {
// ant failed, so see if we have buffered output to emit, then rethrow the failure
String buffer = outputBuffer.toString()
if (buffer.isEmpty() == false) {
logger.error("=== Ant output ===\n${buffer}")
}
throw e
} finally {
project.ant.project.removeBuildListener(antLogger)
// add back the old loggers before returning
for (BuildLogger l : savedLoggers) {
project.ant.project.addBuildListener(l)
}
}
}

/** Runs the doAnt closure. This can be overridden by subclasses instead of having to set a closure. */
protected void runAnt(AntBuilder ant) {
if (doAnt == null) {
throw new GradleException("Missing doAnt for ${name}")
}
doAnt(ant)
}

/** Create the logger the ant runner will use, with the given stream for error/output. */
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
return new DefaultLogger(
errorPrintStream: stream,
outputPrintStream: stream,
messageOutputLevel: outputLevel)
}

/**
* Returns true if the ant logger should write to stdout, or false if to the buffer.
* The default implementation writes to the buffer when gradle info logging is disabled.
*/
protected boolean useStdout() {
return logger.isInfoEnabled()
}


}
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,33 @@
*/
package org.elasticsearch.gradle.precommit

import java.nio.file.Files

import org.gradle.api.DefaultTask
import org.apache.rat.anttasks.Report
import org.apache.rat.anttasks.SubstringLicenseMatcher
import org.apache.rat.license.SimpleLicenseFamily
import org.elasticsearch.gradle.AntTask
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskAction

import groovy.xml.NamespaceBuilder
import groovy.xml.NamespaceBuilderSupport
import java.nio.file.Files

/**
* Checks files for license headers.
* <p>
* This is a port of the apache lucene check
*/
public class LicenseHeadersTask extends DefaultTask {
public class LicenseHeadersTask extends AntTask {

LicenseHeadersTask() {
description = "Checks sources for missing, incorrect, or unacceptable license headers"

if (ant.project.taskDefinitions.contains('ratReport') == false) {
ant.project.addTaskDefinition('ratReport', Report)
ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher)
ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily)
}
}

@TaskAction
public void check() {
// load rat tasks
AntBuilder ant = new AntBuilder()
ant.typedef(resource: "org/apache/rat/anttasks/antlib.xml",
uri: "antlib:org.apache.rat.anttasks",
classpath: project.configurations.buildTools.asPath)
NamespaceBuilderSupport rat = NamespaceBuilder.newInstance(ant, "antlib:org.apache.rat.anttasks")
@Override
protected void runAnt(AntBuilder ant) {

// create a file for the log to go to under reports/
File reportDir = new File(project.buildDir, "reports/licenseHeaders")
Expand All @@ -54,7 +53,7 @@ public class LicenseHeadersTask extends DefaultTask {
Files.deleteIfExists(reportFile.toPath())

// run rat, going to the file
rat.report(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
// checks all the java sources (allJava)
for (SourceSet set : project.sourceSets) {
for (File dir : set.allJava.srcDirs) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
*/
package org.elasticsearch.gradle.precommit

import org.apache.tools.ant.DefaultLogger
import org.elasticsearch.gradle.AntTask
import org.gradle.api.artifacts.Configuration

import java.nio.file.Files
import java.nio.file.FileVisitResult
import java.nio.file.Path
Expand All @@ -35,7 +39,7 @@ import org.apache.tools.ant.Project
/**
* Basic static checking to keep tabs on third party JARs
*/
public class ThirdPartyAuditTask extends DefaultTask {
public class ThirdPartyAuditTask extends AntTask {

// true to be lenient about MISSING CLASSES
private boolean missingClasses;
Expand All @@ -46,6 +50,10 @@ public class ThirdPartyAuditTask extends DefaultTask {
ThirdPartyAuditTask() {
dependsOn(project.configurations.testCompile)
description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"

if (ant.project.taskDefinitions.contains('thirdPartyAudit') == false) {
ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask)
}
}

/**
Expand Down Expand Up @@ -84,65 +92,62 @@ public class ThirdPartyAuditTask extends DefaultTask {
return excludes;
}

@TaskAction
public void check() {
AntBuilder ant = new AntBuilder()
@Override
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
return new DefaultLogger(
errorPrintStream: stream,
outputPrintStream: stream,
// ignore passed in outputLevel for now, until we are filtering warning messages
messageOutputLevel: Project.MSG_ERR)
}

// we are noisy for many reasons, working around performance problems with forbidden-apis, dealing
// with warnings about missing classes, etc. so we use our own "quiet" AntBuilder
ant.project.buildListeners.each { listener ->
if (listener instanceof BuildLogger) {
listener.messageOutputLevel = Project.MSG_ERR;
}
};

@Override
protected void runAnt(AntBuilder ant) {
// we only want third party dependencies.
FileCollection jars = project.configurations.testCompile.fileCollection({ dependency ->
FileCollection jars = project.configurations.testCompile.fileCollection({ dependency ->
dependency.group.startsWith("org.elasticsearch") == false
})

// we don't want provided dependencies, which we have already scanned. e.g. don't
// scan ES core's dependencies for every single plugin
try {
jars -= project.configurations.getByName("provided")
} catch (UnknownConfigurationException ignored) {}

Configuration provided = project.configurations.findByName('provided')
if (provided != null) {
jars -= provided
}

// no dependencies matched, we are done
if (jars.isEmpty()) {
return;
}

ant.taskdef(name: "thirdPartyAudit",
classname: "de.thetaphi.forbiddenapis.ant.AntTask",
classpath: project.configurations.buildTools.asPath)



// print which jars we are going to scan, always
// this is not the time to try to be succinct! Forbidden will print plenty on its own!
Set<String> names = new HashSet<>()
for (File jar : jars) {
names.add(jar.getName())
}
logger.error("[thirdPartyAudit] Scanning: " + names)

// warn that classes are missing
// TODO: move these to excludes list!
if (missingClasses) {
logger.warn("[thirdPartyAudit] WARNING: CLASSES ARE MISSING! Expect NoClassDefFoundError in bug reports from users!")
}
// TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,

// TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
// and then remove our temp dir afterwards. don't complain: try it yourself.
// we don't use gradle temp dir handling, just google it, or try it yourself.

File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit')

// clean up any previous mess (if we failed), then unzip everything to one directory
ant.delete(dir: tmpDir.getAbsolutePath())
tmpDir.mkdirs()
for (File jar : jars) {
ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath())
}

// convert exclusion class names to binary file names
String[] excludedFiles = new String[excludes.length];
for (int i = 0; i < excludes.length; i++) {
Expand All @@ -152,12 +157,12 @@ public class ThirdPartyAuditTask extends DefaultTask {
throw new IllegalStateException("bogus thirdPartyAudit exclusion: '" + excludes[i] + "', not found in any dependency")
}
}

// jarHellReprise
checkSheistyClasses(tmpDir.toPath(), new HashSet<>(Arrays.asList(excludedFiles)));
ant.thirdPartyAudit(internalRuntimeForbidden: true,
failOnUnsupportedJava: false,

ant.thirdPartyAudit(internalRuntimeForbidden: true,
failOnUnsupportedJava: false,
failOnMissingClasses: !missingClasses,
classpath: project.configurations.testCompile.asPath) {
fileset(dir: tmpDir, excludes: excludedFiles.join(','))
Expand All @@ -169,7 +174,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
/**
* check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk!
*/
private void checkSheistyClasses(Path root, Set<String> excluded) {
protected void checkSheistyClasses(Path root, Set<String> excluded) {
// system.parent = extensions loader.
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
// but groovy/gradle needs to work at all first!
Expand Down

0 comments on commit 9f1dfdb

Please sign in to comment.