diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java index 69239fce6a6..807663705f7 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java @@ -1,6 +1,6 @@ package datadog.trace.civisibility.config; -import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestFQN; import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -22,13 +22,19 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) { } @Override - public Map> getFlakyTestsByModule( + public Map> getFlakyTestsByModule( TracerEnvironment tracerEnvironment) { return Collections.emptyMap(); } @Override - public Map> getKnownTestsByModule( + public Map> getKnownTestsByModule( + TracerEnvironment tracerEnvironment) { + return Collections.emptyMap(); + } + + @Override + public Map>> getTestManagementTestsByModule( TracerEnvironment tracerEnvironment) { return Collections.emptyMap(); } @@ -43,12 +49,15 @@ public ChangedFiles getChangedFiles(TracerEnvironment tracerEnvironment) { SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) throws IOException; - Map> getFlakyTestsByModule(TracerEnvironment tracerEnvironment) + Map> getFlakyTestsByModule(TracerEnvironment tracerEnvironment) throws IOException; @Nullable - Map> getKnownTestsByModule(TracerEnvironment tracerEnvironment) + Map> getKnownTestsByModule(TracerEnvironment tracerEnvironment) throws IOException; + Map>> getTestManagementTestsByModule( + TracerEnvironment tracerEnvironment) throws IOException; + ChangedFiles getChangedFiles(TracerEnvironment tracerEnvironment) throws IOException; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java index c2330e584fd..6a2c33545de 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java @@ -8,6 +8,7 @@ import datadog.communication.BackendApi; import datadog.communication.http.OkHttpUtils; import datadog.trace.api.civisibility.config.Configurations; +import datadog.trace.api.civisibility.config.TestFQN; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestMetadata; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; @@ -54,6 +55,7 @@ public class ConfigurationApiImpl implements ConfigurationApi { private static final String CHANGED_FILES_URI = "ci/tests/diffs"; private static final String FLAKY_TESTS_URI = "ci/libraries/tests/flaky"; private static final String KNOWN_TESTS_URI = "ci/libraries/tests"; + private static final String TEST_MANAGEMENT_TESTS_URI = "test/libraries/test-management/tests"; private final BackendApi backendApi; private final CiVisibilityMetricCollector metricCollector; @@ -63,6 +65,7 @@ public class ConfigurationApiImpl implements ConfigurationApi { private final JsonAdapter> settingsResponseAdapter; private final JsonAdapter> testIdentifiersResponseAdapter; private final JsonAdapter> testFullNamesResponseAdapter; + private final JsonAdapter> testManagementTestsResponseAdapter; private final JsonAdapter> changedFilesResponseAdapter; public ConfigurationApiImpl(BackendApi backendApi, CiVisibilityMetricCollector metricCollector) { @@ -105,6 +108,11 @@ public ConfigurationApiImpl(BackendApi backendApi, CiVisibilityMetricCollector m ConfigurationApiImpl.class, EnvelopeDto.class, KnownTestsDto.class); testFullNamesResponseAdapter = moshi.adapter(testFullNamesResponseType); + ParameterizedType testManagementTestsResponseType = + Types.newParameterizedTypeWithOwner( + ConfigurationApiImpl.class, EnvelopeDto.class, TestManagementTestsDto.class); + testManagementTestsResponseAdapter = moshi.adapter(testManagementTestsResponseType); + ParameterizedType changedFilesResponseAdapterType = Types.newParameterizedTypeWithOwner( ConfigurationApiImpl.class, EnvelopeDto.class, ChangedFiles.class); @@ -202,8 +210,8 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr } @Override - public Map> getFlakyTestsByModule( - TracerEnvironment tracerEnvironment) throws IOException { + public Map> getFlakyTestsByModule(TracerEnvironment tracerEnvironment) + throws IOException { OkHttpUtils.CustomListener telemetryListener = new TelemetryListener.Builder(metricCollector) .requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST) @@ -231,7 +239,7 @@ public Map> getFlakyTestsByModule( Configurations requestConf = tracerEnvironment.getConfigurations(); int flakyTestsCount = 0; - Map> testIdentifiers = new HashMap<>(); + Map> testIdentifiers = new HashMap<>(); for (DataDto dataDto : response) { TestIdentifierJson testIdentifierJson = dataDto.getAttributes(); Configurations conf = testIdentifierJson.getConfigurations(); @@ -239,7 +247,7 @@ public Map> getFlakyTestsByModule( (conf != null && conf.getTestBundle() != null ? conf : requestConf).getTestBundle(); testIdentifiers .computeIfAbsent(moduleName, k -> new HashSet<>()) - .add(testIdentifierJson.toTestIdentifier()); + .add(testIdentifierJson.toTestIdentifier().toFQN()); flakyTestsCount++; } @@ -249,8 +257,8 @@ public Map> getFlakyTestsByModule( @Nullable @Override - public Map> getKnownTestsByModule( - TracerEnvironment tracerEnvironment) throws IOException { + public Map> getKnownTestsByModule(TracerEnvironment tracerEnvironment) + throws IOException { OkHttpUtils.CustomListener telemetryListener = new TelemetryListener.Builder(metricCollector) .requestCount(CiVisibilityCountMetric.KNOWN_TESTS_REQUEST) @@ -276,10 +284,10 @@ public Map> getKnownTestsByModule( return parseTestIdentifiers(knownTests); } - private Map> parseTestIdentifiers(KnownTestsDto knownTests) { + private Map> parseTestIdentifiers(KnownTestsDto knownTests) { int knownTestsCount = 0; - Map> testIdentifiers = new HashMap<>(); + Map> testIdentifiers = new HashMap<>(); for (Map.Entry>> e : knownTests.tests.entrySet()) { String moduleName = e.getKey(); Map> testsBySuiteName = e.getValue(); @@ -292,7 +300,7 @@ private Map> parseTestIdentifiers(KnownTestsD for (String testName : testNames) { testIdentifiers .computeIfAbsent(moduleName, k -> new HashSet<>()) - .add(new TestIdentifier(suiteName, testName, null)); + .add(new TestFQN(suiteName, testName)); } } } @@ -309,6 +317,90 @@ private Map> parseTestIdentifiers(KnownTestsD : null; } + @Override + public Map>> getTestManagementTestsByModule( + TracerEnvironment tracerEnvironment) throws IOException { + OkHttpUtils.CustomListener telemetryListener = + new TelemetryListener.Builder(metricCollector) + .requestCount(CiVisibilityCountMetric.TEST_MANAGEMENT_TESTS_REQUEST) + .requestErrors(CiVisibilityCountMetric.TEST_MANAGEMENT_TESTS_REQUEST_ERRORS) + .requestDuration(CiVisibilityDistributionMetric.TEST_MANAGEMENT_TESTS_REQUEST_MS) + .responseBytes(CiVisibilityDistributionMetric.TEST_MANAGEMENT_TESTS_RESPONSE_BYTES) + .build(); + + String uuid = uuidGenerator.get(); + EnvelopeDto request = + new EnvelopeDto<>(new DataDto<>(uuid, "ci_app_libraries_tests_request", tracerEnvironment)); + String json = requestAdapter.toJson(request); + RequestBody requestBody = RequestBody.create(JSON, json); + TestManagementTestsDto testManagementTestsDto = + backendApi.post( + TEST_MANAGEMENT_TESTS_URI, + requestBody, + is -> + testManagementTestsResponseAdapter.fromJson(Okio.buffer(Okio.source(is))) + .data + .attributes, + telemetryListener, + false); + + return parseTestManagementTests(testManagementTestsDto); + } + + private Map>> parseTestManagementTests( + TestManagementTestsDto testsManagementTestsDto) { + int testManagementTestsCount = 0; + + Map> quarantinedTestsByModule = new HashMap<>(); + Map> disabledTestsByModule = new HashMap<>(); + Map> attemptToFixTestsByModule = new HashMap<>(); + + for (Map.Entry e : + testsManagementTestsDto.getModules().entrySet()) { + String moduleName = e.getKey(); + Map testsBySuiteName = e.getValue().getSuites(); + + for (Map.Entry se : testsBySuiteName.entrySet()) { + String suiteName = se.getKey(); + Map tests = se.getValue().getTests(); + + testManagementTestsCount += tests.size(); + + for (Map.Entry te : tests.entrySet()) { + String testName = te.getKey(); + TestManagementTestsDto.Properties properties = te.getValue(); + if (properties.isQuarantined()) { + quarantinedTestsByModule + .computeIfAbsent(moduleName, k -> new HashSet<>()) + .add(new TestFQN(suiteName, testName)); + } + if (properties.isDisabled()) { + disabledTestsByModule + .computeIfAbsent(moduleName, k -> new HashSet<>()) + .add(new TestFQN(suiteName, testName)); + } + if (properties.isAttemptToFix()) { + attemptToFixTestsByModule + .computeIfAbsent(moduleName, k -> new HashSet<>()) + .add(new TestFQN(suiteName, testName)); + } + } + } + } + + Map>> testsByTypeByModule = new HashMap<>(); + testsByTypeByModule.put(TestSetting.QUARANTINED, quarantinedTestsByModule); + testsByTypeByModule.put(TestSetting.DISABLED, disabledTestsByModule); + testsByTypeByModule.put(TestSetting.ATTEMPT_TO_FIX, attemptToFixTestsByModule); + + LOGGER.debug("Received {} test management tests in total", testManagementTestsCount); + metricCollector.add( + CiVisibilityDistributionMetric.TEST_MANAGEMENT_TESTS_RESPONSE_TESTS, + testManagementTestsCount); + + return testsByTypeByModule; + } + @Override public ChangedFiles getChangedFiles(TracerEnvironment tracerEnvironment) throws IOException { OkHttpUtils.CustomListener telemetryListener = @@ -427,4 +519,66 @@ private KnownTestsDto(Map>> tests) { this.tests = tests; } } + + private static final class TestManagementTestsDto { + private static final class Properties { + private final Map properties; + + private Properties(Map properties) { + this.properties = properties; + } + + public Boolean isQuarantined() { + return properties != null + ? properties.getOrDefault(TestSetting.QUARANTINED.asString(), false) + : false; + } + + public Boolean isDisabled() { + return properties != null + ? properties.getOrDefault(TestSetting.DISABLED.asString(), false) + : false; + } + + public Boolean isAttemptToFix() { + return properties != null + ? properties.getOrDefault(TestSetting.ATTEMPT_TO_FIX.asString(), false) + : false; + } + } + + private static final class Tests { + private final Map tests; + + private Tests(Map tests) { + this.tests = tests; + } + + public Map getTests() { + return tests != null ? tests : Collections.emptyMap(); + } + } + + private static final class Suites { + private final Map suites; + + private Suites(Map suites) { + this.suites = suites; + } + + public Map getSuites() { + return suites != null ? suites : Collections.emptyMap(); + } + } + + private final Map modules; + + private TestManagementTestsDto(Map modules) { + this.modules = modules; + } + + public Map getModules() { + return modules != null ? modules : Collections.emptyMap(); + } + } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java index 70a9003b2a3..57e8ee5366d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility.config; +import datadog.trace.api.civisibility.config.TestFQN; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestMetadata; import datadog.trace.civisibility.diff.Diff; @@ -9,6 +10,8 @@ import java.util.BitSet; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import javax.annotation.Nonnull; @@ -29,9 +32,11 @@ public class ExecutionSettings { null, Collections.emptyMap(), Collections.emptyMap(), - Collections.emptyList(), null, null, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), LineDiff.EMPTY); private final boolean itrEnabled; @@ -44,9 +49,8 @@ public class ExecutionSettings { @Nullable private final String itrCorrelationId; @Nonnull private final Map skippableTests; @Nonnull private final Map skippableTestsCoverage; - @Nonnull private final Collection quarantinedTests; - @Nullable private final Collection flakyTests; - @Nullable private final Collection knownTests; + @Nonnull private final Map testSettings; + @Nonnull private final Map settingsCount; @Nonnull private final Diff pullRequestDiff; public ExecutionSettings( @@ -60,9 +64,11 @@ public ExecutionSettings( @Nullable String itrCorrelationId, @Nonnull Map skippableTests, @Nonnull Map skippableTestsCoverage, - @Nonnull Collection quarantinedTests, - @Nullable Collection flakyTests, - @Nullable Collection knownTests, + @Nullable Collection flakyTests, + @Nullable Collection knownTests, + @Nonnull Collection quarantinedTests, + @Nonnull Collection disabledTests, + @Nonnull Collection attemptToFixTests, @Nonnull Diff pullRequestDiff) { this.itrEnabled = itrEnabled; this.codeCoverageEnabled = codeCoverageEnabled; @@ -74,9 +80,57 @@ public ExecutionSettings( this.itrCorrelationId = itrCorrelationId; this.skippableTests = skippableTests; this.skippableTestsCoverage = skippableTestsCoverage; - this.quarantinedTests = quarantinedTests; - this.flakyTests = flakyTests; - this.knownTests = knownTests; + this.pullRequestDiff = pullRequestDiff; + + testSettings = new HashMap<>(); + if (flakyTests != null) { + flakyTests.forEach(fqn -> addTest(fqn, TestSetting.FLAKY)); + } + if (knownTests != null) { + knownTests.forEach(fqn -> addTest(fqn, TestSetting.KNOWN)); + } + quarantinedTests.forEach(fqn -> addTest(fqn, TestSetting.QUARANTINED)); + disabledTests.forEach(fqn -> addTest(fqn, TestSetting.DISABLED)); + attemptToFixTests.forEach(fqn -> addTest(fqn, TestSetting.ATTEMPT_TO_FIX)); + + settingsCount = new EnumMap<>(TestSetting.class); + settingsCount.put(TestSetting.FLAKY, flakyTests != null ? flakyTests.size() : -1); + settingsCount.put(TestSetting.KNOWN, knownTests != null ? knownTests.size() : -1); + settingsCount.put(TestSetting.QUARANTINED, quarantinedTests.size()); + settingsCount.put(TestSetting.DISABLED, disabledTests.size()); + settingsCount.put(TestSetting.ATTEMPT_TO_FIX, attemptToFixTests.size()); + } + + private void addTest(TestFQN test, TestSetting setting) { + testSettings.merge(test, setting.getFlag(), (a, b) -> a | b); + } + + private ExecutionSettings( + boolean itrEnabled, + boolean codeCoverageEnabled, + boolean testSkippingEnabled, + boolean flakyTestRetriesEnabled, + boolean impactedTestsDetectionEnabled, + @Nonnull EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, + @Nonnull TestManagementSettings testManagementSettings, + @Nullable String itrCorrelationId, + @Nonnull Map skippableTests, + @Nonnull Map skippableTestsCoverage, + @Nonnull Map testSettings, + @Nonnull EnumMap settingsCount, + @Nonnull Diff pullRequestDiff) { + this.itrEnabled = itrEnabled; + this.codeCoverageEnabled = codeCoverageEnabled; + this.testSkippingEnabled = testSkippingEnabled; + this.flakyTestRetriesEnabled = flakyTestRetriesEnabled; + this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled; + this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings; + this.testManagementSettings = testManagementSettings; + this.itrCorrelationId = itrCorrelationId; + this.skippableTests = skippableTests; + this.skippableTestsCoverage = skippableTestsCoverage; + this.testSettings = testSettings; + this.settingsCount = settingsCount; this.pullRequestDiff = pullRequestDiff; } @@ -119,38 +173,56 @@ public String getItrCorrelationId() { return itrCorrelationId; } + @Nonnull + public Map getSkippableTests() { + return skippableTests; + } + /** A bit vector of covered lines by relative source file path. */ @Nonnull public Map getSkippableTestsCoverage() { return skippableTestsCoverage; } - @Nonnull - public Map getSkippableTests() { - return skippableTests; + public boolean isFlakyTestsDataAvailable() { + return getSettingCount(TestSetting.FLAKY) != -1; } - @Nonnull - public Collection getQuarantinedTests() { - return quarantinedTests; + public boolean isKnownTestsDataAvailable() { + return getSettingCount(TestSetting.KNOWN) != -1; } - /** - * @return the list of known tests for the given module (can be empty), or {@code null} if known - * tests could not be obtained - */ - @Nullable - public Collection getKnownTests() { - return knownTests; + private boolean isSetting(TestFQN test, TestSetting setting) { + int mask = testSettings.getOrDefault(test, 0); + return TestSetting.isSet(mask, setting); + } + + public boolean isFlaky(TestFQN test) { + return isSetting(test, TestSetting.FLAKY); + } + + public boolean isKnown(TestFQN test) { + return isSetting(test, TestSetting.KNOWN); + } + + public boolean isQuarantined(TestFQN test) { + return isSetting(test, TestSetting.QUARANTINED); + } + + public boolean isDisabled(TestFQN test) { + return isSetting(test, TestSetting.DISABLED); + } + + public boolean isAttemptToFix(TestFQN test) { + return isSetting(test, TestSetting.ATTEMPT_TO_FIX); } /** - * @return the list of flaky tests for the given module (can be empty), or {@code null} if flaky - * tests could not be obtained + * @return number of tests with a certain setting "activated". For flaky and known test, it will + * return -1 if a null value was provided to the constructor. */ - @Nullable - public Collection getFlakyTests() { - return flakyTests; + public int getSettingCount(TestSetting setting) { + return settingsCount.getOrDefault(setting, 0); } @Nonnull @@ -170,14 +242,15 @@ public boolean equals(Object o) { return itrEnabled == that.itrEnabled && codeCoverageEnabled == that.codeCoverageEnabled && testSkippingEnabled == that.testSkippingEnabled + && flakyTestRetriesEnabled == that.flakyTestRetriesEnabled + && impactedTestsDetectionEnabled == that.impactedTestsDetectionEnabled && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings) && Objects.equals(testManagementSettings, that.testManagementSettings) && Objects.equals(itrCorrelationId, that.itrCorrelationId) && Objects.equals(skippableTests, that.skippableTests) && Objects.equals(skippableTestsCoverage, that.skippableTestsCoverage) - && Objects.equals(quarantinedTests, that.quarantinedTests) - && Objects.equals(flakyTests, that.flakyTests) - && Objects.equals(knownTests, that.knownTests) + && Objects.equals(testSettings, that.testSettings) + && Objects.equals(settingsCount, that.settingsCount) && Objects.equals(pullRequestDiff, that.pullRequestDiff); } @@ -187,14 +260,15 @@ public int hashCode() { itrEnabled, codeCoverageEnabled, testSkippingEnabled, + flakyTestRetriesEnabled, + impactedTestsDetectionEnabled, earlyFlakeDetectionSettings, testManagementSettings, itrCorrelationId, skippableTests, skippableTestsCoverage, - quarantinedTests, - flakyTests, - knownTests, + testSettings, + settingsCount, pullRequestDiff); } @@ -231,9 +305,10 @@ public static ByteBuffer serialize(ExecutionSettings settings) { TestMetadataSerializer::serialize); s.write(settings.skippableTestsCoverage, Serializer::write, Serializer::write); - s.write(settings.quarantinedTests, TestIdentifierSerializer::serialize); - s.write(settings.flakyTests, TestIdentifierSerializer::serialize); - s.write(settings.knownTests, TestIdentifierSerializer::serialize); + + s.write(settings.testSettings, TestFQNSerializer::serialize, Serializer::write); + s.write( + settings.settingsCount, TestSetting.TestSettingsSerializer::serialize, Serializer::write); Diff.SERIALIZER.serialize(settings.pullRequestDiff, s); @@ -262,12 +337,16 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { Map skippableTestsCoverage = Serializer.readMap(buffer, Serializer::readString, Serializer::readBitSet); - Collection quarantinedTests = - Serializer.readSet(buffer, TestIdentifierSerializer::deserialize); - Collection flakyTests = - Serializer.readSet(buffer, TestIdentifierSerializer::deserialize); - Collection knownTests = - Serializer.readSet(buffer, TestIdentifierSerializer::deserialize); + + Map testSettings = + Serializer.readMap(buffer, TestFQNSerializer::deserialize, Serializer::readInt); + EnumMap settingsCount = + (EnumMap) + Serializer.readMap( + buffer, + () -> new EnumMap<>(TestSetting.class), + TestSetting.TestSettingsSerializer::deserialize, + Serializer::readInt); Diff diff = Diff.SERIALIZER.deserialize(buffer); @@ -282,9 +361,8 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { itrCorrelationId, skippableTests, skippableTestsCoverage, - quarantinedTests, - flakyTests, - knownTests, + testSettings, + settingsCount, diff); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java index 916661efa13..3ad98f2f02d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java @@ -3,7 +3,7 @@ import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; -import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestFQN; import datadog.trace.api.git.GitInfo; import datadog.trace.api.git.GitInfoProvider; import datadog.trace.civisibility.ci.PullRequestInfo; @@ -205,21 +205,44 @@ private Map doCreate( Future skippableTestsFuture = executor.submit(() -> getSkippableTests(tracerEnvironment, itrEnabled)); - Future>> flakyTestsFuture = + Future>> flakyTestsFuture = executor.submit(() -> getFlakyTestsByModule(tracerEnvironment, flakyTestRetriesEnabled)); - Future>> knownTestsFuture = + Future>> knownTestsFuture = executor.submit(() -> getKnownTestsByModule(tracerEnvironment, knownTestsRequest)); + Future>>> testManagementTestsFuture = + executor.submit( + () -> + getTestManagementTestsByModule( + tracerEnvironment, testManagementSettings.isEnabled())); Future pullRequestDiffFuture = executor.submit(() -> getPullRequestDiff(tracerEnvironment, impactedTestsEnabled)); SkippableTests skippableTests = skippableTestsFuture.get(); - Map> flakyTestsByModule = flakyTestsFuture.get(); - Map> knownTestsByModule = knownTestsFuture.get(); + Map> flakyTestsByModule = flakyTestsFuture.get(); + Map> knownTestsByModule = knownTestsFuture.get(); + + Map>> testManagementTestsByModule = + testManagementTestsFuture.get(); + Map> quarantinedTestsByModule = + testManagementTestsByModule.getOrDefault(TestSetting.QUARANTINED, Collections.emptyMap()); + Map> disabledTestsByModule = + testManagementTestsByModule.getOrDefault(TestSetting.DISABLED, Collections.emptyMap()); + Map> attemptToFixTestsByModule = + testManagementTestsByModule.getOrDefault( + TestSetting.ATTEMPT_TO_FIX, Collections.emptyMap()); + Diff pullRequestDiff = pullRequestDiffFuture.get(); Map settingsByModule = new HashMap<>(); Set moduleNames = - getModuleNames(skippableTests, flakyTestsByModule, knownTestsByModule); + getModuleNames( + skippableTests, + flakyTestsByModule, + knownTestsByModule, + quarantinedTestsByModule, + disabledTestsByModule, + attemptToFixTestsByModule); + for (String moduleName : moduleNames) { settingsByModule.put( moduleName, @@ -238,11 +261,13 @@ private Map doCreate( .getIdentifiersByModule() .getOrDefault(moduleName, Collections.emptyMap()), skippableTests.getCoveredLinesByRelativeSourcePath(), - Collections.emptyList(), // FIXME implement retrieving quarantined tests flakyTestsByModule != null ? flakyTestsByModule.getOrDefault(moduleName, Collections.emptyList()) : null, knownTestsByModule != null ? knownTestsByModule.get(moduleName) : null, + quarantinedTestsByModule.getOrDefault(moduleName, Collections.emptyList()), + disabledTestsByModule.getOrDefault(moduleName, Collections.emptyList()), + attemptToFixTestsByModule.getOrDefault(moduleName, Collections.emptyList()), pullRequestDiff)); } return settingsByModule; @@ -334,7 +359,7 @@ private SkippableTests getSkippableTests( } @Nullable - private Map> getFlakyTestsByModule( + private Map> getFlakyTestsByModule( TracerEnvironment tracerEnvironment, boolean flakyTestRetriesEnabled) { if (!(flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()) && !CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())) { @@ -349,7 +374,7 @@ private Map> getFlakyTestsByModule( } @Nullable - private Map> getKnownTestsByModule( + private Map> getKnownTestsByModule( TracerEnvironment tracerEnvironment, boolean knownTestsRequest) { if (!knownTestsRequest) { return null; @@ -363,6 +388,21 @@ private Map> getKnownTestsByModule( } } + @Nullable + private Map>> getTestManagementTestsByModule( + TracerEnvironment tracerEnvironment, boolean testManagementTestsRequest) { + if (!testManagementTestsRequest) { + return Collections.emptyMap(); + } + try { + return configurationApi.getTestManagementTestsByModule(tracerEnvironment); + + } catch (Exception e) { + LOGGER.error("Could not obtain list of test management tests", e); + return Collections.emptyMap(); + } + } + @Nonnull private Diff getPullRequestDiff( TracerEnvironment tracerEnvironment, boolean impactedTestsDetectionEnabled) { @@ -420,9 +460,12 @@ private Diff getPullRequestDiff( @Nonnull private static Set getModuleNames( - SkippableTests skippableTests, - Map> flakyTestsByModule, - Map> knownTestsByModule) { + @Nonnull SkippableTests skippableTests, + @Nullable Map> flakyTestsByModule, + @Nullable Map> knownTestsByModule, + @Nonnull Map> quarantinedTestsByModule, + @Nonnull Map> disabledTestsByModule, + @Nonnull Map> attemptToFixTestsByModule) { Set moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS)); moduleNames.addAll(skippableTests.getIdentifiersByModule().keySet()); if (flakyTestsByModule != null) { @@ -431,6 +474,9 @@ private static Set getModuleNames( if (knownTestsByModule != null) { moduleNames.addAll(knownTestsByModule.keySet()); } + moduleNames.addAll(quarantinedTestsByModule.keySet()); + moduleNames.addAll(disabledTestsByModule.keySet()); + moduleNames.addAll(attemptToFixTestsByModule.keySet()); return moduleNames; } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestFQNSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestFQNSerializer.java new file mode 100644 index 00000000000..1928facd378 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestFQNSerializer.java @@ -0,0 +1,19 @@ +package datadog.trace.civisibility.config; + +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.civisibility.ipc.serialization.Serializer; +import java.nio.ByteBuffer; + +public abstract class TestFQNSerializer { + public static void serialize(Serializer serializer, TestFQN testFQN) { + serializer.write(testFQN.getSuite()); + serializer.write(testFQN.getName()); + } + + public static TestFQN deserialize(ByteBuffer buffer) { + String suiteName = Serializer.readString(buffer); + return new TestFQN( + // suite name repeats a lot; interning it to save memory + suiteName != null ? suiteName.intern() : null, Serializer.readString(buffer)); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java index 27c57702a00..df179695366 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java @@ -7,17 +7,11 @@ public abstract class TestIdentifierSerializer { public static void serialize(Serializer serializer, TestIdentifier testIdentifier) { - serializer.write(testIdentifier.getSuite()); - serializer.write(testIdentifier.getName()); + TestFQNSerializer.serialize(serializer, testIdentifier.toFQN()); serializer.write(testIdentifier.getParameters()); } public static TestIdentifier deserialize(ByteBuffer buffer) { - String suiteName = Serializer.readString(buffer); - return new TestIdentifier( - // suite name repeats a lot; interning it to save memory - suiteName != null ? suiteName.intern() : null, - Serializer.readString(buffer), - Serializer.readString(buffer)); + return new TestIdentifier(TestFQNSerializer.deserialize(buffer), Serializer.readString(buffer)); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestSetting.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestSetting.java new file mode 100644 index 00000000000..130287cd91b --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestSetting.java @@ -0,0 +1,52 @@ +package datadog.trace.civisibility.config; + +import datadog.trace.civisibility.ipc.serialization.Serializer; +import java.nio.ByteBuffer; + +public enum TestSetting { + FLAKY(1, "flaky"), + KNOWN(2, "known"), + QUARANTINED(4, "quarantined"), + DISABLED(8, "disabled"), + ATTEMPT_TO_FIX(16, "attempt_to_fix"); + + private final int flag; + private final String name; + + TestSetting(int flag, String name) { + this.flag = flag; + this.name = name; + } + + public int getFlag() { + return flag; + } + + public String asString() { + return name; + } + + public static int set(int mask, TestSetting setting) { + return mask | setting.flag; + } + + public static boolean isSet(int mask, TestSetting setting) { + return (mask & setting.flag) != 0; + } + + public static class TestSettingsSerializer { + public static void serialize(Serializer serializer, TestSetting setting) { + serializer.write(setting.flag); + } + + public static TestSetting deserialize(ByteBuffer buf) { + int flag = Serializer.readInt(buf); + for (TestSetting setting : TestSetting.values()) { + if (setting.flag == flag) { + return setting; + } + } + return null; + } + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/serialization/Serializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/serialization/Serializer.java index 3c03dad6e21..60feee6fd9d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/serialization/Serializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/serialization/Serializer.java @@ -13,6 +13,7 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.function.Supplier; public class Serializer { @@ -183,6 +184,28 @@ public static Map readMap( return null; } Map m = new HashMap<>(size * 4 / 3); + return fillMap(byteBuffer, m, keyDeserializer, valueDeserializer, size); + } + + public static Map readMap( + ByteBuffer byteBuffer, + Supplier> mapSupplier, + Function keyDeserializer, + Function valueDeserializer) { + int size = byteBuffer.getInt(); + if (size == -1) { + return null; + } + Map m = mapSupplier.get(); + return fillMap(byteBuffer, m, keyDeserializer, valueDeserializer, size); + } + + private static Map fillMap( + ByteBuffer byteBuffer, + Map m, + Function keyDeserializer, + Function valueDeserializer, + int size) { for (int i = 0; i < size; i++) { m.put(keyDeserializer.apply(byteBuffer), valueDeserializer.apply(byteBuffer)); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index 2cbc488b02c..c1cb17ab1c5 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -9,6 +9,7 @@ import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings; import datadog.trace.civisibility.config.ExecutionSettings; import datadog.trace.civisibility.config.TestManagementSettings; +import datadog.trace.civisibility.config.TestSetting; import datadog.trace.civisibility.execution.Regular; import datadog.trace.civisibility.execution.RetryUntilSuccessful; import datadog.trace.civisibility.execution.RunNTimes; @@ -16,7 +17,6 @@ import datadog.trace.civisibility.source.LinesResolver; import datadog.trace.civisibility.source.SourcePathResolver; import java.lang.reflect.Method; -import java.util.Collection; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; @@ -53,13 +53,12 @@ public ExecutionSettings getExecutionSettings() { } public boolean isNew(TestIdentifier test) { - Collection knownTests = executionSettings.getKnownTests(); - return knownTests != null && !knownTests.contains(test.withoutParameters()); + return executionSettings.isKnownTestsDataAvailable() + && !executionSettings.isKnown(test.toFQN()); } public boolean isFlaky(TestIdentifier test) { - Collection flakyTests = executionSettings.getFlakyTests(); - return flakyTests != null && flakyTests.contains(test.withoutParameters()); + return executionSettings.isFlaky(test.toFQN()); } public boolean isQuarantined(TestIdentifier test) { @@ -67,8 +66,23 @@ public boolean isQuarantined(TestIdentifier test) { if (!testManagementSettings.isEnabled()) { return false; } - Collection quarantinedTests = executionSettings.getQuarantinedTests(); - return quarantinedTests.contains(test.withoutParameters()); + return executionSettings.isQuarantined(test.toFQN()); + } + + public boolean isDisabled(TestIdentifier test) { + TestManagementSettings testManagementSettings = executionSettings.getTestManagementSettings(); + if (!testManagementSettings.isEnabled()) { + return false; + } + return executionSettings.isDisabled(test.toFQN()); + } + + public boolean isAttemptToFix(TestIdentifier test) { + TestManagementSettings testManagementSettings = executionSettings.getTestManagementSettings(); + if (!testManagementSettings.isEnabled()) { + return false; + } + return executionSettings.isAttemptToFix(test.toFQN()); } @Nullable @@ -122,8 +136,8 @@ private boolean isAutoRetryApplicable(TestIdentifier test) { return false; } - Collection flakyTests = executionSettings.getFlakyTests(); - return (flakyTests == null || flakyTests.contains(test.withoutParameters())) + return (!executionSettings.isFlakyTestsDataAvailable() + || executionSettings.isFlaky(test.toFQN())) && autoRetriesUsed.get() < config.getCiVisibilityTotalFlakyRetryCount(); } @@ -135,13 +149,12 @@ private boolean isEFDApplicable(TestIdentifier test, TestSourceData testSource) } public boolean isEFDLimitReached() { - Collection knownTests = executionSettings.getKnownTests(); - if (knownTests == null) { + if (!executionSettings.isKnownTestsDataAvailable()) { return false; } int detectionsUsed = earlyFlakeDetectionsUsed.get(); - int totalTests = knownTests.size() + detectionsUsed; + int totalTests = executionSettings.getSettingCount(TestSetting.KNOWN) + detectionsUsed; EarlyFlakeDetectionSettings earlyFlakeDetectionSettings = executionSettings.getEarlyFlakeDetectionSettings(); int threshold = diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy index a7b2e50640c..a36c93aedba 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy @@ -7,6 +7,7 @@ import datadog.communication.IntakeApi import datadog.communication.http.HttpRetryPolicy import datadog.communication.http.OkHttpUtils import datadog.trace.agent.test.server.http.TestHttpServer +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.config.TestMetadata import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector @@ -133,11 +134,11 @@ class ConfigurationApiImplTest extends Specification { where: testBundle | request | response | expectedTests null | "flaky-request.ftl" | "flaky-response.ftl" | [ - "testBundle-a": new HashSet<>([new TestIdentifier("suite-a", "name-a", "parameters-a")]), - "testBundle-b": new HashSet<>([new TestIdentifier("suite-b", "name-b", "parameters-b")]), + "testBundle-a": new HashSet<>([new TestFQN("suite-a", "name-a")]), + "testBundle-b": new HashSet<>([new TestFQN("suite-b", "name-b")]), ] "testBundle-a" | "flaky-request-one-module.ftl" | "flaky-response-one-module.ftl" | [ - "testBundle-a": new HashSet<>([new TestIdentifier("suite-a", "name-a", "parameters-a")]) + "testBundle-a": new HashSet<>([new TestFQN("suite-a", "name-a")]) ] } @@ -158,23 +159,23 @@ class ConfigurationApiImplTest extends Specification { when: def knownTests = configurationApi.getKnownTestsByModule(tracerEnvironment) - for (Map.Entry> e : knownTests.entrySet()) { + for (Map.Entry> e : knownTests.entrySet()) { def sortedTests = new ArrayList<>(e.value) - Collections.sort(sortedTests, Comparator.comparing(TestIdentifier::getSuite).thenComparing((Function) TestIdentifier::getName)) + Collections.sort(sortedTests, Comparator.comparing(TestFQN::getSuite).thenComparing((Function)TestFQN::getName)) e.value = sortedTests } then: knownTests == [ "test-bundle-a": [ - new TestIdentifier("test-suite-a", "test-name-1", null), - new TestIdentifier("test-suite-a", "test-name-2", null), - new TestIdentifier("test-suite-b", "another-test-name-1", null), - new TestIdentifier("test-suite-b", "test-name-2", null) + new TestFQN("test-suite-a", "test-name-1"), + new TestFQN("test-suite-a", "test-name-2"), + new TestFQN("test-suite-b", "another-test-name-1"), + new TestFQN("test-suite-b", "test-name-2") ], "test-bundle-N": [ - new TestIdentifier("test-suite-M", "test-name-1", null), - new TestIdentifier("test-suite-M", "test-name-2", null) + new TestFQN("test-suite-M", "test-name-1"), + new TestFQN("test-suite-M", "test-name-2") ] ] @@ -182,6 +183,44 @@ class ConfigurationApiImplTest extends Specification { intakeServer.close() } + def "test test management tests request"() { + given: + def tracerEnvironment = givenTracerEnvironment() + + def intakeServer = givenBackendEndpoint( + "/api/v2/test/libraries/test-management/tests", + "/datadog/trace/civisibility/config/test-management-tests-request.ftl", + [uid: REQUEST_UID, tracerEnvironment: tracerEnvironment], + "/datadog/trace/civisibility/config/test-management-tests-response.ftl", + [:] + ) + + def configurationApi = givenConfigurationApi(intakeServer) + + when: + def testManagementTests = configurationApi.getTestManagementTestsByModule(tracerEnvironment) + def quarantinedTests = testManagementTests.get(TestSetting.QUARANTINED) + def disabledTests = testManagementTests.get(TestSetting.DISABLED) + def attemptToFixTests = testManagementTests.get(TestSetting.ATTEMPT_TO_FIX) + + then: + quarantinedTests == [ + "module-a": new HashSet<>([new TestFQN("suite-a", "test-a"), new TestFQN("suite-b", "test-c")]), + "module-b": new HashSet<>([new TestFQN("suite-c", "test-e")]) + ] + disabledTests == [ + "module-a": new HashSet<>([new TestFQN("suite-a", "test-b")]), + "module-b": new HashSet<>([new TestFQN("suite-c", "test-d"), new TestFQN("suite-c", "test-f")]) + ] + attemptToFixTests == [ + "module-a": new HashSet<>([new TestFQN("suite-b", "test-c")]), + "module-b": new HashSet<>([new TestFQN("suite-c", "test-d"), new TestFQN("suite-c", "test-e")]) + ] + + cleanup: + intakeServer.close() + } + def "test changed files request"() { given: def tracerEnvironment = givenTracerEnvironment() diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy index ccd3cc78ff8..73776ecbd9b 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy @@ -1,5 +1,6 @@ package datadog.trace.civisibility.config +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.config.TestMetadata import datadog.trace.civisibility.diff.LineDiff @@ -30,9 +31,11 @@ class ExecutionSettingsTest extends Specification { null, [:], [:], - new HashSet<>([]), null, new HashSet<>([]), + new HashSet<>([]), + new HashSet<>([]), + new HashSet<>([]), LineDiff.EMPTY), new ExecutionSettings( @@ -46,9 +49,11 @@ class ExecutionSettingsTest extends Specification { "", [(new TestIdentifier("bc", "def", "g")): new TestMetadata(true), (new TestIdentifier("de", "f", null)): new TestMetadata(false)], [:], - new HashSet<>([new TestIdentifier("suite", "quarantined", null)]), - new HashSet<>([new TestIdentifier("name", null, null)]), - new HashSet<>([new TestIdentifier("b", "c", "g")]), + new HashSet<>([new TestFQN("name", null)]), + new HashSet<>([new TestFQN("b", "c")]), + new HashSet<>([new TestFQN("suite", "quarantined")]), + new HashSet<>([new TestFQN("suite", "disabled")]), + new HashSet<>([new TestFQN("suite", "attemptToFix")]), new LineDiff(["path": lines()]) ), @@ -67,9 +72,11 @@ class ExecutionSettingsTest extends Specification { }), "cov2": BitSet.valueOf(new byte[]{ 4, 5, 6 })], - new HashSet<>([new TestIdentifier("suite", "quarantined", null), new TestIdentifier("another", "another-quarantined", null)]), - new HashSet<>([new TestIdentifier("name", null, "g"), new TestIdentifier("b", "c", null)]), - new HashSet<>([new TestIdentifier("b", "c", null), new TestIdentifier("bb", "cc", null)]), + new HashSet<>([new TestFQN("name", null), new TestFQN("b", "c")]), + new HashSet<>([new TestFQN("b", "c"), new TestFQN("bb", "cc")]), + new HashSet<>([new TestFQN("suite", "quarantined"), new TestFQN("another", "another-quarantined")]), + new HashSet<>([new TestFQN("suite", "disabled"), new TestFQN("another", "another-disabled")]), + new HashSet<>([new TestFQN("suite", "attemptToFix"), new TestFQN("another", "another-attemptToFix")]), new LineDiff(["path": lines(1, 2, 3)]), ), @@ -88,9 +95,11 @@ class ExecutionSettingsTest extends Specification { }), "cov2": BitSet.valueOf(new byte[]{ 4, 5, 6 })], - new HashSet<>([new TestIdentifier("suite", "quarantined", null), new TestIdentifier("another", "another-quarantined", null)]), new HashSet<>([]), - new HashSet<>([new TestIdentifier("b", "c", null), new TestIdentifier("bb", "cc", "g")]), + new HashSet<>([new TestFQN("b", "c"), new TestFQN("bb", "cc")]), + new HashSet<>([new TestFQN("suite", "quarantined"), new TestFQN("another", "another-quarantined")]), + new HashSet<>([new TestFQN("suite", "disabled"), new TestFQN("another", "another-disabled")]), + new HashSet<>([new TestFQN("suite", "attemptToFix"), new TestFQN("another", "another-attemptToFix")]), new LineDiff(["path": lines(1, 2, 3), "path-b": lines(1, 2, 128, 257, 999)]), ), ] diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestFQNSerializerTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestFQNSerializerTest.groovy new file mode 100644 index 00000000000..7f46be03450 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestFQNSerializerTest.groovy @@ -0,0 +1,37 @@ +package datadog.trace.civisibility.config + +import datadog.trace.api.civisibility.config.TestFQN +import datadog.trace.civisibility.ipc.serialization.Serializer +import spock.lang.Specification + +class TestFQNSerializerTest extends Specification { + + def "test serialization: #tests"() { + given: + when: + Serializer s = new Serializer() + s.write(tests, TestFQNSerializer::serialize) + def serializedTests = s.flush() + def deserializedTests = Serializer.readList(serializedTests, TestFQNSerializer::deserialize) + + then: + deserializedTests == tests + + where: + tests << [ + // empty + [], + // single test + [new TestFQN("suite", "name")], + [new TestFQN("suite", "𝕄 add user properties 𝕎 addUserProperties()")], + [new TestFQN("suite", "name")], + [new TestFQN("suite", "name")], + [new TestFQN("suite", "name")], + // multiple tests + [new TestFQN("suite", "name"), new TestFQN("a", "b")], + [new TestFQN("suite", "name"), new TestFQN("a", "b")], + [new TestFQN("suite", "name"), new TestFQN("a", "b")], + [new TestFQN("suite", "name"), new TestFQN("a", "b")], + ] + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestSettingsTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestSettingsTest.groovy new file mode 100644 index 00000000000..810df8a76f0 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/TestSettingsTest.groovy @@ -0,0 +1,32 @@ +package datadog.trace.civisibility.config + + +import datadog.trace.civisibility.ipc.serialization.Serializer +import spock.lang.Specification + +class TestSettingsTest extends Specification { + + def "test serialization: #tests"() { + given: + when: + Serializer s = new Serializer() + s.write(tests, TestSetting.TestSettingsSerializer::serialize) + def serializedTests = s.flush() + def deserializedTests = Serializer.readList(serializedTests, TestSetting.TestSettingsSerializer::deserialize) + + then: + deserializedTests == tests + + where: + tests << [ + // empty + [], + [TestSetting.FLAKY], + [TestSetting.KNOWN], + [TestSetting.QUARANTINED], + [TestSetting.DISABLED], + [TestSetting.ATTEMPT_TO_FIX], + [TestSetting.ATTEMPT_TO_FIX, TestSetting.QUARANTINED], + ] + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/serialization/SerializerTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/serialization/SerializerTest.groovy index efb45a67799..dad95d8c954 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/serialization/SerializerTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/serialization/SerializerTest.groovy @@ -119,6 +119,31 @@ class SerializerTest extends Specification { m << [null, [:], ["a": "b"], ["a": "b", "1": "2"], [null: "b", "1": null]] } + def "test map deserialization with provider: #m #clazz #provider"() { + given: + def serializer = new Serializer() + + when: + serializer.write((Map) m, keySerializer, Serializer::write) + def buf = serializer.flush() + + then: + def deserializedMap = Serializer.readMap(buf, provider, keyDeserializer, Serializer::readString) + deserializedMap == m + deserializedMap.getClass() == clazz + + where: + m | clazz | provider | keySerializer | keyDeserializer + [:] | HashMap | HashMap::new | Serializer::write | Serializer::readString + ["a": "1", "b": "2"] | HashMap | HashMap::new | Serializer::write | Serializer::readString + [:] | LinkedHashMap | LinkedHashMap::new | Serializer::write | Serializer::readString + ["a": "1", "b": "2"] | LinkedHashMap | LinkedHashMap::new | Serializer::write | Serializer::readString + [:] | TreeMap | TreeMap::new | Serializer::write | Serializer::readString + ["a": "1", "b": "2"] | TreeMap | TreeMap::new | Serializer::write | Serializer::readString + [:] | EnumMap | (() -> new EnumMap<>(MyEnum.class)) | MyEnum::serialize | MyEnum::deserialize + [(MyEnum.A): "1", (MyEnum.B): "2"] | EnumMap | (() -> new EnumMap<>(MyEnum.class)) | MyEnum::serialize | MyEnum::deserialize + } + def "test mixed serialization"() { given: def serializer = new Serializer() @@ -205,4 +230,30 @@ class SerializerTest extends Specification { return new MyPojo(Serializer.readInt(buf), Serializer.readString(buf)) } } + + private enum MyEnum { + A(1), + B(2), + C(3) + + public final int val + + MyEnum(int val) { + this.val = val + } + + private static void serialize(Serializer serializer, MyEnum en) { + serializer.write(en.val) + } + + private static MyEnum deserialize(ByteBuffer buf) { + int val = Serializer.readInt(buf) + for (MyEnum en : values()) { + if (en.val == val) { + return en + } + } + return null + } + } } diff --git a/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-request.ftl b/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-request.ftl new file mode 100644 index 00000000000..4b9593e0a43 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-request.ftl @@ -0,0 +1,29 @@ +{ + "data": { + "type" : "ci_app_libraries_tests_request", + "id" : "${uid}", + "attributes": { + "service" : "${tracerEnvironment.service}", + "env" : "${tracerEnvironment.env}", + "repository_url": "${tracerEnvironment.repositoryUrl}", + "branch" : "${tracerEnvironment.branch}", + "sha" : "${tracerEnvironment.sha}", + "test_level" : "${tracerEnvironment.testLevel}", + "configurations": { + "os.platform" : "${tracerEnvironment.configurations.osPlatform}", + "os.architecture" : "${tracerEnvironment.configurations.osArchitecture}", + "os.arch" : "${tracerEnvironment.configurations.osArchitecture}", + "os.version" : "${tracerEnvironment.configurations.osVersion}", + "runtime.name" : "${tracerEnvironment.configurations.runtimeName}", + "runtime.version" : "${tracerEnvironment.configurations.runtimeVersion}", + "runtime.vendor" : "${tracerEnvironment.configurations.runtimeVendor}", + "runtime.architecture": "${tracerEnvironment.configurations.runtimeArchitecture}", + "custom" : { + <#list tracerEnvironment.configurations.custom as customTag, customValue> + "${customTag}": "${customValue}"<#if customTag?has_next>, + + } + } + } + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-response.ftl b/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-response.ftl new file mode 100644 index 00000000000..c455100b8f2 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/test-management-tests-response.ftl @@ -0,0 +1,69 @@ +{ + "data": { + "id": "9p1jTQLXB8g", + "type": "ci_app_libraries_tests", + "attributes": { + "modules": { + "module-a": { + "suites": { + "suite-a": { + "tests": { + "test-a": { + "properties": { + "quarantined": true, + "disabled": false, + "attempt_to_fix": false + } + }, + "test-b": { + "properties": { + "quarantined": false, + "disabled": true, + "attempt_to_fix": false + } + } + } + }, + "suite-b": { + "tests": { + "test-c": { + "properties": { + "quarantined": true, + "disabled": false, + "attempt_to_fix": true + } + } + } + } + } + }, + "module-b": { + "suites": { + "suite-c": { + "tests": { + "test-d": { + "properties": { + "quarantined": false, + "disabled": true, + "attempt_to_fix": true + } + }, + "test-e": { + "properties": { + "quarantined": true, + "attempt_to_fix": true + } + }, + "test-f": { + "properties": { + "disabled": true + } + } + } + } + } + } + } + } + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index 18247029797..e35c294871b 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -9,6 +9,7 @@ import datadog.trace.api.civisibility.CIConstants import datadog.trace.api.civisibility.DDTest import datadog.trace.api.civisibility.DDTestSuite import datadog.trace.api.civisibility.InstrumentationBridge +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.config.TestMetadata import datadog.trace.api.civisibility.coverage.CoveragePerTestBridge @@ -71,9 +72,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { private static Path agentKeyFile private static final List skippableTests = new ArrayList<>() - private static final List quarantinedTests = new ArrayList<>() - private static final List flakyTests = new ArrayList<>() - private static final List knownTests = new ArrayList<>() + private static final List flakyTests = new ArrayList<>() + private static final List knownTests = new ArrayList<>() + private static final List quarantinedTests = new ArrayList<>() + private static final List disabledTests = new ArrayList<>() + private static final List attemptToFixTests = new ArrayList<>() private static volatile Diff diff = LineDiff.EMPTY private static volatile boolean itrEnabled = false @@ -137,9 +140,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { itrEnabled ? "itrCorrelationId" : null, skippableTestsWithMetadata, [:], - quarantinedTests, flakyTests, earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null, + quarantinedTests, + disabledTests, + attemptToFixTests, diff) } } @@ -259,9 +264,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { @Override void setup() { skippableTests.clear() - quarantinedTests.clear() flakyTests.clear() knownTests.clear() + quarantinedTests.clear() + disabledTests.clear() + attemptToFixTests.clear() diff = LineDiff.EMPTY itrEnabled = false flakyRetryEnabled = false @@ -277,18 +284,26 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { itrEnabled = true } - def givenFlakyTests(List tests) { + def givenFlakyTests(List tests) { flakyTests.addAll(tests) } - def givenKnownTests(List tests) { + def givenKnownTests(List tests) { knownTests.addAll(tests) } - def givenQuarantinedTests(List tests) { + def givenQuarantinedTests(List tests) { quarantinedTests.addAll(tests) } + def givenDisabledTests(List tests) { + disabledTests.addAll(tests) + } + + def givenAttemptToFixTests(List tests) { + attemptToFixTests.addAll(tests) + } + def givenDiff(Diff diff) { this.diff = diff } @@ -359,7 +374,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements) } - def assertTestsOrder(List expectedOrder) { + def assertTestsOrder(List expectedOrder) { TEST_WRITER.waitForTraces(expectedOrder.size() + 1) def traces = TEST_WRITER.toList() def events = getEventsAsJson(traces) @@ -381,9 +396,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { return testIdentifiers } - def test(String suite, String name, String parameters = null) { - - return new TestIdentifier(suite, name, parameters) + def test(String suite, String name) { + return new TestFQN(suite, name) } def getEventsAsJson(List> traces) { diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy index 3af7ef49258..271e648ad60 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.junit4.CucumberTracingListener @@ -64,10 +65,10 @@ class CucumberTest extends CiVisibilityInstrumentationTest { testcaseName | success | features | retriedTests "test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] "test-retry-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-retry-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions") ] } @@ -82,7 +83,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | knownTestsList "test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition") ] "test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [] "test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | [] @@ -100,7 +101,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined "test-quarantined-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } @@ -119,9 +120,9 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined | retried "test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } @@ -140,12 +141,12 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined | known "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [] } diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index 0e81d910ce4..de264157aa9 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -1,5 +1,5 @@ import datadog.trace.api.DisableTestTrace -import datadog.trace.api.civisibility.config.TestIdentifier +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff import datadog.trace.civisibility.diff.LineDiff @@ -43,8 +43,8 @@ class MUnitTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailedMUnit] | [] - "test-retry-failed" | false | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] - "test-failed-then-succeed" | true | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)] + "test-retry-failed" | false | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] + "test-failed-then-succeed" | true | [TestFailedThenSucceedMUnit] | [new TestFQN("org.example.TestFailedThenSucceedMUnit", "Calculator.add")] } def "test early flakiness detection #testcaseName"() { @@ -57,7 +57,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceedMUnit] | [new TestIdentifier("org.example.TestSucceedMUnit", "Calculator.add", null)] + "test-efd-known-test" | [TestSucceedMUnit] | [new TestFQN("org.example.TestSucceedMUnit", "Calculator.add")] "test-efd-new-test" | [TestSucceedMUnit] | [] "test-efd-new-slow-test" | [TestSucceedMUnitSlow] | [] // is executed only twice } @@ -89,7 +89,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + "test-quarantined-failed" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] } def "test quarantined auto-retries #testcaseName"() { @@ -106,7 +106,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + "test-quarantined-failed-atr" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -123,8 +123,8 @@ class MUnitTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] - "test-quarantined-failed-efd" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [] + "test-quarantined-failed-known" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] + "test-quarantined-failed-efd" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [] } private void runTests(Collection> tests, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index fe4229e373b..f0c2490b8a6 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff @@ -76,14 +77,14 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailed] | [] - "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] - "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] - "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] + "test-retry-failed" | false | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestFQN("org.example.TestFailedThenSucceed", "test_failed_then_succeed")] + "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestFQN("org.example.TestAssumption", "test_fail_assumption")] + "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestFQN("org.example.TestSkipped", "test_skipped")] "test-retry-parameterized" | false | [TestFailedParameterized] | [ - new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", /* backend cannot provide parameters for flaky parameterized tests yet */ null) + new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized") /* backend cannot provide parameters for flaky parameterized tests yet */ ] - "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] + "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestFQN("org.example.TestSucceedExpectedException", "test_succeed")] } def "test early flakiness detection #testcaseName"() { @@ -96,13 +97,13 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | knownTestsList - "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] + "test-efd-known-test" | true | [TestSucceed] | [new TestFQN("org.example.TestSucceed", "test_succeed")] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestFQN("org.example.TestParameterized", "parameterized_test_succeed")] "test-efd-new-test" | true | [TestSucceed] | [] "test-efd-new-parameterized-test" | true | [TestParameterized] | [] "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ - new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), - new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) + new TestFQN("org.example.TestFailedAndSucceed", "test_failed"), + new TestFQN("org.example.TestFailedAndSucceed", "test_succeed") ] "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once @@ -136,8 +137,8 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] + "test-quarantined-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized")] } def "test quarantined auto-retries #testcaseName"() { @@ -154,7 +155,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -171,8 +172,8 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } private void runTests(Collection> tests, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index 80dfe6be817..bb517db5ae5 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder @@ -71,13 +72,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest { testcaseName | success | features | retriedTests "test-no-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] "test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-failed-then-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature:Basic Arithmetic", "Addition") ] "test-retry-failed-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature:Basic Arithmetic With Examples", "Many additions.Single digits.${parameterizedTestNameSuffix()}", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature:Basic Arithmetic With Examples", "Many additions.Single digits.${parameterizedTestNameSuffix()}") ] } @@ -92,7 +93,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | knownTestsList "test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition") ] "test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [] "test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | [] @@ -110,7 +111,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined "test-quarantined-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } @@ -129,9 +130,9 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined | retried "test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } @@ -150,12 +151,12 @@ class CucumberTest extends CiVisibilityInstrumentationTest { where: testcaseName | features | quarantined | known "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ - new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] | [] } diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index 1196821b5c6..dbfa08b7554 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff @@ -85,10 +86,10 @@ class SpockTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailedSpock] | [] - "test-retry-failed" | false | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] - "test-failed-then-succeed" | true | [TestFailedThenSucceedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedSpock", "test failed then succeed", null)] - "test-retry-parameterized" | false | [TestFailedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedParameterizedSpock", "test add 4 and 4", null)] - "test-parameterized-failed-then-succeed" | true | [TestFailedThenSucceedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedParameterizedSpock", "test add 1 and 2", null)] + "test-retry-failed" | false | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] + "test-failed-then-succeed" | true | [TestFailedThenSucceedSpock] | [new TestFQN("org.example.TestFailedThenSucceedSpock", "test failed then succeed")] + "test-retry-parameterized" | false | [TestFailedParameterizedSpock] | [new TestFQN("org.example.TestFailedParameterizedSpock", "test add 4 and 4")] + "test-parameterized-failed-then-succeed" | true | [TestFailedThenSucceedParameterizedSpock] | [new TestFQN("org.example.TestFailedThenSucceedParameterizedSpock", "test add 1 and 2")] } def "test early flakiness detection #testcaseName"() { @@ -101,14 +102,14 @@ class SpockTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | knownTestsList - "test-efd-known-test" | true | [TestSucceedSpock] | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null)] + "test-efd-known-test" | true | [TestSucceedSpock] | [new TestFQN("org.example.TestSucceedSpock", "test success")] "test-efd-known-parameterized-test" | true | [TestParameterizedSpock] | [ - new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null), - new TestIdentifier("org.example.TestParameterizedSpock", "test add 4 and 4", null) + new TestFQN("org.example.TestParameterizedSpock", "test add 1 and 2"), + new TestFQN("org.example.TestParameterizedSpock", "test add 4 and 4") ] "test-efd-new-test" | true | [TestSucceedSpock] | [] "test-efd-new-parameterized-test" | true | [TestParameterizedSpock] | [] - "test-efd-known-tests-and-new-test" | true | [TestParameterizedSpock] | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null)] + "test-efd-known-tests-and-new-test" | true | [TestParameterizedSpock] | [new TestFQN("org.example.TestParameterizedSpock", "test add 1 and 2")] "test-efd-new-slow-test" | true | [TestSucceedSpockSlow] | [] // is executed only twice "test-efd-new-very-slow-test" | true | [TestSucceedSpockVerySlow] | [] // is executed only once "test-efd-faulty-session-threshold" | false | [TestSucceedAndFailedSpock] | [] @@ -141,8 +142,8 @@ class SpockTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] - "test-quarantined-failed-parameterized" | [TestFailedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedParameterizedSpock", "test add 4 and 4", null)] + "test-quarantined-failed" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] + "test-quarantined-failed-parameterized" | [TestFailedParameterizedSpock] | [new TestFQN("org.example.TestFailedParameterizedSpock", "test add 4 and 4")] } def "test quarantined auto-retries #testcaseName"() { @@ -159,7 +160,7 @@ class SpockTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] + "test-quarantined-failed-atr" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [new TestFQN("org.example.TestFailedSpock", "test failed")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -176,8 +177,8 @@ class SpockTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] - "test-quarantined-failed-efd" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [] + "test-quarantined-failed-known" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [new TestFQN("org.example.TestFailedSpock", "test failed")] + "test-quarantined-failed-efd" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [] } private static void runTests(List> classes, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index 049253836b4..08d6b877a3f 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff @@ -114,14 +115,14 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailed] | [] - "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] - "test-retry-template" | false | [TestFailedTemplate] | [new TestIdentifier("org.example.TestFailedTemplate", "test_template", null)] - "test-retry-factory" | false | [TestFailedFactory] | [new TestIdentifier("org.example.TestFailedFactory", "test_factory", null)] - "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] - "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] - "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] - "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] + "test-retry-failed" | false | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestFQN("org.example.TestFailedThenSucceed", "test_failed_then_succeed")] + "test-retry-template" | false | [TestFailedTemplate] | [new TestFQN("org.example.TestFailedTemplate", "test_template")] + "test-retry-factory" | false | [TestFailedFactory] | [new TestFQN("org.example.TestFailedFactory", "test_factory")] + "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestFQN("org.example.TestAssumption", "test_fail_assumption")] + "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestFQN("org.example.TestSkipped", "test_skipped")] + "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized")] + "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestFQN("org.example.TestSucceedExpectedException", "test_succeed")] } def "test early flakiness detection #testcaseName"() { @@ -134,13 +135,13 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | knownTestsList - "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "test_parameterized", null)] + "test-efd-known-test" | true | [TestSucceed] | [new TestFQN("org.example.TestSucceed", "test_succeed")] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestFQN("org.example.TestParameterized", "test_parameterized")] "test-efd-new-test" | true | [TestSucceed] | [] "test-efd-new-parameterized-test" | true | [TestParameterized] | [] "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ - new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), - new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) + new TestFQN("org.example.TestFailedAndSucceed", "test_failed"), + new TestFQN("org.example.TestFailedAndSucceed", "test_succeed") ] "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once @@ -174,8 +175,8 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] + "test-quarantined-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized")] } def "test quarantined auto-retries #testcaseName"() { @@ -192,7 +193,7 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -209,8 +210,8 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } protected void runTests(List> tests, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy index 367562a8992..df54a3f96df 100644 --- a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy @@ -1,5 +1,6 @@ import com.intuit.karate.FileUtils import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.karate.TestEventsHandlerHolder @@ -68,10 +69,10 @@ class KarateTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailedKarate] | [] - "test-retry-failed" | false | [TestFailedKarate] | [new TestIdentifier("[org/example/test_failed] test failed", "second scenario", null)] - "test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestIdentifier("[org/example/test_failed_then_succeed] test failed", "flaky scenario", null)] + "test-retry-failed" | false | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] + "test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestFQN("[org/example/test_failed_then_succeed] test failed", "flaky scenario")] "test-retry-parameterized" | false | [TestFailedParameterizedKarate] | [ - new TestIdentifier("[org/example/test_failed_parameterized] test parameterized", "first scenario as an outline", null) + new TestFQN("[org/example/test_failed_parameterized] test parameterized", "first scenario as an outline") ] } @@ -85,9 +86,9 @@ class KarateTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestIdentifier("[org/example/test_succeed_one_case] test succeed", "first scenario", null)] + "test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestFQN("[org/example/test_succeed_one_case] test succeed", "first scenario")] "test-efd-known-parameterized-test" | [TestParameterizedKarate] | [ - new TestIdentifier("[org/example/test_parameterized] test parameterized", "first scenario as an outline", null) + new TestFQN("[org/example/test_parameterized] test parameterized", "first scenario as an outline") ] "test-efd-new-test" | [TestSucceedOneCaseKarate] | [] "test-efd-new-parameterized-test" | [TestParameterizedKarate] | [] @@ -105,7 +106,7 @@ class KarateTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailedKarate] | [new TestIdentifier("[org/example/test_failed] test failed", "second scenario", null)] + "test-quarantined-failed" | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] } private void runTests(List> tests, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index fdc41360152..62f67ae5efa 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -1,3 +1,4 @@ +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff @@ -61,11 +62,11 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed" | false | [TestFailed] | [] - "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + "test-retry-failed" | false | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] "test-retry-parameterized" | false | [TestFailedParameterized] | [ - new TestIdentifier("org.example.TestFailedParameterized", "addition should correctly add two numbers", null) + new TestFQN("org.example.TestFailedParameterized", "addition should correctly add two numbers") ] - "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "Example.add adds two numbers", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestFQN("org.example.TestFailedThenSucceed", "Example.add adds two numbers")] } def "test early flakiness detection #testcaseName"() { @@ -78,7 +79,7 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "Example.add adds two numbers", null)] + "test-efd-known-test" | [TestSucceed] | [new TestFQN("org.example.TestSucceed", "Example.add adds two numbers")] "test-efd-new-test" | [TestSucceed] | [] "test-efd-new-slow-test" | [TestSucceedSlow] | [] // is executed only twice "test-efd-faulty-session-threshold" | [TestSucceedMoreCases] | [] @@ -111,7 +112,7 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + "test-quarantined-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] } def "test quarantined auto-retries #testcaseName"() { @@ -128,7 +129,7 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -145,8 +146,8 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] - "test-quarantined-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [] + "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] + "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [] } @Override diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index d5d6fec2f24..dd8e1541027 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.testng +import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff @@ -105,11 +106,11 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | retriedTests "test-failed-${version()}" | false | [TestFailed] | [] - "test-skipped" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] - "test-retry-failed-${version()}" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-retry-error" | false | [TestError] | [new TestIdentifier("org.example.TestError", "test_error", null)] - "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "parameterized_test_succeed", null)] - "test-failed-then-succeed-${version()}" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed", null)] + "test-skipped" | true | [TestSkipped] | [new TestFQN("org.example.TestSkipped", "test_skipped")] + "test-retry-failed-${version()}" | false | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-retry-error" | false | [TestError] | [new TestFQN("org.example.TestError", "test_error")] + "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "parameterized_test_succeed")] + "test-failed-then-succeed-${version()}" | true | [TestFailedThenSucceed] | [new TestFQN("org.example.TestFailedThenSucceed", "test_failed")] } def "test early flakiness detection #testcaseName"() { @@ -124,13 +125,13 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { where: testcaseName | success | tests | knownTestsList - "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] + "test-efd-known-test" | true | [TestSucceed] | [new TestFQN("org.example.TestSucceed", "test_succeed")] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestFQN("org.example.TestParameterized", "parameterized_test_succeed")] "test-efd-new-test" | true | [TestSucceed] | [] "test-efd-new-parameterized-test" | true | [TestParameterized] | [] "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ - new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), - new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) + new TestFQN("org.example.TestFailedAndSucceed", "test_failed"), + new TestFQN("org.example.TestFailedAndSucceed", "test_succeed") ] "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once @@ -166,8 +167,8 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined - "test-quarantined-failed-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "parameterized_test_succeed", null)] + "test-quarantined-failed-${version()}" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "parameterized_test_succeed")] } def "test quarantined auto-retries #testcaseName"() { @@ -186,7 +187,7 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | retried - "test-quarantined-failed-atr-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-quarantined-failed-atr-${version()}" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { @@ -205,8 +206,8 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { where: testcaseName | tests | quarantined | known - "test-quarantined-failed-known" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-quarantined-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } private static boolean isEFDSupported() { diff --git a/internal-api/build.gradle b/internal-api/build.gradle index 1867ac148ba..ac0941b5ad2 100644 --- a/internal-api/build.gradle +++ b/internal-api/build.gradle @@ -99,6 +99,7 @@ excludedClassesCoverage += [ "datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings", "datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings.ExecutionsByDuration", "datadog.trace.api.civisibility.config.TestIdentifier", + "datadog.trace.api.civisibility.config.TestFQN", "datadog.trace.api.civisibility.config.TestMetadata", "datadog.trace.api.civisibility.config.TestSourceData", "datadog.trace.api.civisibility.coverage.CoveragePerTestBridge", diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestFQN.java b/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestFQN.java new file mode 100644 index 00000000000..9534f4e6db1 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestFQN.java @@ -0,0 +1,47 @@ +package datadog.trace.api.civisibility.config; + +import java.util.Objects; + +/** + * Fully Qualified Name: uniquely identifies a test case within a module by name. Multiple + * executions of the same test case (for example when retries are done) will have the same FQN. + */ +public class TestFQN { + private final String suite; + private final String name; + + public TestFQN(String suite, String name) { + this.suite = suite; + this.name = name; + } + + public String getSuite() { + return suite; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TestFQN that = (TestFQN) o; + return Objects.equals(suite, that.suite) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(suite, name); + } + + @Override + public String toString() { + return "TestFQN{" + "suite='" + suite + '\'' + ", name='" + name + '\'' + '}'; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestIdentifier.java b/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestIdentifier.java index f954a2646ce..012cbeb886b 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestIdentifier.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/config/TestIdentifier.java @@ -3,14 +3,10 @@ import java.util.Objects; import javax.annotation.Nullable; -/** - * Uniquely identifies a test case within a module. Multiple executions of the same test case (for - * example when retries are done) will have the same test identifier. - */ +/** Uniquely identifies a test case with FQN and parameters. */ public class TestIdentifier { + private final TestFQN fqn; - private final String suite; - private final String name; /** * Some API endpoints do not return parameters data. If this field is {@code null} then either * corresponding test case is not parameterized, or this identifier refers to all @@ -19,17 +15,21 @@ public class TestIdentifier { private @Nullable final String parameters; public TestIdentifier(String suite, String name, @Nullable String parameters) { - this.suite = suite; - this.name = name; + this.fqn = new TestFQN(suite, name); + this.parameters = parameters; + } + + public TestIdentifier(TestFQN testFQN, @Nullable String parameters) { + this.fqn = testFQN; this.parameters = parameters; } public String getSuite() { - return suite; + return fqn.getSuite(); } public String getName() { - return name; + return fqn.getName(); } @Nullable @@ -37,8 +37,8 @@ public String getParameters() { return parameters; } - public TestIdentifier withoutParameters() { - return parameters == null ? this : new TestIdentifier(suite, name, null); + public TestFQN toFQN() { + return fqn; } @Override @@ -50,28 +50,16 @@ public boolean equals(Object o) { return false; } TestIdentifier that = (TestIdentifier) o; - return Objects.equals(suite, that.suite) - && Objects.equals(name, that.name) - && Objects.equals(parameters, that.parameters); + return Objects.equals(fqn, that.fqn) && Objects.equals(parameters, that.parameters); } @Override public int hashCode() { - return Objects.hash(suite, name, parameters); + return Objects.hash(fqn, parameters); } @Override public String toString() { - return "TestIdentifier{" - + "suite='" - + suite - + '\'' - + ", name='" - + name - + '\'' - + ", parameters='" - + parameters - + '\'' - + '}'; + return "TestIdentifier{" + "fqn=" + fqn + ", parameters='" + parameters + '\'' + '}'; } } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java index f6b253d3177..6442aeed041 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java @@ -151,7 +151,12 @@ public enum CiVisibilityCountMetric { IMPACTED_TESTS_DETECTION_REQUEST("impacted_tests_detection.request", RequestCompressed.class), /** The number of tests requests sent to the changed files endpoint that errored */ IMPACTED_TESTS_DETECTION_REQUEST_ERRORS( - "impacted_tests_detection.request_errors", ErrorType.class, StatusCode.class); + "impacted_tests_detection.request_errors", ErrorType.class, StatusCode.class), + /** The number of requests sent to the test management tests endpoint */ + TEST_MANAGEMENT_TESTS_REQUEST("test_management.request", RequestCompressed.class), + /** The number of tests requests sent to the test management tests endpoint that errored */ + TEST_MANAGEMENT_TESTS_REQUEST_ERRORS( + "test_management.request_errors", ErrorType.class, StatusCode.class); // need a "holder" class, as accessing static fields from enum constructors is illegal static class IndexHolder { diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java index a7e7de917dc..a50f2a2d61c 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java @@ -57,7 +57,13 @@ public enum CiVisibilityDistributionMetric { IMPACTED_TESTS_DETECTION_RESPONSE_BYTES( "impacted_tests_detection.response_bytes", ResponseCompressed.class), /** The number of files received by the changed files endpoint */ - IMPACTED_TESTS_DETECTION_RESPONSE_FILES("impacted_tests_detection.response_files"); + IMPACTED_TESTS_DETECTION_RESPONSE_FILES("impacted_tests_detection.response_files"), + /** The time it takes to get the response of the test management tests endpoint request in ms */ + TEST_MANAGEMENT_TESTS_REQUEST_MS("test_management.request_ms"), + /** The number of bytes received by the test management tests endpoint */ + TEST_MANAGEMENT_TESTS_RESPONSE_BYTES("test_management.response_bytes", ResponseCompressed.class), + /** The number of tests received by the test management tests endpoint */ + TEST_MANAGEMENT_TESTS_RESPONSE_TESTS("test_management.response_tests"); private static final String NAMESPACE = "civisibility";