Skip to content

Commit bb0bc17

Browse files
brianseedersbrianseeders
authored
Muted test automation (#106784)
1 parent bc12623 commit bb0bc17

File tree

5 files changed

+214
-0
lines changed

5 files changed

+214
-0
lines changed

‎build-tools-internal/muted-tests.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
tests:
2+
# Examples:
3+
#
4+
# Mute a single test case in a YAML test suite:
5+
# - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
6+
# method: test {yaml=analysis-common/30_tokenizers/letter}
7+
# issue: https://github.com/elastic/elasticsearch/...
8+
#
9+
# Mute several methods of a Java test:
10+
# - class: org.elasticsearch.common.CharArraysTests
11+
# methods:
12+
# - testCharsBeginsWith
13+
# - testCharsToBytes
14+
# - testConstantTimeEquals
15+
# issue: https://github.com/elastic/elasticsearch/...
16+
#
17+
# Mute an entire test class:
18+
# - class: org.elasticsearch.common.unit.TimeValueTests
19+
# issue: https://github.com/elastic/elasticsearch/...
20+
#
21+
# Mute a single method in a test class:
22+
# - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIPTests
23+
# method: testCrankyEvaluateBlockWithoutNulls
24+
# issue: https://github.com/elastic/elasticsearch/...

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
1313
import org.elasticsearch.gradle.internal.info.BuildParams;
1414
import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
15+
import org.elasticsearch.gradle.internal.test.MutedTestPlugin;
1516
import org.elasticsearch.gradle.internal.test.TestUtil;
1617
import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
1718
import org.elasticsearch.gradle.util.GradleUtils;
@@ -62,6 +63,7 @@ public void apply(Project project) {
6263
project.getPluginManager().apply(RepositoriesSetupPlugin.class);
6364
project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
6465
project.getPluginManager().apply(PrecommitTaskPlugin.class);
66+
project.getPluginManager().apply(MutedTestPlugin.class);
6567

6668
configureConfigurations(project);
6769
configureCompile(project);

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.gradle.internal.test;
10+
11+
import org.elasticsearch.gradle.internal.conventions.util.Util;
12+
import org.elasticsearch.gradle.internal.info.BuildParams;
13+
import org.gradle.api.Plugin;
14+
import org.gradle.api.Project;
15+
import org.gradle.api.provider.Provider;
16+
import org.gradle.api.tasks.testing.Test;
17+
18+
import java.io.File;
19+
20+
public class MutedTestPlugin implements Plugin<Project> {
21+
@Override
22+
public void apply(Project project) {
23+
File infoPath = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "build-tools-internal");
24+
Provider<MutedTestsBuildService> mutedTestsProvider = project.getGradle()
25+
.getSharedServices()
26+
.registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> {
27+
spec.getParameters().getInfoPath().set(infoPath);
28+
});
29+
30+
project.getTasks().withType(Test.class).configureEach(test -> {
31+
test.filter(filter -> {
32+
for (String exclude : mutedTestsProvider.get().getExcludePatterns()) {
33+
filter.excludeTestsMatching(exclude);
34+
}
35+
36+
// Don't fail when all tests are ignored when running in CI
37+
filter.setFailOnNoMatchingTests(BuildParams.isCi() == false);
38+
});
39+
});
40+
}
41+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.gradle.internal.test;
10+
11+
import com.fasterxml.jackson.annotation.JsonCreator;
12+
import com.fasterxml.jackson.annotation.JsonProperty;
13+
import com.fasterxml.jackson.databind.ObjectMapper;
14+
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
15+
16+
import org.gradle.api.file.RegularFileProperty;
17+
import org.gradle.api.services.BuildService;
18+
import org.gradle.api.services.BuildServiceParameters;
19+
20+
import java.io.BufferedInputStream;
21+
import java.io.File;
22+
import java.io.FileInputStream;
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.io.UncheckedIOException;
26+
import java.util.ArrayList;
27+
import java.util.Collections;
28+
import java.util.List;
29+
30+
public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
31+
private final List<String> excludePatterns;
32+
33+
public MutedTestsBuildService() {
34+
File infoPath = getParameters().getInfoPath().get().getAsFile();
35+
File mutedTestsFile = new File(infoPath, "muted-tests.yml");
36+
try (InputStream is = new BufferedInputStream(new FileInputStream(mutedTestsFile))) {
37+
ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
38+
List<MutedTest> mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
39+
excludePatterns = buildExcludePatterns(mutedTests == null ? Collections.emptyList() : mutedTests);
40+
} catch (IOException e) {
41+
throw new UncheckedIOException(e);
42+
}
43+
}
44+
45+
public List<String> getExcludePatterns() {
46+
return excludePatterns;
47+
}
48+
49+
private static List<String> buildExcludePatterns(List<MutedTest> mutedTests) {
50+
List<String> excludes = new ArrayList<>();
51+
if (mutedTests.isEmpty() == false) {
52+
for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
53+
if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {
54+
for (String method : mutedTest.getMethods()) {
55+
// Tests that use the randomized runner and parameters end up looking like this:
56+
// test {yaml=analysis-common/30_tokenizers/letter}
57+
// We need to detect this and handle them a little bit different than non-parameterized tests, because of some
58+
// quirks in the randomized runner
59+
int index = method.indexOf(" {");
60+
String methodWithoutParams = index >= 0 ? method.substring(0, index) : method;
61+
String paramString = index >= 0 ? method.substring(index) : null;
62+
63+
excludes.add(mutedTest.getClassName() + "." + method);
64+
65+
if (paramString != null) {
66+
// Because of randomized runner quirks, we need skip the test method by itself whenever we want to skip a test
67+
// that has parameters
68+
// This is because the runner has *two* separate checks that can cause the test to end up getting executed, so
69+
// we need filters that cover both checks
70+
excludes.add(mutedTest.getClassName() + "." + methodWithoutParams);
71+
} else {
72+
// We need to add the following, in case we're skipping an entire class of parameterized tests
73+
excludes.add(mutedTest.getClassName() + "." + method + " *");
74+
}
75+
}
76+
} else if (mutedTest.getClassName() != null) {
77+
excludes.add(mutedTest.getClassName() + ".*");
78+
}
79+
}
80+
}
81+
82+
return excludes;
83+
}
84+
85+
public interface Params extends BuildServiceParameters {
86+
RegularFileProperty getInfoPath();
87+
}
88+
89+
public static class MutedTest {
90+
private final String className;
91+
private final String method;
92+
private final List<String> methods;
93+
private final String issue;
94+
95+
@JsonCreator
96+
public MutedTest(
97+
@JsonProperty("class") String className,
98+
@JsonProperty("method") String method,
99+
@JsonProperty("methods") List<String> methods,
100+
@JsonProperty("issue") String issue
101+
) {
102+
this.className = className;
103+
this.method = method;
104+
this.methods = methods;
105+
this.issue = issue;
106+
}
107+
108+
public List<String> getMethods() {
109+
List<String> allMethods = new ArrayList<>();
110+
if (methods != null) {
111+
allMethods.addAll(methods);
112+
}
113+
if (method != null) {
114+
allMethods.add(method);
115+
}
116+
117+
return allMethods;
118+
}
119+
120+
public String getClassName() {
121+
return className;
122+
}
123+
124+
public String getIssue() {
125+
return issue;
126+
}
127+
}
128+
129+
private static class MutedTests {
130+
private final List<MutedTest> tests;
131+
132+
@JsonCreator
133+
MutedTests(@JsonProperty("tests") List<MutedTest> tests) {
134+
this.tests = tests;
135+
}
136+
137+
public List<MutedTest> getTests() {
138+
return tests;
139+
}
140+
}
141+
}

‎build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

+6
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import spock.lang.Specification
2121
import spock.lang.TempDir
2222

2323
import java.lang.management.ManagementFactory
24+
import java.nio.file.Files
25+
import java.nio.file.Path
2426
import java.util.jar.JarEntry
2527
import java.util.jar.JarOutputStream
2628

@@ -57,6 +59,10 @@ abstract class AbstractGradleFuncTest extends Specification {
5759
id 'base'
5860
}
5961
"""
62+
def mutedTestsFile = Files.createFile(Path.of(testProjectDir.newFolder("build-tools-internal").path, "muted-tests.yml"))
63+
mutedTestsFile << """
64+
tests: []
65+
"""
6066
}
6167

6268
def cleanup() {

0 commit comments

Comments
 (0)