Skip to content

Commit 62c0629

Browse files
lktslkts
authored
Add new-style block loader tests for constant_keyword, version, wildcard (#126968)
1 parent af6dac5 commit 62c0629

File tree

19 files changed

+424
-31
lines changed

19 files changed

+424
-31
lines changed

‎test/framework/build.gradle

+4-1
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,12 @@ dependencies {
3232

3333
api "org.elasticsearch:mocksocket:${versions.mocksocket}"
3434

35+
testImplementation project(":modules:mapper-extras")
36+
testImplementation project(':x-pack:plugin:core')
3537
testImplementation project(':x-pack:plugin:mapper-unsigned-long')
3638
testImplementation project(':x-pack:plugin:mapper-counted-keyword')
37-
testImplementation project(":modules:mapper-extras")
39+
testImplementation project(':x-pack:plugin:mapper-constant-keyword')
40+
testImplementation project(':x-pack:plugin:wildcard')
3841
}
3942

4043
sourceSets {

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
1313
import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator;
1414
import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
15+
import org.elasticsearch.logsdb.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator;
1516
import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator;
1617
import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator;
1718
import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
@@ -26,6 +27,7 @@
2627
import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
2728
import org.elasticsearch.logsdb.datageneration.fields.leaf.TextFieldDataGenerator;
2829
import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
30+
import org.elasticsearch.logsdb.datageneration.fields.leaf.WildcardFieldDataGenerator;
2931

3032
/**
3133
* Lists all leaf field types that are supported for data generation by default.
@@ -46,7 +48,9 @@ public enum FieldType {
4648
DATE("date"),
4749
GEO_POINT("geo_point"),
4850
TEXT("text"),
49-
IP("ip");
51+
IP("ip"),
52+
CONSTANT_KEYWORD("constant_keyword"),
53+
WILDCARD("wildcard");
5054

5155
private final String name;
5256

@@ -56,7 +60,7 @@ public enum FieldType {
5660

5761
public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
5862
return switch (this) {
59-
case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
63+
case KEYWORD -> new KeywordFieldDataGenerator(dataSource);
6064
case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
6165
case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
6266
case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
@@ -72,6 +76,8 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
7276
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
7377
case TEXT -> new TextFieldDataGenerator(dataSource);
7478
case IP -> new IpFieldDataGenerator(dataSource);
79+
case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator();
80+
case WILDCARD -> new WildcardFieldDataGenerator(dataSource);
7581
};
7682
}
7783

@@ -93,6 +99,8 @@ public static FieldType tryParse(String name) {
9399
case "geo_point" -> FieldType.GEO_POINT;
94100
case "text" -> FieldType.TEXT;
95101
case "ip" -> FieldType.IP;
102+
case "constant_keyword" -> FieldType.CONSTANT_KEYWORD;
103+
case "wildcard" -> FieldType.WILDCARD;
96104
default -> null;
97105
};
98106
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java

+4
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator requ
7878
return null;
7979
}
8080

81+
default DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
82+
return null;
83+
}
84+
8185
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
8286
return null;
8387
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java

+6
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,12 @@ public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) {
126126
}
127127
}
128128

129+
record VersionStringGenerator() implements DataSourceRequest<DataSourceResponse.VersionStringGenerator> {
130+
public DataSourceResponse.VersionStringGenerator accept(DataSourceHandler handler) {
131+
return handler.handle(this);
132+
}
133+
}
134+
129135
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
130136
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
131137
return handler.handle(this);

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java

+2
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ record GeoPointGenerator(Supplier<Object> generator) implements DataSourceRespon
5353

5454
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
5555

56+
record VersionStringGenerator(Supplier<String> generator) implements DataSourceResponse {}
57+
5658
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
5759

5860
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java

+25
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
5151
case GEO_POINT -> geoPointMapping(map);
5252
case TEXT -> textMapping(request, new HashMap<>());
5353
case IP -> ipMapping(map);
54+
case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>());
55+
case WILDCARD -> wildcardMapping(new HashMap<>());
5456
});
5557
}
5658

@@ -225,6 +227,29 @@ private Supplier<Map<String, Object>> ipMapping(Map<String, Object> injected) {
225227
};
226228
}
227229

230+
private Supplier<Map<String, Object>> constantKeywordMapping(Map<String, Object> injected) {
231+
return () -> {
232+
// value is optional and can be set from the first document
233+
// we don't cover this case here
234+
injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
235+
236+
return injected;
237+
};
238+
}
239+
240+
private Supplier<Map<String, Object>> wildcardMapping(Map<String, Object> injected) {
241+
return () -> {
242+
if (ESTestCase.randomDouble() <= 0.2) {
243+
injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
244+
}
245+
if (ESTestCase.randomDouble() <= 0.2) {
246+
injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
247+
}
248+
249+
return injected;
250+
};
251+
}
252+
228253
private static HashMap<String, Object> commonMappingParameters() {
229254
var map = new HashMap<String, Object>();
230255
map.put("store", ESTestCase.randomBoolean());

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ConstantKeywordFieldDataGenerator.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.logsdb.datageneration.fields.leaf;
11+
12+
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
13+
14+
import java.util.Map;
15+
16+
public class ConstantKeywordFieldDataGenerator implements FieldDataGenerator {
17+
@Override
18+
public Object generateValue(Map<String, Object> fieldMapping) {
19+
if (fieldMapping == null) {
20+
// Dynamically mapped, skip it because it will be mapped as text, and we cover this case already
21+
return null;
22+
}
23+
24+
var value = fieldMapping.get("value");
25+
assert value != null;
26+
27+
return value;
28+
}
29+
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java

-3
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,11 @@
1313
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
1414
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
1515

16-
import java.util.HashSet;
1716
import java.util.Map;
18-
import java.util.Set;
1917
import java.util.function.Supplier;
2018

2119
public class CountedKeywordFieldDataGenerator implements FieldDataGenerator {
2220
private final Supplier<Object> valueGenerator;
23-
private final Set<String> previousStrings = new HashSet<>();
2421

2522
public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
2623
var strings = dataSource.get(new DataSourceRequest.StringGenerator());

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
public class KeywordFieldDataGenerator implements FieldDataGenerator {
2020
private final Supplier<Object> valueGenerator;
2121

22-
public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
22+
public KeywordFieldDataGenerator(DataSource dataSource) {
2323
var strings = dataSource.get(new DataSourceRequest.StringGenerator());
2424
var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
2525
var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/WildcardFieldDataGenerator.java

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.logsdb.datageneration.fields.leaf;
11+
12+
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
13+
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
14+
15+
import java.util.Map;
16+
17+
public class WildcardFieldDataGenerator implements FieldDataGenerator {
18+
private final FieldDataGenerator keywordGenerator;
19+
20+
public WildcardFieldDataGenerator(DataSource dataSource) {
21+
this.keywordGenerator = new KeywordFieldDataGenerator(dataSource);
22+
}
23+
24+
@Override
25+
public Object generateValue(Map<String, Object> fieldMapping) {
26+
return keywordGenerator.generateValue(fieldMapping);
27+
}
28+
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java

+43-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ static Map<String, FieldSpecificMatcher> matchers(
4646
return new HashMap<>() {
4747
{
4848
put("keyword", new KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
49-
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
5049
put("long", new NumberMatcher("long", actualMappings, actualSettings, expectedMappings, expectedSettings));
5150
put("unsigned_long", new UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
5251
put("integer", new NumberMatcher("integer", actualMappings, actualSettings, expectedMappings, expectedSettings));
@@ -58,11 +57,14 @@ static Map<String, FieldSpecificMatcher> matchers(
5857
put("scaled_float", new ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
5958
put("counted_keyword", new CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
6059
put("boolean", new BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
60+
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
6161
put("geo_shape", new ExactMatcher("geo_shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
6262
put("shape", new ExactMatcher("shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
6363
put("geo_point", new GeoPointMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
6464
put("text", new TextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
6565
put("ip", new IpMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
66+
put("constant_keyword", new ConstantKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
67+
put("wildcard", new WildcardMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
6668
}
6769
};
6870
}
@@ -691,6 +693,46 @@ Object convert(Object value, Object nullValue) {
691693
}
692694
}
693695

696+
class ConstantKeywordMatcher extends GenericMappingAwareMatcher {
697+
ConstantKeywordMatcher(
698+
XContentBuilder actualMappings,
699+
Settings.Builder actualSettings,
700+
XContentBuilder expectedMappings,
701+
Settings.Builder expectedSettings
702+
) {
703+
super("constant_keyword", actualMappings, actualSettings, expectedMappings, expectedSettings);
704+
}
705+
706+
@Override
707+
Object convert(Object value, Object nullValue) {
708+
// We just need to get rid of literal `null`s which is done in the caller.
709+
return value;
710+
}
711+
}
712+
713+
class WildcardMatcher extends GenericMappingAwareMatcher {
714+
WildcardMatcher(
715+
XContentBuilder actualMappings,
716+
Settings.Builder actualSettings,
717+
XContentBuilder expectedMappings,
718+
Settings.Builder expectedSettings
719+
) {
720+
super("wildcard", actualMappings, actualSettings, expectedMappings, expectedSettings);
721+
}
722+
723+
@Override
724+
Object convert(Object value, Object nullValue) {
725+
if (value == null) {
726+
if (nullValue != null) {
727+
return nullValue;
728+
}
729+
return null;
730+
}
731+
732+
return value;
733+
}
734+
}
735+
694736
/**
695737
* Generic matcher that supports common matching logic like null values.
696738
*/

‎test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
import org.elasticsearch.test.ESTestCase;
2121
import org.elasticsearch.xcontent.XContentBuilder;
2222
import org.elasticsearch.xcontent.XContentType;
23+
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
2324
import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin;
2425
import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin;
26+
import org.elasticsearch.xpack.wildcard.Wildcard;
2527

2628
import java.io.IOException;
2729
import java.util.Collection;
@@ -111,7 +113,13 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG
111113
var mappingService = new MapperServiceTestCase() {
112114
@Override
113115
protected Collection<? extends Plugin> getPlugins() {
114-
return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin(), new CountedKeywordMapperPlugin());
116+
return List.of(
117+
new UnsignedLongMapperPlugin(),
118+
new MapperExtrasPlugin(),
119+
new CountedKeywordMapperPlugin(),
120+
new ConstantKeywordMapperPlugin(),
121+
new Wildcard()
122+
);
115123
}
116124
}.createMapperService(mappingXContent);
117125

‎x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.constantkeyword.mapper;
9+
10+
import org.apache.lucene.util.BytesRef;
11+
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
12+
import org.elasticsearch.logsdb.datageneration.FieldType;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
15+
16+
import java.util.Collection;
17+
import java.util.List;
18+
import java.util.Map;
19+
20+
public class ConstantKeywordFieldBlockLoaderTests extends BlockLoaderTestCase {
21+
public ConstantKeywordFieldBlockLoaderTests(Params params) {
22+
super(FieldType.CONSTANT_KEYWORD.toString(), params);
23+
}
24+
25+
@Override
26+
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
27+
return new BytesRef((String) fieldMapping.get("value"));
28+
}
29+
30+
@Override
31+
protected Collection<Plugin> getPlugins() {
32+
return List.of(new ConstantKeywordMapperPlugin());
33+
}
34+
}

0 commit comments

Comments
 (0)