Skip to content

Commit a6bcbb6

Browse files
elasticmachineelasticmachine
authored
Merge branch 'multi-project' into mp/merge/main/2025-01-20T10
2 parents 7a1f5c5 + b1a3e9c commit a6bcbb6

File tree

22 files changed

+137
-79
lines changed

22 files changed

+137
-79
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

+42
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.common.xcontent.XContentParserUtils;
3939
import org.elasticsearch.core.Assertions;
4040
import org.elasticsearch.core.FixForMultiProject;
41+
import org.elasticsearch.core.Nullable;
4142
import org.elasticsearch.core.Tuple;
4243
import org.elasticsearch.gateway.MetadataStateFormat;
4344
import org.elasticsearch.index.Index;
@@ -460,6 +461,12 @@ public ProjectMetadata getProject() {
460461
return getSingleProject();
461462
}
462463

464+
@FixForMultiProject(description = "temporarily allow non-multi-project aware code to work with just the default project")
465+
@Deprecated(forRemoval = true)
466+
public ProjectMetadata getDefaultProject() {
467+
return getProject(DEFAULT_PROJECT_ID);
468+
}
469+
463470
public boolean hasProject(ProjectId projectId) {
464471
return projectMetadata.containsKey(projectId);
465472
}
@@ -484,6 +491,41 @@ public int getTotalOpenIndexShards() {
484491
return shards;
485492
}
486493

494+
/**
495+
* Utility method that allows retrieving a {@link ProjectCustom} from a project.
496+
* Throws an exception when multiple projects have that {@link ProjectCustom}.
497+
* @return the {@link ProjectCustom} if and only if it's present in a single project. If it's not present in any project, returns null
498+
*/
499+
@FixForMultiProject
500+
@Nullable
501+
public <T extends ProjectCustom> T getSingleProjectCustom(String type) {
502+
var project = getSingleProjectWithCustom(type);
503+
return project == null ? null : project.custom(type);
504+
}
505+
506+
/**
507+
* Utility method that allows retrieving a project that has a certain {@link ProjectCustom}.
508+
* Throws an exception when multiple projects have that {@link ProjectCustom}.
509+
* @return the project that has the {@link ProjectCustom} if and only if it's present in a single project.
510+
* If it's not present in any project, returns null
511+
*/
512+
@FixForMultiProject
513+
@Nullable
514+
public ProjectMetadata getSingleProjectWithCustom(String type) {
515+
ProjectMetadata resultingProject = null;
516+
for (ProjectMetadata project : projects().values()) {
517+
ProjectCustom projectCustom = project.custom(type);
518+
if (projectCustom == null) {
519+
continue;
520+
}
521+
if (resultingProject != null) {
522+
throw new UnsupportedOperationException("Multiple custom projects found for type [" + type + "]");
523+
}
524+
resultingProject = project;
525+
}
526+
return resultingProject;
527+
}
528+
487529
/**
488530
* @return The total number of shards across all projects in this cluster
489531
*/

‎server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ static Automaton buildAutomaton(String pattern) {
108108
*/
109109
@Override
110110
public List<String> getMatchingIndices(Metadata metadata) {
111-
return metadata.getProject().indices().keySet().stream().filter(this::matchesIndexPattern).toList();
111+
return metadata.getDefaultProject().indices().keySet().stream().filter(this::matchesIndexPattern).toList();
112112
}
113113

114114
/**

‎server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,12 @@ public PersistentTasksCustomMetadata(long lastAllocationId, Map<String, Persiste
132132
);
133133
}
134134

135+
@Deprecated(forRemoval = true)
135136
public static PersistentTasksCustomMetadata getPersistentTasksCustomMetadata(ClusterState clusterState) {
136-
return clusterState.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
137+
return get(clusterState.metadata().getProject());
137138
}
138139

139-
public static PersistentTasksCustomMetadata getPersistentTasksCustomMetadata(ProjectMetadata projectMetadata) {
140+
public static PersistentTasksCustomMetadata get(ProjectMetadata projectMetadata) {
140141
return projectMetadata.custom(PersistentTasksCustomMetadata.TYPE);
141142
}
142143

@@ -259,7 +260,7 @@ public static ClusterState disassociateDeadNodes(ClusterState clusterState) {
259260
}
260261

261262
private static ProjectMetadata disassociateDeadNodesForProject(ProjectState projectState) {
262-
PersistentTasksCustomMetadata tasks = getPersistentTasksCustomMetadata(projectState.metadata());
263+
PersistentTasksCustomMetadata tasks = get(projectState.metadata());
263264
if (tasks == null) {
264265
return projectState.metadata();
265266
}

‎server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java

+1-5
Original file line numberDiff line numberDiff line change
@@ -235,11 +235,7 @@ public void onClusterServiceClose() {
235235
public void onTimeout(TimeValue timeout) {
236236
listener.onFailure(new IllegalStateException("Timed out when waiting for persistent tasks after " + timeout));
237237
}
238-
},
239-
clusterState -> predicate.test(clusterState.metadata().getProject().custom(PersistentTasksCustomMetadata.TYPE)),
240-
timeout,
241-
logger
242-
);
238+
}, clusterState -> predicate.test(PersistentTasksCustomMetadata.get(clusterState.metadata().getDefaultProject())), timeout, logger);
243239
}
244240

245241
public interface WaitForPersistentTaskListener<P extends PersistentTaskParams> extends ActionListener<PersistentTask<P>> {

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

+41
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.elasticsearch.index.alias.RandomAliasActionsGenerator;
4848
import org.elasticsearch.index.mapper.MapperService;
4949
import org.elasticsearch.indices.IndicesModule;
50+
import org.elasticsearch.ingest.IngestMetadata;
5051
import org.elasticsearch.plugins.FieldPredicate;
5152
import org.elasticsearch.plugins.MapperPlugin;
5253
import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
@@ -2870,6 +2871,46 @@ public void testRetrieveIndexModeFromTemplateEmpty() throws IOException {
28702871
}
28712872
}
28722873

2874+
public void testGetSingleProjectWithCustom() {
2875+
var type = IngestMetadata.TYPE;
2876+
{
2877+
Metadata metadata = Metadata.builder().build();
2878+
assertNull(metadata.getSingleProjectCustom(type));
2879+
assertNull(metadata.getSingleProjectWithCustom(type));
2880+
}
2881+
{
2882+
Metadata metadata = Metadata.builder().put(ProjectMetadata.builder(new ProjectId(randomUUID())).build()).build();
2883+
assertNull(metadata.getSingleProjectCustom(type));
2884+
assertNull(metadata.getSingleProjectWithCustom(type));
2885+
}
2886+
{
2887+
var ingestMetadata = new IngestMetadata(Map.of());
2888+
Metadata metadata = Metadata.builder()
2889+
.put(ProjectMetadata.builder(new ProjectId(randomUUID())).putCustom(type, ingestMetadata))
2890+
.build();
2891+
assertEquals(ingestMetadata, metadata.getSingleProjectCustom(type));
2892+
assertEquals(ingestMetadata, metadata.getSingleProjectWithCustom(type).custom(type));
2893+
}
2894+
{
2895+
var ingestMetadata = new IngestMetadata(Map.of());
2896+
Metadata metadata = Metadata.builder()
2897+
.put(ProjectMetadata.builder(new ProjectId(randomUUID())))
2898+
.put(ProjectMetadata.builder(new ProjectId(randomUUID())).putCustom(type, ingestMetadata))
2899+
.build();
2900+
assertEquals(ingestMetadata, metadata.getSingleProjectCustom(type));
2901+
assertEquals(ingestMetadata, metadata.getSingleProjectWithCustom(type).custom(type));
2902+
}
2903+
{
2904+
var ingestMetadata = new IngestMetadata(Map.of());
2905+
Metadata metadata = Metadata.builder()
2906+
.put(ProjectMetadata.builder(new ProjectId(randomUUID())).putCustom(type, new IngestMetadata(Map.of())))
2907+
.put(ProjectMetadata.builder(new ProjectId(randomUUID())).putCustom(type, ingestMetadata))
2908+
.build();
2909+
assertThrows(UnsupportedOperationException.class, () -> metadata.getSingleProjectCustom(type));
2910+
assertThrows(UnsupportedOperationException.class, () -> metadata.getSingleProjectWithCustom(type));
2911+
}
2912+
}
2913+
28732914
public void testProjectLookupWithSingleProject() {
28742915
final ProjectId projectId = Metadata.DEFAULT_PROJECT_ID;
28752916
final ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(projectId);

‎server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java

+4-12
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,8 @@ public void testDisassociateDeadNodes_givenAssignedPersistentTask() {
335335
ClusterState returnedState = PersistentTasksCustomMetadata.disassociateDeadNodes(originalState);
336336
assertThat(originalState, sameInstance(returnedState));
337337

338-
PersistentTasksCustomMetadata originalTasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(
339-
originalState.metadata().getProject(projectId)
340-
);
341-
PersistentTasksCustomMetadata returnedTasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(
342-
returnedState.metadata().getProject(projectId)
343-
);
338+
PersistentTasksCustomMetadata originalTasks = PersistentTasksCustomMetadata.get(originalState.metadata().getProject(projectId));
339+
PersistentTasksCustomMetadata returnedTasks = PersistentTasksCustomMetadata.get(returnedState.metadata().getProject(projectId));
344340
assertEquals(originalTasks, returnedTasks);
345341
}
346342

@@ -377,12 +373,8 @@ public void testDisassociateDeadNodes() {
377373
ClusterState returnedState = PersistentTasksCustomMetadata.disassociateDeadNodes(originalState);
378374
assertThat(originalState, not(sameInstance(returnedState)));
379375

380-
PersistentTasksCustomMetadata originalTasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(
381-
originalState.metadata().getProject(projectId)
382-
);
383-
PersistentTasksCustomMetadata returnedTasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(
384-
returnedState.metadata().getProject(projectId)
385-
);
376+
PersistentTasksCustomMetadata originalTasks = PersistentTasksCustomMetadata.get(originalState.metadata().getProject(projectId));
377+
PersistentTasksCustomMetadata returnedTasks = PersistentTasksCustomMetadata.get(returnedState.metadata().getProject(projectId));
386378
assertNotEquals(originalTasks, returnedTasks);
387379

388380
assertEquals(originalTasks.getTask("assigned-task"), returnedTasks.getTask("assigned-task"));

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,9 @@ public MlMetadata build() {
244244
}
245245
}
246246

247+
@Deprecated(forRemoval = true)
247248
public static MlMetadata getMlMetadata(ClusterState state) {
248-
MlMetadata mlMetadata = (state == null) ? null : state.getMetadata().getProject().custom(TYPE);
249+
MlMetadata mlMetadata = (state == null) ? null : state.metadata().getSingleProjectCustom(TYPE);
249250
if (mlMetadata == null) {
250251
return EMPTY_METADATA;
251252
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentMetadata.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,11 @@ public static Builder builder(ClusterState clusterState) {
7171
return Builder.fromMetadata(fromState(clusterState));
7272
}
7373

74+
@Deprecated(forRemoval = true)
7475
public static TrainedModelAssignmentMetadata fromState(ClusterState clusterState) {
75-
TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = clusterState.getMetadata().getProject().custom(NAME);
76+
TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = clusterState.metadata().getSingleProjectCustom(NAME);
7677
if (trainedModelAssignmentMetadata == null) {
77-
trainedModelAssignmentMetadata = clusterState.getMetadata().getProject().custom(DEPRECATED_NAME);
78+
trainedModelAssignmentMetadata = clusterState.metadata().getSingleProjectCustom(DEPRECATED_NAME);
7879
}
7980
return trainedModelAssignmentMetadata == null ? EMPTY : trainedModelAssignmentMetadata;
8081
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public static int countInferenceProcessors(ClusterState state) {
4545
if (metadata == null) {
4646
return 0;
4747
}
48-
IngestMetadata ingestMetadata = metadata.getProject().custom(IngestMetadata.TYPE);
48+
IngestMetadata ingestMetadata = metadata.getDefaultProject().custom(IngestMetadata.TYPE);
4949
if (ingestMetadata == null) {
5050
return 0;
5151
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,9 @@ public TransformMetadata build() {
209209
}
210210
}
211211

212+
@Deprecated(forRemoval = true)
212213
public static TransformMetadata getTransformMetadata(ClusterState state) {
213-
TransformMetadata TransformMetadata = (state == null) ? null : state.getMetadata().getProject().custom(TYPE);
214+
TransformMetadata TransformMetadata = (state == null) ? null : state.metadata().getSingleProjectCustom(TYPE);
214215
if (TransformMetadata == null) {
215216
return EMPTY_METADATA;
216217
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCancelJobModelSnapshotUpgradeAction.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,9 @@ public void doExecute(Task task, Request request, ActionListener<Response> liste
7171
ActionListener<List<Job.Builder>> expandIdsListener = listener.delegateFailureAndWrap((delegate, jobs) -> {
7272
SimpleIdsMatcher matcher = new SimpleIdsMatcher(request.getSnapshotId());
7373
Set<String> jobIds = jobs.stream().map(Job.Builder::getId).collect(Collectors.toSet());
74-
PersistentTasksCustomMetadata tasksInProgress = clusterService.state()
75-
.metadata()
76-
.getProject()
77-
.custom(PersistentTasksCustomMetadata.TYPE);
74+
PersistentTasksCustomMetadata tasksInProgress = PersistentTasksCustomMetadata.get(
75+
clusterService.state().metadata().getDefaultProject()
76+
);
7877
// allow_no_match plays no part here. The reason is that we have a principle that stopping
7978
// a stopped entity is a no-op, and upgrades that have already completed won't have a task.
8079
// This is a bit different to jobs and datafeeds, where the entity continues to exist even

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java

+4-7
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ protected void doExecute(Task task, CloseJobAction.Request request, ActionListen
149149
final boolean isForce = request.isForce();
150150
final TimeValue timeout = request.getCloseTimeout();
151151

152-
PersistentTasksCustomMetadata tasksMetadata = state.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
152+
PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.get(state.metadata().getDefaultProject());
153153
jobConfigProvider.expandJobsIds(
154154
request.getJobId(),
155155
request.allowNoMatch(),
@@ -181,13 +181,10 @@ protected void doExecute(Task task, CloseJobAction.Request request, ActionListen
181181
forceCloseJob(state, request, jobIdsToForceClose, delegate3);
182182
} else {
183183
Set<String> executorNodes = new HashSet<>();
184-
PersistentTasksCustomMetadata tasks = state.metadata()
185-
.getProject()
186-
.custom(PersistentTasksCustomMetadata.TYPE);
187184
for (String resolvedJobId : request.getOpenJobIds()) {
188185
PersistentTasksCustomMetadata.PersistentTask<?> jobTask = MlTasks.getJobTask(
189186
resolvedJobId,
190-
tasks
187+
tasksMetadata
191188
);
192189
if (jobTask == null) {
193190
// This should not happen, because openJobIds was
@@ -509,7 +506,7 @@ private void forceCloseJob(
509506
List<String> jobIdsToForceClose,
510507
ActionListener<CloseJobAction.Response> listener
511508
) {
512-
PersistentTasksCustomMetadata tasks = currentState.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
509+
PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(currentState);
513510

514511
final int numberOfJobs = jobIdsToForceClose.size();
515512
final AtomicInteger counter = new AtomicInteger();
@@ -577,7 +574,7 @@ private void normalCloseJob(
577574
List<String> closingJobIds,
578575
ActionListener<CloseJobAction.Response> listener
579576
) {
580-
PersistentTasksCustomMetadata tasks = currentState.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
577+
PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(currentState);
581578

582579
WaitForCloseRequest waitForCloseRequest = buildWaitForCloseRequest(openJobIds, closingJobIds, tasks, auditor);
583580

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ protected void taskOperation(
9696
@Override
9797
protected void doExecute(Task task, KillProcessAction.Request request, ActionListener<KillProcessAction.Response> listener) {
9898
DiscoveryNodes nodes = clusterService.state().nodes();
99-
PersistentTasksCustomMetadata tasks = clusterService.state().getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
99+
PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.get(clusterService.state().metadata().getDefaultProject());
100100
List<PersistentTasksCustomMetadata.PersistentTask<?>> jobTasks;
101101
if (Strings.isAllOrWildcard(request.getJobId())) {
102102
jobTasks = MlTasks.openJobTasks(tasks).stream().filter(t -> t.getExecutorNode() != null).collect(Collectors.toList());

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetResetModeAction.java

+7-12
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import org.elasticsearch.action.support.ActionFilters;
1010
import org.elasticsearch.cluster.ClusterState;
1111
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
12-
import org.elasticsearch.cluster.metadata.Metadata;
12+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1313
import org.elasticsearch.cluster.service.ClusterService;
1414
import org.elasticsearch.injection.guice.Inject;
1515
import org.elasticsearch.threadpool.ThreadPool;
@@ -45,20 +45,15 @@ protected String featureName() {
4545

4646
@Override
4747
protected ClusterState setState(ClusterState oldState, SetResetModeActionRequest request) {
48-
ClusterState.Builder newState = ClusterState.builder(oldState);
48+
final ProjectMetadata project = oldState.metadata().getDefaultProject();
49+
final ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(project);
4950
if (request.shouldDeleteMetadata()) {
5051
assert request.isEnabled() == false; // SetResetModeActionRequest should have enforced this
51-
newState.metadata(
52-
Metadata.builder(oldState.getMetadata())
53-
.removeProjectCustom(MlMetadata.TYPE)
54-
.removeProjectCustom(ModelAliasMetadata.NAME)
55-
.build()
56-
);
52+
projectBuilder.removeCustom(MlMetadata.TYPE).removeCustom(ModelAliasMetadata.NAME);
5753
} else {
58-
MlMetadata.Builder builder = MlMetadata.Builder.from(oldState.metadata().getProject().custom(MlMetadata.TYPE))
59-
.isResetMode(request.isEnabled());
60-
newState.metadata(Metadata.builder(oldState.getMetadata()).putCustom(MlMetadata.TYPE, builder.build()).build());
54+
MlMetadata.Builder builder = MlMetadata.Builder.from(project.custom(MlMetadata.TYPE)).isResetMode(request.isEnabled());
55+
projectBuilder.putCustom(MlMetadata.TYPE, builder.build());
6156
}
62-
return newState.build();
57+
return ClusterState.builder(oldState).putProjectMetadata(projectBuilder.build()).build();
6358
}
6459
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ protected void doExecute(
118118
ActionListener<Set<String>> expandedIdsListener = ActionListener.wrap(idsToStop -> {
119119
logger.debug("Resolved data frame analytics to stop: {}", idsToStop);
120120

121-
PersistentTasksCustomMetadata tasks = state.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
121+
PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.get(state.metadata().getDefaultProject());
122122
AnalyticsByTaskState analyticsByTaskState = AnalyticsByTaskState.build(idsToStop, tasks);
123123

124124
if (analyticsByTaskState.isEmpty()) {
@@ -165,7 +165,7 @@ private void findIdsToStop(
165165
}
166166

167167
private static Set<String> getAllStartedIds(ClusterState clusterState) {
168-
PersistentTasksCustomMetadata tasksMetadata = clusterState.getMetadata().getProject().custom(PersistentTasksCustomMetadata.TYPE);
168+
PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.get(clusterState.metadata().getDefaultProject());
169169
return tasksMetadata == null
170170
? Collections.emptySet()
171171
: tasksMetadata.tasks()

0 commit comments

Comments
 (0)