Skip to content

Commit 352023d

Browse files
csviricsviri
committed
Merge branch 'integration-testing' into not_scheduled_yet_event_processing
# Conflicts: # operator-framework/src/main/java/com/github/containersolutions/operator/processing/EventScheduler.java
2 parents 9a563a2 + 3745198 commit 352023d

File tree

11 files changed

+367
-8
lines changed

11 files changed

+367
-8
lines changed

‎operator-framework/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -72,5 +72,11 @@
7272
<version>3.4.1</version>
7373
<scope>test</scope>
7474
</dependency>
75+
<dependency>
76+
<groupId>org.awaitility</groupId>
77+
<artifactId>awaitility</artifactId>
78+
<version>4.0.1</version>
79+
<scope>test</scope>
80+
</dependency>
7581
</dependencies>
7682
</project>

‎operator-framework/src/main/java/com/github/containersolutions/operator/processing/EventDispatcher.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,27 @@ public EventDispatcher(ResourceController<R> controller,
4141
}
4242

4343
public void handleEvent(Watcher.Action action, R resource) {
44+
log.info("Handling event {} for resource {}", action, resource.getMetadata());
4445
if (action == Watcher.Action.MODIFIED || action == Watcher.Action.ADDED) {
4546
// we don't want to call delete resource if it not contains our finalizer,
4647
// since the resource still can be updates when marked for deletion and contains other finalizers
4748
if (markedForDeletion(resource) && hasDefaultFinalizer(resource)) {
4849
boolean removeFinalizer = controller.deleteResource(resource, new Context(k8sClient, resourceClient));
4950
if (removeFinalizer) {
51+
log.debug("Removing finalizer on {}: {}", resource.getMetadata().getName(), resource.getMetadata());
5052
removeDefaultFinalizer(resource);
5153
}
52-
} else {
54+
} else if (!markedForDeletion(resource)){
5355
Optional<R> updateResult = controller.createOrUpdateResource(resource, new Context<>(k8sClient, resourceClient));
5456
if (updateResult.isPresent()) {
5557
R updatedResource = updateResult.get();
58+
log.info("Actual resource in etcd {}", resourceOperation.withName(resource.getMetadata().getName()).get());
59+
log.info("Updated resource handled {}", updatedResource.getMetadata());
5660
addFinalizerIfNotPresent(updatedResource);
5761
replace(updatedResource);
5862
// We always add the default finalizer if missing and not marked for deletion.
59-
} else if (!hasDefaultFinalizer(resource) && !markedForDeletion(resource)) {
63+
} else if (!hasDefaultFinalizer(resource)) {
64+
log.info("Actual resource with no finalizer: {}", resourceOperation.withName(resource.getMetadata().getName()).get());
6065
addFinalizerIfNotPresent(resource);
6166
replace(resource);
6267
}
@@ -81,15 +86,18 @@ private boolean hasDefaultFinalizer(R resource) {
8186

8287
private void removeDefaultFinalizer(R resource) {
8388
resource.getMetadata().getFinalizers().remove(resourceDefaultFinalizer);
89+
log.debug("Removed finalizer. Trying to replace resource {}, version: {}", resource.getMetadata().getName(), resource.getMetadata().getResourceVersion());
8490
resourceOperation.lockResourceVersion(resource.getMetadata().getResourceVersion()).replace(resource);
8591
}
8692

8793
private void replace(R resource) {
94+
log.debug("Trying to replace resource {}, version: {}", resource.getMetadata().getName(), resource.getMetadata().getResourceVersion());
8895
resourceOperation.lockResourceVersion(resource.getMetadata().getResourceVersion()).replace(resource);
8996
}
9097

9198
private void addFinalizerIfNotPresent(R resource) {
9299
if (!hasDefaultFinalizer(resource)) {
100+
log.info("Adding default finalizer to {}", resource.getMetadata());
93101
if (resource.getMetadata().getFinalizers() == null) {
94102
resource.getMetadata().setFinalizers(new ArrayList<>(1));
95103
}

‎operator-framework/src/main/java/com/github/containersolutions/operator/processing/EventScheduler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,12 @@ void scheduleEvent(CustomResourceEvent event) {
103103
ScheduledFuture<?> scheduledFuture = scheduleHolder.getScheduledFuture();
104104
// If newEvent is older than existing in queue, don't schedule and remove from cache
105105
if (scheduledEvent.isSameResourceAndNewerVersion(event)) {
106-
log.debug("Incoming event discarded because already scheduled event is newer. {}", event);
106+
log.debug("Incoming event discarded because already scheduled event is newer. Discarded: {}, Scheduled: {}", event, scheduledEvent);
107107
return;
108108
}
109109
// If newEvent is newer than existing in queue, cancel and remove queuedEvent
110110
if (event.isSameResourceAndNewerVersion(scheduledEvent)) {
111-
log.debug("Scheduled event canceled because incoming event is newer. {}", scheduledEvent);
111+
log.debug("Scheduled event canceled because incoming event is newer. Discarded: {}, New: {}", scheduledEvent, event);
112112
scheduledFuture.cancel(false);
113113
eventStore.removeEventScheduledForProcessing(scheduledEvent.resourceUid());
114114
}

‎operator-framework/src/main/java/com/github/containersolutions/operator/processing/EventStore.java

+3
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ private void updateLatestResourceVersionProcessed(CustomResourceEvent event) {
6767
long received = Long.parseLong(event.getResource().getMetadata().getResourceVersion());
6868
if (current == null || received > current) {
6969
lastResourceVersion.put(event.resourceUid(), received);
70+
log.debug("Resource version for {} updated from {} to {}", event.getResource().getMetadata().getName(), current, received);
71+
} else {
72+
log.debug("Resource version for {} not updated from {}", event.getResource().getMetadata().getName(), current);
7073
}
7174
}
7275

‎operator-framework/src/test/java/com/github/containersolutions/operator/ConcurrencyTest.java

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package com.github.containersolutions.operator;
2+
3+
import com.github.containersolutions.operator.sample.TestCustomResource;
4+
import com.github.containersolutions.operator.sample.TestCustomResourceSpec;
5+
import io.fabric8.kubernetes.api.model.ConfigMap;
6+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
7+
import org.awaitility.Awaitility;
8+
import org.junit.jupiter.api.*;
9+
10+
import java.util.List;
11+
import java.util.concurrent.TimeUnit;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
15+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
16+
public class ConcurrencyTest {
17+
18+
private IntegrationTest integrationTest = new IntegrationTest();
19+
20+
@BeforeAll
21+
public void setup() {
22+
integrationTest.setup();
23+
}
24+
25+
@BeforeEach
26+
public void cleanup() {
27+
integrationTest.cleanup();
28+
}
29+
30+
@AfterAll
31+
public void teardown() {
32+
integrationTest.teardown();
33+
}
34+
35+
@Test
36+
public void manyResourcesGetCreatedAndUpdated() {
37+
for (int i = 0; i < 35; i++) {
38+
TestCustomResource tcr = createTCR(String.valueOf(i));
39+
integrationTest.crOperations.inNamespace(IntegrationTest.TEST_NAMESPACE).create(tcr);
40+
// if (i % 3 == 0) {
41+
// TestCustomResource newTcr = createTCR(String.valueOf(i));
42+
// integrationTest.crOperations.inNamespace(IntegrationTest.TEST_NAMESPACE).createOrReplace(newTcr);
43+
// }
44+
}
45+
Awaitility.await().atMost(1, TimeUnit.MINUTES)
46+
.untilAsserted(() -> {
47+
List<ConfigMap> items = integrationTest.k8sClient.configMaps()
48+
.inNamespace(IntegrationTest.TEST_NAMESPACE)
49+
.list().getItems();
50+
assertThat(items).hasSize(35);
51+
});
52+
}
53+
54+
55+
@Test
56+
public void manyResourcesGetCreatedUpdatedAndDeleted() {
57+
for (int i = 0; i < 35; i++) {
58+
TestCustomResource tcr = createTCR(String.valueOf(i));
59+
integrationTest.crOperations.inNamespace(IntegrationTest.TEST_NAMESPACE).create(tcr);
60+
// if (i % 3 == 0) {
61+
// TestCustomResource newTcr = createTCR(String.valueOf(i));
62+
// integrationTest.crOperations.inNamespace(IntegrationTest.TEST_NAMESPACE).createOrReplace(newTcr);
63+
// }
64+
}
65+
66+
Awaitility.await().atMost(1, TimeUnit.MINUTES)
67+
.untilAsserted(() -> {
68+
List<ConfigMap> items = integrationTest.k8sClient.configMaps()
69+
.inNamespace(IntegrationTest.TEST_NAMESPACE)
70+
.list().getItems();
71+
assertThat(items).hasSize(35);
72+
});
73+
74+
for (int i = 0; i < 10; i++) {
75+
TestCustomResource tcr = createTCR(String.valueOf(i));
76+
integrationTest.crOperations.inNamespace(IntegrationTest.TEST_NAMESPACE).delete(tcr);
77+
}
78+
79+
Awaitility.await().atMost(1, TimeUnit.MINUTES)
80+
.untilAsserted(() -> {
81+
List<ConfigMap> items = integrationTest.k8sClient.configMaps()
82+
.inNamespace(IntegrationTest.TEST_NAMESPACE)
83+
.list().getItems();
84+
assertThat(items).hasSize(25);
85+
});
86+
}
87+
88+
private TestCustomResource createTCR(String id) {
89+
TestCustomResource resource = new TestCustomResource();
90+
resource.setMetadata(new ObjectMetaBuilder()
91+
.withName("test-custom-resource-" + id)
92+
.withNamespace(IntegrationTest.TEST_NAMESPACE)
93+
.build());
94+
resource.setKind("CustomService");
95+
resource.setSpec(new TestCustomResourceSpec());
96+
resource.getSpec().setConfigMapName("test-config-map-" + id);
97+
resource.getSpec().setKey("test-key");
98+
resource.getSpec().setValue(id);
99+
return resource;
100+
}
101+
}

‎operator-framework/src/test/java/com/github/containersolutions/operator/IntegrationTest.java

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package com.github.containersolutions.operator;
2+
3+
import com.github.containersolutions.operator.sample.*;
4+
import io.fabric8.kubernetes.api.model.ConfigMap;
5+
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
6+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
7+
import io.fabric8.kubernetes.api.model.apiextensions.CustomResourceDefinition;
8+
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
9+
import io.fabric8.kubernetes.client.KubernetesClient;
10+
import io.fabric8.kubernetes.client.dsl.MixedOperation;
11+
import io.fabric8.kubernetes.client.dsl.Resource;
12+
import io.fabric8.kubernetes.client.utils.Serialization;
13+
import io.fabric8.kubernetes.internal.KubernetesDeserializer;
14+
import org.junit.jupiter.api.*;
15+
import org.mockito.internal.matchers.Any;
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
18+
19+
import java.io.IOException;
20+
import java.io.InputStream;
21+
import java.util.concurrent.TimeUnit;
22+
23+
import static org.assertj.core.api.Assertions.assertThat;
24+
import static org.awaitility.Awaitility.await;
25+
26+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
27+
public class IntegrationTest {
28+
29+
public static final String TEST_NAMESPACE = "java-operator-sdk-int-test";
30+
31+
public final KubernetesClient k8sClient = new DefaultKubernetesClient();
32+
public MixedOperation<TestCustomResource, TestCustomResourceList, TestCustomResourceDoneable, Resource<TestCustomResource, TestCustomResourceDoneable>> crOperations;
33+
34+
private final Logger log = LoggerFactory.getLogger(getClass());
35+
36+
private Operator operator;
37+
38+
@BeforeAll
39+
public void setup() {
40+
log.info("Running integration test in namespace " + TEST_NAMESPACE);
41+
42+
CustomResourceDefinition crd = loadYaml(CustomResourceDefinition.class, "test-crd.yaml");
43+
k8sClient.customResourceDefinitions().createOrReplace(crd);
44+
45+
if (k8sClient.namespaces().withName(TEST_NAMESPACE).get() == null) {
46+
k8sClient.namespaces().create(new NamespaceBuilder()
47+
.withMetadata(new ObjectMetaBuilder().withName(TEST_NAMESPACE).build()).build());
48+
}
49+
50+
operator = new Operator(k8sClient);
51+
operator.registerController(new TestCustomResourceController());
52+
53+
}
54+
55+
@BeforeEach
56+
public void cleanup() {
57+
58+
CustomResourceDefinition crd = loadYaml(CustomResourceDefinition.class, "test-crd.yaml");
59+
k8sClient.customResourceDefinitions().createOrReplace(crd);
60+
KubernetesDeserializer.registerCustomKind(crd.getApiVersion(), crd.getKind(), TestCustomResource.class);
61+
62+
k8sClient.configMaps().inNamespace(TEST_NAMESPACE)
63+
.withLabel("managedBy", TestCustomResourceController.class.getSimpleName())
64+
.delete();
65+
66+
crOperations = k8sClient.customResources(crd, TestCustomResource.class, TestCustomResourceList.class, TestCustomResourceDoneable.class);
67+
crOperations.inNamespace(TEST_NAMESPACE).delete(crOperations.list().getItems());
68+
//we depend on the actual operator from the startup to handle the finalizers and clean up
69+
//resources from previous test runs
70+
71+
await("all resources cleaned up").atMost(60, TimeUnit.SECONDS)
72+
.untilAsserted(() -> {
73+
assertThat(crOperations.inNamespace(TEST_NAMESPACE).list().getItems()).isEmpty();
74+
assertThat(k8sClient.configMaps().inNamespace(TEST_NAMESPACE).list().getItems()).isEmpty();
75+
});
76+
77+
log.info("Cleaned up namespace " + TEST_NAMESPACE);
78+
}
79+
80+
@AfterAll
81+
public void teardown() {
82+
// CustomResourceDefinition crd = loadYaml(CustomResourceDefinition.class, "test-crd.yaml");
83+
// k8sClient.customResourceDefinitions().delete(crd);
84+
operator.stop();
85+
}
86+
87+
@Test
88+
public void configMapGetsCreatedForTestCustomResource() {
89+
TestCustomResource resource = new TestCustomResource();
90+
resource.setMetadata(new ObjectMetaBuilder()
91+
.withName("test-custom-resource")
92+
.withNamespace(TEST_NAMESPACE)
93+
.build());
94+
resource.setKind("CustomService");
95+
resource.setSpec(new TestCustomResourceSpec());
96+
resource.getSpec().setConfigMapName("test-config-map");
97+
resource.getSpec().setKey("test-key");
98+
resource.getSpec().setValue("test-value");
99+
crOperations.inNamespace(TEST_NAMESPACE).create(resource);
100+
101+
await("configmap created").atMost(5, TimeUnit.SECONDS)
102+
.untilAsserted(() -> {
103+
ConfigMap configMap = k8sClient.configMaps().inNamespace(TEST_NAMESPACE)
104+
.withName("test-config-map").get();
105+
assertThat(configMap).isNotNull();
106+
assertThat(configMap.getData().get("test-key")).isEqualTo("test-value");
107+
});
108+
await("cr status updated").atMost(5, TimeUnit.SECONDS)
109+
.untilAsserted(() -> {
110+
TestCustomResource cr = crOperations.inNamespace(TEST_NAMESPACE).withName("test-custom-resource").get();
111+
assertThat(cr).isNotNull();
112+
assertThat(cr.getStatus()).isNotNull();
113+
assertThat(cr.getStatus().getConfigMapStatus()).isEqualTo("ConfigMap Ready");
114+
});
115+
}
116+
117+
private <T> T loadYaml(Class<T> clazz, String yaml) {
118+
try (InputStream is = getClass().getResourceAsStream(yaml)) {
119+
return Serialization.unmarshal(is, clazz);
120+
} catch (IOException ex) {
121+
throw new IllegalStateException("Cannot find yaml on classpath: " + yaml);
122+
}
123+
}
124+
}

‎operator-framework/src/test/java/com/github/containersolutions/operator/sample/TestCustomResource.java

+19
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,23 @@
44

55
public class TestCustomResource extends CustomResource {
66

7+
private TestCustomResourceSpec spec;
8+
9+
private TestCustomResourceStatus status;
10+
11+
public TestCustomResourceSpec getSpec() {
12+
return spec;
13+
}
14+
15+
public void setSpec(TestCustomResourceSpec spec) {
16+
this.spec = spec;
17+
}
18+
19+
public TestCustomResourceStatus getStatus() {
20+
return status;
21+
}
22+
23+
public void setStatus(TestCustomResourceStatus status) {
24+
this.status = status;
25+
}
726
}

0 commit comments

Comments
 (0)