Skip to content

Commit 3b5c91e

Browse files
xstefankmetacosm
authored and
csviri
committed
feat: allow overriding test infrastructure kube client separately (#2764)
Co-authored-by: Chris Laprun <[email protected]>
1 parent 0153b3a commit 3b5c91e

File tree

9 files changed

+276
-12
lines changed

9 files changed

+276
-12
lines changed

‎operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/AbstractOperatorExtension.java‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public abstract class AbstractOperatorExtension
3838
public static final int DEFAULT_NAMESPACE_DELETE_TIMEOUT = 90;
3939

4040
private final KubernetesClient kubernetesClient;
41+
private final KubernetesClient infrastructureKubernetesClient;
4142
protected final List<HasMetadata> infrastructure;
4243
protected Duration infrastructureTimeout;
4344
protected final boolean oneNamespacePerClass;
@@ -56,10 +57,15 @@ protected AbstractOperatorExtension(
5657
boolean preserveNamespaceOnError,
5758
boolean waitForNamespaceDeletion,
5859
KubernetesClient kubernetesClient,
60+
KubernetesClient infrastructureKubernetesClient,
5961
Function<ExtensionContext, String> namespaceNameSupplier,
6062
Function<ExtensionContext, String> perClassNamespaceNameSupplier) {
63+
this.infrastructureKubernetesClient =
64+
infrastructureKubernetesClient != null
65+
? infrastructureKubernetesClient
66+
: new KubernetesClientBuilder().build();
6167
this.kubernetesClient =
62-
kubernetesClient != null ? kubernetesClient : new KubernetesClientBuilder().build();
68+
kubernetesClient != null ? kubernetesClient : this.infrastructureKubernetesClient;
6369
this.infrastructure = infrastructure;
6470
this.infrastructureTimeout = infrastructureTimeout;
6571
this.oneNamespacePerClass = oneNamespacePerClass;
@@ -94,6 +100,11 @@ public KubernetesClient getKubernetesClient() {
94100
return kubernetesClient;
95101
}
96102

103+
@Override
104+
public KubernetesClient getInfrastructureKubernetesClient() {
105+
return infrastructureKubernetesClient;
106+
}
107+
97108
public String getNamespace() {
98109
return namespace;
99110
}
@@ -141,16 +152,16 @@ protected void beforeEachImpl(ExtensionContext context) {
141152
protected void before(ExtensionContext context) {
142153
LOGGER.info("Initializing integration test in namespace {}", namespace);
143154

144-
kubernetesClient
155+
infrastructureKubernetesClient
145156
.namespaces()
146157
.resource(
147158
new NamespaceBuilder()
148159
.withMetadata(new ObjectMetaBuilder().withName(namespace).build())
149160
.build())
150161
.serverSideApply();
151162

152-
kubernetesClient.resourceList(infrastructure).serverSideApply();
153-
kubernetesClient
163+
infrastructureKubernetesClient.resourceList(infrastructure).serverSideApply();
164+
infrastructureKubernetesClient
154165
.resourceList(infrastructure)
155166
.waitUntilReady(infrastructureTimeout.toMillis(), TimeUnit.MILLISECONDS);
156167
}
@@ -172,16 +183,19 @@ protected void after(ExtensionContext context) {
172183
if (preserveNamespaceOnError && context.getExecutionException().isPresent()) {
173184
LOGGER.info("Preserving namespace {}", namespace);
174185
} else {
175-
kubernetesClient.resourceList(infrastructure).delete();
186+
infrastructureKubernetesClient.resourceList(infrastructure).delete();
176187
deleteOperator();
177188
LOGGER.info("Deleting namespace {} and stopping operator", namespace);
178-
kubernetesClient.namespaces().withName(namespace).delete();
189+
infrastructureKubernetesClient.namespaces().withName(namespace).delete();
179190
if (waitForNamespaceDeletion) {
180191
LOGGER.info("Waiting for namespace {} to be deleted", namespace);
181192
Awaitility.await("namespace deleted")
182193
.pollInterval(50, TimeUnit.MILLISECONDS)
183194
.atMost(namespaceDeleteTimeout, TimeUnit.SECONDS)
184-
.until(() -> kubernetesClient.namespaces().withName(namespace).get() == null);
195+
.until(
196+
() ->
197+
infrastructureKubernetesClient.namespaces().withName(namespace).get()
198+
== null);
185199
}
186200
}
187201
}

‎operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ private ClusterDeployedOperatorExtension(
3939
boolean waitForNamespaceDeletion,
4040
boolean oneNamespacePerClass,
4141
KubernetesClient kubernetesClient,
42+
KubernetesClient infrastructureKubernetesClient,
4243
Function<ExtensionContext, String> namespaceNameSupplier,
4344
Function<ExtensionContext, String> perClassNamespaceNameSupplier) {
4445
super(
@@ -48,6 +49,7 @@ private ClusterDeployedOperatorExtension(
4849
preserveNamespaceOnError,
4950
waitForNamespaceDeletion,
5051
kubernetesClient,
52+
infrastructureKubernetesClient,
5153
namespaceNameSupplier,
5254
perClassNamespaceNameSupplier);
5355
this.operatorDeployment = operatorDeployment;
@@ -69,7 +71,7 @@ protected void before(ExtensionContext context) {
6971
final var crdPath = "./target/classes/META-INF/fabric8/";
7072
final var crdSuffix = "-v1.yml";
7173

72-
final var kubernetesClient = getKubernetesClient();
74+
final var kubernetesClient = getInfrastructureKubernetesClient();
7375
for (var crdFile :
7476
Objects.requireNonNull(
7577
new File(crdPath).listFiles((ignored, name) -> name.endsWith(crdSuffix)))) {
@@ -107,13 +109,17 @@ protected void before(ExtensionContext context) {
107109

108110
@Override
109111
protected void deleteOperator() {
110-
getKubernetesClient().resourceList(operatorDeployment).inNamespace(namespace).delete();
112+
getInfrastructureKubernetesClient()
113+
.resourceList(operatorDeployment)
114+
.inNamespace(namespace)
115+
.delete();
111116
}
112117

113118
public static class Builder extends AbstractBuilder<Builder> {
114119
private final List<HasMetadata> operatorDeployment;
115120
private Duration deploymentTimeout;
116121
private KubernetesClient kubernetesClient;
122+
private KubernetesClient infrastructureKubernetesClient;
117123

118124
protected Builder() {
119125
super();
@@ -150,7 +156,18 @@ public Builder withKubernetesClient(KubernetesClient kubernetesClient) {
150156
return this;
151157
}
152158

159+
public Builder withInfrastructureKubernetesClient(KubernetesClient kubernetesClient) {
160+
this.infrastructureKubernetesClient = kubernetesClient;
161+
return this;
162+
}
163+
153164
public ClusterDeployedOperatorExtension build() {
165+
infrastructureKubernetesClient =
166+
infrastructureKubernetesClient != null
167+
? infrastructureKubernetesClient
168+
: new KubernetesClientBuilder().build();
169+
kubernetesClient =
170+
kubernetesClient != null ? kubernetesClient : infrastructureKubernetesClient;
154171
return new ClusterDeployedOperatorExtension(
155172
operatorDeployment,
156173
deploymentTimeout,
@@ -159,7 +176,8 @@ public ClusterDeployedOperatorExtension build() {
159176
preserveNamespaceOnError,
160177
waitForNamespaceDeletion,
161178
oneNamespacePerClass,
162-
kubernetesClient != null ? kubernetesClient : new KubernetesClientBuilder().build(),
179+
kubernetesClient,
180+
infrastructureKubernetesClient,
163181
namespaceNameSupplier,
164182
perClassNamespaceNameSupplier);
165183
}

‎operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/HasKubernetesClient.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,20 @@
33
import io.fabric8.kubernetes.client.KubernetesClient;
44

55
public interface HasKubernetesClient {
6+
/**
7+
* Returns the main Kubernetes client that is used to deploy the operator to the cluster.
8+
*
9+
* @return the main Kubernetes client
10+
*/
611
KubernetesClient getKubernetesClient();
12+
13+
/**
14+
* Returns the Kubernetes client that is used to deploy infrastructure resources to the cluster
15+
* such as clusterroles, clusterrolebindings, etc. This client can be different from the main
16+
* client in case you need to test the operator with a different restrictions more closely
17+
* resembling the real restrictions it will have in production.
18+
*
19+
* @return the infrastructure Kubernetes client
20+
*/
21+
KubernetesClient getInfrastructureKubernetesClient();
722
}

‎operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/LocallyRunOperatorExtension.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ private LocallyRunOperatorExtension(
6666
boolean waitForNamespaceDeletion,
6767
boolean oneNamespacePerClass,
6868
KubernetesClient kubernetesClient,
69+
KubernetesClient infrastructureKubernetesClient,
6970
Consumer<ConfigurationServiceOverrider> configurationServiceOverrider,
7071
Function<ExtensionContext, String> namespaceNameSupplier,
7172
Function<ExtensionContext, String> perClassNamespaceNameSupplier,
@@ -78,6 +79,7 @@ private LocallyRunOperatorExtension(
7879
preserveNamespaceOnError,
7980
waitForNamespaceDeletion,
8081
kubernetesClient,
82+
infrastructureKubernetesClient,
8183
namespaceNameSupplier,
8284
perClassNamespaceNameSupplier);
8385
this.reconcilers = reconcilers;
@@ -240,7 +242,7 @@ public Operator getOperator() {
240242
protected void before(ExtensionContext context) {
241243
super.before(context);
242244

243-
final var kubernetesClient = getKubernetesClient();
245+
final var kubernetesClient = getInfrastructureKubernetesClient();
244246

245247
for (var ref : portForwards) {
246248
String podName =
@@ -313,7 +315,7 @@ protected void before(ExtensionContext context) {
313315
protected void after(ExtensionContext context) {
314316
super.after(context);
315317

316-
var kubernetesClient = getKubernetesClient();
318+
var kubernetesClient = getInfrastructureKubernetesClient();
317319

318320
var iterator = appliedCRDs.iterator();
319321
while (iterator.hasNext()) {
@@ -365,6 +367,7 @@ public static class Builder extends AbstractBuilder<Builder> {
365367
private final List<String> additionalCRDs = new ArrayList<>();
366368
private Consumer<LocallyRunOperatorExtension> beforeStartHook;
367369
private KubernetesClient kubernetesClient;
370+
private KubernetesClient infrastructureKubernetesClient;
368371

369372
protected Builder() {
370373
super();
@@ -419,6 +422,12 @@ public Builder withKubernetesClient(KubernetesClient kubernetesClient) {
419422
return this;
420423
}
421424

425+
public Builder withInfrastructureKubernetesClient(
426+
KubernetesClient infrastructureKubernetesClient) {
427+
this.infrastructureKubernetesClient = infrastructureKubernetesClient;
428+
return this;
429+
}
430+
422431
public Builder withAdditionalCustomResourceDefinition(
423432
Class<? extends CustomResource> customResource) {
424433
additionalCustomResourceDefinitions.add(customResource);
@@ -452,6 +461,7 @@ public LocallyRunOperatorExtension build() {
452461
waitForNamespaceDeletion,
453462
oneNamespacePerClass,
454463
kubernetesClient,
464+
infrastructureKubernetesClient,
455465
configurationServiceOverrider,
456466
namespaceNameSupplier,
457467
perClassNamespaceNameSupplier,

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/infrastructureclient/InfrastructureClientIT.java‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package io.javaoperatorsdk.operator.baseapi.infrastructureclient;
2+
3+
import java.util.concurrent.TimeUnit;
4+
5+
import org.junit.jupiter.api.AfterEach;
6+
import org.junit.jupiter.api.BeforeEach;
7+
import org.junit.jupiter.api.Test;
8+
import org.junit.jupiter.api.extension.RegisterExtension;
9+
10+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
11+
import io.fabric8.kubernetes.api.model.rbac.ClusterRole;
12+
import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding;
13+
import io.fabric8.kubernetes.client.ConfigBuilder;
14+
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
15+
import io.fabric8.kubernetes.client.KubernetesClientException;
16+
import io.javaoperatorsdk.operator.ReconcilerUtils;
17+
import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
18+
19+
import static org.assertj.core.api.Assertions.assertThat;
20+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
21+
import static org.awaitility.Awaitility.await;
22+
23+
class InfrastructureClientIT {
24+
25+
private static final String RBAC_TEST_ROLE = "rbac-test-role.yaml";
26+
private static final String RBAC_TEST_ROLE_BINDING = "rbac-test-role-binding.yaml";
27+
private static final String RBAC_TEST_USER = "rbac-test-user";
28+
29+
@RegisterExtension
30+
LocallyRunOperatorExtension operator =
31+
LocallyRunOperatorExtension.builder()
32+
.withReconciler(new InfrastructureClientTestReconciler())
33+
.withKubernetesClient(
34+
new KubernetesClientBuilder()
35+
.withConfig(new ConfigBuilder().withImpersonateUsername(RBAC_TEST_USER).build())
36+
.build())
37+
.withInfrastructureKubernetesClient(
38+
new KubernetesClientBuilder().build()) // no limitations
39+
.build();
40+
41+
/**
42+
* We need to apply the cluster role also before the CRD deployment so the rbac-test-user is
43+
* permitted to deploy it
44+
*/
45+
public InfrastructureClientIT() {
46+
applyClusterRole(RBAC_TEST_ROLE);
47+
applyClusterRoleBinding(RBAC_TEST_ROLE_BINDING);
48+
}
49+
50+
@BeforeEach
51+
void setup() {
52+
applyClusterRole(RBAC_TEST_ROLE);
53+
applyClusterRoleBinding(RBAC_TEST_ROLE_BINDING);
54+
}
55+
56+
@AfterEach
57+
void cleanup() {
58+
removeClusterRoleBinding(RBAC_TEST_ROLE_BINDING);
59+
removeClusterRole(RBAC_TEST_ROLE);
60+
}
61+
62+
@Test
63+
void canCreateInfrastructure() {
64+
var resource = new InfrastructureClientTestCustomResource();
65+
resource.setMetadata(
66+
new ObjectMetaBuilder().withName("infrastructure-client-resource").build());
67+
operator.create(resource);
68+
69+
await()
70+
.atMost(5, TimeUnit.SECONDS)
71+
.untilAsserted(
72+
() -> {
73+
InfrastructureClientTestCustomResource r =
74+
operator.get(
75+
InfrastructureClientTestCustomResource.class,
76+
"infrastructure-client-resource");
77+
assertThat(r).isNotNull();
78+
});
79+
80+
assertThat(
81+
operator
82+
.getReconcilerOfType(InfrastructureClientTestReconciler.class)
83+
.getNumberOfExecutions())
84+
.isEqualTo(1);
85+
}
86+
87+
@Test
88+
void shouldNotAccessNotPermittedResources() {
89+
assertThatThrownBy(
90+
() ->
91+
operator
92+
.getKubernetesClient()
93+
.apiextensions()
94+
.v1()
95+
.customResourceDefinitions()
96+
.list())
97+
.isInstanceOf(KubernetesClientException.class)
98+
.hasMessageContaining(
99+
"User \"%s\" cannot list resource \"customresourcedefinitions\""
100+
.formatted(RBAC_TEST_USER));
101+
102+
// but we should be able to access all resources with the infrastructure client
103+
var deploymentList =
104+
operator
105+
.getInfrastructureKubernetesClient()
106+
.apiextensions()
107+
.v1()
108+
.customResourceDefinitions()
109+
.list();
110+
assertThat(deploymentList).isNotNull();
111+
}
112+
113+
private void applyClusterRoleBinding(String filename) {
114+
var clusterRoleBinding =
115+
ReconcilerUtils.loadYaml(ClusterRoleBinding.class, this.getClass(), filename);
116+
operator.getInfrastructureKubernetesClient().resource(clusterRoleBinding).serverSideApply();
117+
}
118+
119+
private void applyClusterRole(String filename) {
120+
var clusterRole = ReconcilerUtils.loadYaml(ClusterRole.class, this.getClass(), filename);
121+
operator.getInfrastructureKubernetesClient().resource(clusterRole).serverSideApply();
122+
}
123+
124+
private void removeClusterRoleBinding(String filename) {
125+
var clusterRoleBinding =
126+
ReconcilerUtils.loadYaml(ClusterRoleBinding.class, this.getClass(), filename);
127+
operator.getInfrastructureKubernetesClient().resource(clusterRoleBinding).delete();
128+
}
129+
130+
private void removeClusterRole(String filename) {
131+
var clusterRole = ReconcilerUtils.loadYaml(ClusterRole.class, this.getClass(), filename);
132+
operator.getInfrastructureKubernetesClient().resource(clusterRole).delete();
133+
}
134+
}

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/infrastructureclient/InfrastructureClientTestCustomResource.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package io.javaoperatorsdk.operator.baseapi.infrastructureclient;
2+
3+
import io.fabric8.kubernetes.api.model.Namespaced;
4+
import io.fabric8.kubernetes.client.CustomResource;
5+
import io.fabric8.kubernetes.model.annotation.Group;
6+
import io.fabric8.kubernetes.model.annotation.ShortNames;
7+
import io.fabric8.kubernetes.model.annotation.Version;
8+
9+
@Group("sample.javaoperatorsdk")
10+
@Version("v1")
11+
@ShortNames("ict")
12+
public class InfrastructureClientTestCustomResource extends CustomResource<Void, Void>
13+
implements Namespaced {}

0 commit comments

Comments
 (0)