From 94bf820aa01faa0bb49e77944f61a820ac582e5f Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 12 Mar 2025 11:13:11 +0000 Subject: [PATCH 01/45] Set new SNAPSHOT version into pom files. --- bootstrapper-maven-plugin/pom.xml | 2 +- caffeine-bounded-cache-support/pom.xml | 2 +- micrometer-support/pom.xml | 2 +- operator-framework-bom/pom.xml | 2 +- operator-framework-core/pom.xml | 2 +- operator-framework-junit5/pom.xml | 2 +- operator-framework/pom.xml | 2 +- pom.xml | 2 +- sample-operators/controller-namespace-deletion/pom.xml | 2 +- sample-operators/leader-election/pom.xml | 2 +- sample-operators/mysql-schema/pom.xml | 2 +- sample-operators/pom.xml | 2 +- sample-operators/tomcat-operator/pom.xml | 2 +- sample-operators/webpage/pom.xml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bootstrapper-maven-plugin/pom.xml b/bootstrapper-maven-plugin/pom.xml index 535945760f..1ab08b975a 100644 --- a/bootstrapper-maven-plugin/pom.xml +++ b/bootstrapper-maven-plugin/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT bootstrapper diff --git a/caffeine-bounded-cache-support/pom.xml b/caffeine-bounded-cache-support/pom.xml index bd51e0af11..924164c6cb 100644 --- a/caffeine-bounded-cache-support/pom.xml +++ b/caffeine-bounded-cache-support/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT caffeine-bounded-cache-support diff --git a/micrometer-support/pom.xml b/micrometer-support/pom.xml index 67fe0d4b4c..3c568e76fd 100644 --- a/micrometer-support/pom.xml +++ b/micrometer-support/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT micrometer-support diff --git a/operator-framework-bom/pom.xml b/operator-framework-bom/pom.xml index d0bae9c140..e1cff7980d 100644 --- a/operator-framework-bom/pom.xml +++ b/operator-framework-bom/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk operator-framework-bom - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT pom Operator SDK - Bill of Materials Java SDK for implementing Kubernetes operators diff --git a/operator-framework-core/pom.xml b/operator-framework-core/pom.xml index a4dc87a3d3..cad50ebc32 100644 --- a/operator-framework-core/pom.xml +++ b/operator-framework-core/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT ../pom.xml diff --git a/operator-framework-junit5/pom.xml b/operator-framework-junit5/pom.xml index d2075303be..7e68616edf 100644 --- a/operator-framework-junit5/pom.xml +++ b/operator-framework-junit5/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT operator-framework-junit-5 diff --git a/operator-framework/pom.xml b/operator-framework/pom.xml index e2aae87401..cb49b0d39b 100644 --- a/operator-framework/pom.xml +++ b/operator-framework/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT operator-framework diff --git a/pom.xml b/pom.xml index 6e7482f024..ecd5d88da9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT pom Operator SDK for Java Java SDK for implementing Kubernetes operators diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml index 9608b44db1..ee0d5bb3d2 100644 --- a/sample-operators/controller-namespace-deletion/pom.xml +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-controller-namespace-deletion diff --git a/sample-operators/leader-election/pom.xml b/sample-operators/leader-election/pom.xml index 74aab104f1..4b1088fa3b 100644 --- a/sample-operators/leader-election/pom.xml +++ b/sample-operators/leader-election/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-leader-election diff --git a/sample-operators/mysql-schema/pom.xml b/sample-operators/mysql-schema/pom.xml index e726a2242a..92a5cb5c45 100644 --- a/sample-operators/mysql-schema/pom.xml +++ b/sample-operators/mysql-schema/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-mysql-schema-operator diff --git a/sample-operators/pom.xml b/sample-operators/pom.xml index 450a6bc153..cbe10340fc 100644 --- a/sample-operators/pom.xml +++ b/sample-operators/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-operators diff --git a/sample-operators/tomcat-operator/pom.xml b/sample-operators/tomcat-operator/pom.xml index 314e0ef96c..cd340c525d 100644 --- a/sample-operators/tomcat-operator/pom.xml +++ b/sample-operators/tomcat-operator/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-tomcat-operator diff --git a/sample-operators/webpage/pom.xml b/sample-operators/webpage/pom.xml index 2266303adc..6ae09c835d 100644 --- a/sample-operators/webpage/pom.xml +++ b/sample-operators/webpage/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.4-SNAPSHOT + 5.0.5-SNAPSHOT sample-webpage-operator From bc9d0a957021b94b7ba734074a5c6ea717561050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 13 Mar 2025 16:10:38 +0100 Subject: [PATCH 02/45] docs: skip reconcile of a DR (#2732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros Signed-off-by: Chris Laprun Co-authored-by: Chris Laprun --- docs/content/en/docs/faq/_index.md | 43 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/docs/content/en/docs/faq/_index.md b/docs/content/en/docs/faq/_index.md index 5e4975a385..67020845c1 100644 --- a/docs/content/en/docs/faq/_index.md +++ b/docs/content/en/docs/faq/_index.md @@ -3,7 +3,7 @@ title: FAQ weight: 80 --- -### Q: How can I access the events which triggered the Reconciliation? +### How can I access the events which triggered the Reconciliation? In the v1.* version events were exposed to `Reconciler` (which was called `ResourceController` then). This included events (Create, Update) of the custom resource, but also events produced by @@ -16,7 +16,7 @@ sound agreement between the developers that this is the way to go. Note that this is also consistent with Kubernetes [level based](https://cloud.redhat.com/blog/kubernetes-operators-best-practices) reconciliation approach. -### Q: Can I re-schedule a reconciliation, possibly with a specific delay? +### Can I re-schedule a reconciliation, possibly with a specific delay? Yes, this can be done using [`UpdateControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/UpdateControl.java) @@ -46,7 +46,7 @@ without an update: Although you might consider using `EventSources`, to handle reconciliation triggering in a smarter way. -### Q: How can I run an operator without cluster scope rights? +### How can I run an operator without cluster scope rights? By default, JOSDK requires access to CRs at cluster scope. You may not be granted such rights and you will see some error at startup that looks like: @@ -74,7 +74,7 @@ is `true` (`false` by default). To disable, set it to `false` at [Operator-level Operator operator = new Operator( override -> override.checkingCRDAndValidateLocalModel(false)); ``` -### Q: I'm managing an external resource that has a generated ID, where should I store that? +### I'm managing an external resource that has a generated ID, where should I store that? It is common that a non-Kubernetes or external resource is managed from a controller. Those external resources might have a generated ID, so are not simply addressable based on the spec of a custom resources. Therefore, the @@ -91,8 +91,39 @@ it is not guaranteed that during the next reconciliation you will see the fresh which do this, usually cache the updated status in memory to make sure it is present for next reconciliation. Dependent Resources feature supports the [first approach](../dependent-resources/_index.md#external-state-tracking-dependent-resources). + +### How can I skip the reconciliation of a dependent resource? -### Q: How to fix `sun.security.provider.certpath.SunCertPathBuilderException` on Rancher Desktop and k3d/k3s Kubernetes +Skipping workflow reconciliation altogether is possible with the explicit invocation feature since v5. +You can read more about this in [v5 release notes](https://javaoperatorsdk.io/blog/2025/01/06/version-5-released/#explicit-workflow-invocation). + +However, what if you want to avoid reconciling a single dependent resource based on some state? +First of all, remember that the dependent resource won't be modified if the desired state and the actual state match. +Moreover, it is generally a good practice to reconcile all your resources, JOSDK taking care of only processing the +resources which state doesn't match the desired one. +However, in some corner cases (for example, if it is expensive to compute the desired state or compare it to the actual +state), it is somtimes useful to be able to only skip the reconcilation of some resources but not all, if it is known +that they don't need to be processed based for example on the status of the custom resource. + +A common mistake is to use `ReconcilePrecondition`, if the condition does not hold it will delete the resources. +This is by design (although it's true that the name of this condition might be misleading), but not what we want in this +case. + +The way to go is to override the matcher in the dependent resource: + +```java +public Result match(R actualResource, R desired, P primary, Context

context) { + if (alreadyIsCertainState(primary.getStatus())) { + return true; + } else { + return super.match(actual, desired, primary, context); + } +} +``` + +This will make sure that the dependent resource is not updated if the primary resource is in certain state. + +### How to fix `sun.security.provider.certpath.SunCertPathBuilderException` on Rancher Desktop and k3d/k3s Kubernetes It's a common issue when using k3d and the fabric8 client tries to connect to the cluster an exception is thrown: @@ -111,4 +142,4 @@ the following dependency on the classpath: org.bouncycastle bcpkix-jdk15on -``` \ No newline at end of file +``` From 328906575b08fd9684ac694857d39e6bb6d6453c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 08:43:54 +0100 Subject: [PATCH 03/45] chore(deps): bump org.junit:junit-bom from 5.12.0 to 5.12.1 (#2733) Bumps [org.junit:junit-bom](https://github.com/junit-team/junit5) from 5.12.0 to 5.12.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.12.0...r5.12.1) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ecd5d88da9..712e6d8922 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ https://sonarcloud.io jdk - 5.12.0 + 5.12.1 7.1.0 2.0.12 2.24.3 From dfbce2caa800cf4658627f8bcc26e312a9f799db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 08:44:09 +0100 Subject: [PATCH 04/45] chore(deps): bump org.mockito:mockito-core from 5.16.0 to 5.16.1 (#2734) Bumps [org.mockito:mockito-core](https://github.com/mockito/mockito) from 5.16.0 to 5.16.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v5.16.0...v5.16.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 712e6d8922..2141e40255 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 7.1.0 2.0.12 2.24.3 - 5.16.0 + 5.16.1 3.17.0 0.21.0 1.13.0 From 0dc0f6adfcaad052ff236edb955e9fc37b3288a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 09:26:04 +0100 Subject: [PATCH 05/45] chore(deps): bump com.google.cloud.tools:jib-maven-plugin (#2735) Bumps [com.google.cloud.tools:jib-maven-plugin](https://github.com/GoogleContainerTools/jib) from 3.4.4 to 3.4.5. - [Release notes](https://github.com/GoogleContainerTools/jib/releases) - [Commits](https://github.com/GoogleContainerTools/jib/commits) --- updated-dependencies: - dependency-name: com.google.cloud.tools:jib-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2141e40255..fdcf0f87d8 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 3.0.0 3.1.4 9.0.1 - 3.4.4 + 3.4.5 2.44.3 From cc3d0cc194f5084ea5a8a28b7201274c2f8f123b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 21 Mar 2025 15:44:11 +0100 Subject: [PATCH 06/45] docs: new structure of docs (#2737) --- docs/content/en/blog/_index.md | 2 +- docs/content/en/blog/news/_index.md | 2 +- docs/content/en/blog/releases/_index.md | 2 +- docs/content/en/community/_index.md | 2 +- docs/content/en/docs/_index.md | 4 +- docs/content/en/docs/contributing/_index.md | 2 +- docs/content/en/docs/documentation/_index.md | 4 + .../architecture.md} | 3 +- .../configuration.md} | 71 +- .../_index.md | 9 + .../dependent-resources.md} | 4 +- .../workflows.md} | 2 +- .../documentation/error-handling-retries.md | 115 +++ .../content/en/docs/documentation/eventing.md | 330 +++++++ .../content/en/docs/documentation/features.md | 73 ++ .../en/docs/documentation/observability.md | 112 +++ .../en/docs/documentation/reconciler.md | 178 ++++ docs/content/en/docs/faq/_index.md | 2 +- docs/content/en/docs/features/_index.md | 853 ------------------ .../content/en/docs/getting-started/_index.md | 60 +- .../getting-started/bootstrap-and-samples.md | 38 + .../getting-started/intro-to-operators.md | 32 + .../patterns-best-practices.md} | 3 +- docs/content/en/docs/glossary/_index.md | 2 +- .../en/docs/intro-to-operators/_index.md | 16 - docs/content/en/docs/migration/_index.md | 1 + docs/content/en/docs/using-samples/_index.md | 255 ------ 27 files changed, 974 insertions(+), 1203 deletions(-) create mode 100644 docs/content/en/docs/documentation/_index.md rename docs/content/en/docs/{architecture/_index.md => documentation/architecture.md} (99%) rename docs/content/en/docs/{configuration/_index.md => documentation/configuration.md} (50%) create mode 100644 docs/content/en/docs/documentation/dependent-resource-and-workflows/_index.md rename docs/content/en/docs/{dependent-resources/_index.md => documentation/dependent-resource-and-workflows/dependent-resources.md} (99%) rename docs/content/en/docs/{workflows/_index.md => documentation/dependent-resource-and-workflows/workflows.md} (99%) create mode 100644 docs/content/en/docs/documentation/error-handling-retries.md create mode 100644 docs/content/en/docs/documentation/eventing.md create mode 100644 docs/content/en/docs/documentation/features.md create mode 100644 docs/content/en/docs/documentation/observability.md create mode 100644 docs/content/en/docs/documentation/reconciler.md delete mode 100644 docs/content/en/docs/features/_index.md create mode 100644 docs/content/en/docs/getting-started/bootstrap-and-samples.md create mode 100644 docs/content/en/docs/getting-started/intro-to-operators.md rename docs/content/en/docs/{patterns-and-best-practices/_index.md => getting-started/patterns-best-practices.md} (99%) delete mode 100644 docs/content/en/docs/intro-to-operators/_index.md delete mode 100644 docs/content/en/docs/using-samples/_index.md diff --git a/docs/content/en/blog/_index.md b/docs/content/en/blog/_index.md index c8219f7994..e792e415fe 100644 --- a/docs/content/en/blog/_index.md +++ b/docs/content/en/blog/_index.md @@ -1,6 +1,6 @@ --- title: Blog -menu: {main: {weight: 30}} +menu: {main: {weight: 2}} --- This is the **blog** section. It has two categories: News and Releases. diff --git a/docs/content/en/blog/news/_index.md b/docs/content/en/blog/news/_index.md index 646c97f954..aaf1c2adcd 100644 --- a/docs/content/en/blog/news/_index.md +++ b/docs/content/en/blog/news/_index.md @@ -1,4 +1,4 @@ --- title: Posts -weight: 20 +weight: 220 --- diff --git a/docs/content/en/blog/releases/_index.md b/docs/content/en/blog/releases/_index.md index 9143a23148..dbf2ee1729 100644 --- a/docs/content/en/blog/releases/_index.md +++ b/docs/content/en/blog/releases/_index.md @@ -1,4 +1,4 @@ --- title: Releases -weight: 20 +weight: 230 --- diff --git a/docs/content/en/community/_index.md b/docs/content/en/community/_index.md index 3f237b8a79..fa42c2d974 100644 --- a/docs/content/en/community/_index.md +++ b/docs/content/en/community/_index.md @@ -1,6 +1,6 @@ --- title: Community -menu: {main: {weight: 40}} +menu: {main: {weight: 3}} --- diff --git a/docs/content/en/docs/_index.md b/docs/content/en/docs/_index.md index 76486e22f7..7118464154 100755 --- a/docs/content/en/docs/_index.md +++ b/docs/content/en/docs/_index.md @@ -1,8 +1,8 @@ --- title: Documentation linkTitle: Docs -menu: {main: {weight: 20}} -weight: 20 +menu: {main: {weight: 1}} +weight: 1 --- diff --git a/docs/content/en/docs/contributing/_index.md b/docs/content/en/docs/contributing/_index.md index dfe6dec99c..4cea1f0e5d 100644 --- a/docs/content/en/docs/contributing/_index.md +++ b/docs/content/en/docs/contributing/_index.md @@ -1,6 +1,6 @@ --- title: Contributing To Java Operator SDK -weight: 100 +weight: 110 --- First of all, we'd like to thank you for considering contributing to the project! We really diff --git a/docs/content/en/docs/documentation/_index.md b/docs/content/en/docs/documentation/_index.md new file mode 100644 index 0000000000..cc7fc50a57 --- /dev/null +++ b/docs/content/en/docs/documentation/_index.md @@ -0,0 +1,4 @@ +--- +title: Documentation +weight: 40 +--- \ No newline at end of file diff --git a/docs/content/en/docs/architecture/_index.md b/docs/content/en/docs/documentation/architecture.md similarity index 99% rename from docs/content/en/docs/architecture/_index.md rename to docs/content/en/docs/documentation/architecture.md index a29f70c4f6..8663b64d67 100644 --- a/docs/content/en/docs/architecture/_index.md +++ b/docs/content/en/docs/documentation/architecture.md @@ -1,9 +1,8 @@ --- title: Architecture and Internals -weight: 90 +weight: 85 --- - This document gives an overview of the internal structure and components of Java Operator SDK core, in order to make it easier for developers to understand and contribute to it. This document is not intended to be a comprehensive reference, rather an introduction to the core concepts and we diff --git a/docs/content/en/docs/configuration/_index.md b/docs/content/en/docs/documentation/configuration.md similarity index 50% rename from docs/content/en/docs/configuration/_index.md rename to docs/content/en/docs/documentation/configuration.md index 11929e3358..052c0e0f19 100644 --- a/docs/content/en/docs/configuration/_index.md +++ b/docs/content/en/docs/documentation/configuration.md @@ -1,11 +1,8 @@ --- -title: Configuring JOSDK -layout: docs -permalink: /docs/configuration +title: Configurations +weight: 55 --- -# Configuration options - The Java Operator SDK (JOSDK) provides several abstractions that work great out of the box. However, while we strive to cover the most common cases with the default behavior, we also recognize that that default behavior is not always what any given user might want for their @@ -52,6 +49,68 @@ operator.register(reconciler, configOverrider -> configOverrider.withFinalizer("my-nifty-operator/finalizer").withLabelSelector("foo=bar")); ``` +## Dynamically Changing Target Namespaces + +A controller can be configured to watch a specific set of namespaces in addition of the +namespace in which it is currently deployed or the whole cluster. The framework supports +dynamically changing the list of these namespaces while the operator is running. +When a reconciler is registered, an instance of +[`RegisteredController`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ec37025a15046d8f409c77616110024bf32c3416/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/RegisteredController.java#L5) +is returned, providing access to the methods allowing users to change watched namespaces as the +operator is running. + +A typical scenario would probably involve extracting the list of target namespaces from a +`ConfigMap` or some other input but this part is out of the scope of the framework since this is +use-case specific. For example, reacting to changes to a `ConfigMap` would probably involve +registering an associated `Informer` and then calling the `changeNamespaces` method on +`RegisteredController`. + +```java + +public static void main(String[] args) { + KubernetesClient client = new DefaultKubernetesClient(); + Operator operator = new Operator(client); + RegisteredController registeredController = operator.register(new WebPageReconciler(client)); + operator.installShutdownHook(); + operator.start(); + + // call registeredController further while operator is running +} + +``` + +If watched namespaces change for a controller, it might be desirable to propagate these changes to +`InformerEventSources` associated with the controller. In order to express this, +`InformerEventSource` implementations interested in following such changes need to be +configured appropriately so that the `followControllerNamespaceChanges` method returns `true`: + +```java + +@ControllerConfiguration +public class MyReconciler implements Reconciler { + + @Override + public Map prepareEventSources( + EventSourceContext context) { + + InformerEventSource configMapES = + new InformerEventSource<>(InformerEventSourceConfiguration.from(ConfigMap.class, TestCustomResource.class) + .withNamespacesInheritedFromController(context) + .build(), context); + + return EventSourceUtils.nameEventSources(configMapES); + } + +} +``` + +As seen in the above code snippet, the informer will have the initial namespaces inherited from +controller, but also will adjust the target namespaces if it changes for the controller. + +See also +the [integration test](https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/changenamespace) +for this feature. + ## DependentResource-level configuration `DependentResource` implementations can implement the `DependentResourceConfigurator` interface @@ -61,7 +120,7 @@ provides specific support for the `KubernetesDependentResource`, which can be co `KubernetesDependentResourceConfig` instance, which is then passed to the `configureWith` method implementation. -TODO: still subject to change / uniformization +TODO ## EventSource-level configuration diff --git a/docs/content/en/docs/documentation/dependent-resource-and-workflows/_index.md b/docs/content/en/docs/documentation/dependent-resource-and-workflows/_index.md new file mode 100644 index 0000000000..9446f7ceca --- /dev/null +++ b/docs/content/en/docs/documentation/dependent-resource-and-workflows/_index.md @@ -0,0 +1,9 @@ +--- +title: Dependent resources and workflows +weight: 70 +--- + +Dependent resources and workflows are features sometimes referenced as higher +level abstractions. These two related concepts provides an abstraction +over reconciliation of a single resource (Dependent resource) and the +orchestration of such resources (Workflows). \ No newline at end of file diff --git a/docs/content/en/docs/dependent-resources/_index.md b/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md similarity index 99% rename from docs/content/en/docs/dependent-resources/_index.md rename to docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md index f79443de74..b9fcb7acf5 100644 --- a/docs/content/en/docs/dependent-resources/_index.md +++ b/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md @@ -1,6 +1,6 @@ --- -title: Dependent Resources -weight: 60 +title: Dependent resources +weight: 75 --- ## Motivations and Goals diff --git a/docs/content/en/docs/workflows/_index.md b/docs/content/en/docs/documentation/dependent-resource-and-workflows/workflows.md similarity index 99% rename from docs/content/en/docs/workflows/_index.md rename to docs/content/en/docs/documentation/dependent-resource-and-workflows/workflows.md index 620f8c5436..4b1bea6790 100644 --- a/docs/content/en/docs/workflows/_index.md +++ b/docs/content/en/docs/documentation/dependent-resource-and-workflows/workflows.md @@ -1,6 +1,6 @@ --- title: Workflows -weight: 70 +weight: 80 --- ## Overview diff --git a/docs/content/en/docs/documentation/error-handling-retries.md b/docs/content/en/docs/documentation/error-handling-retries.md new file mode 100644 index 0000000000..f37c10318b --- /dev/null +++ b/docs/content/en/docs/documentation/error-handling-retries.md @@ -0,0 +1,115 @@ +--- +title: Error handling and retries +weight: 46 +--- + +## Automatic Retries on Error + +JOSDK will schedule an automatic retry of the reconciliation whenever an exception is thrown by +your `Reconciler`. The retry is behavior is configurable but a default implementation is provided +covering most of the typical use-cases, see +[GenericRetry](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/retry/GenericRetry.java) +. + +```java + GenericRetry.defaultLimitedExponentialRetry() + .setInitialInterval(5000) + .setIntervalMultiplier(1.5D) + .setMaxAttempts(5); +``` + +You can also configure the default retry behavior using the `@GradualRetry` annotation. + +It is possible to provide a custom implementation using the `retry` field of the +`@ControllerConfiguration` annotation and specifying the class of your custom implementation. +Note that this class will need to provide an accessible no-arg constructor for automated +instantiation. Additionally, your implementation can be automatically configured from an +annotation that you can provide by having your `Retry` implementation implement the +`AnnotationConfigurable` interface, parameterized with your annotation type. See the +`GenericRetry` implementation for more details. + +Information about the current retry state is accessible from +the [Context](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/Context.java) +object. Of note, particularly interesting is the `isLastAttempt` method, which could allow your +`Reconciler` to implement a different behavior based on this status, by setting an error message +in your resource' status, for example, when attempting a last retry. + +Note, though, that reaching the retry limit won't prevent new events to be processed. New +reconciliations will happen for new events as usual. However, if an error also occurs that +would normally trigger a retry, the SDK won't schedule one at this point since the retry limit +is already reached. + +A successful execution resets the retry state. + +### Setting Error Status After Last Retry Attempt + +In order to facilitate error reporting, `Reconciler` can implement the +[ErrorStatusHandler](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ErrorStatusHandler.java) +interface: + +```java +public interface ErrorStatusHandler

{ + + ErrorStatusUpdateControl

updateErrorStatus(P resource, Context

context, Exception e); + +} +``` + +The `updateErrorStatus` method is called in case an exception is thrown from the `Reconciler`. It is +also called even if no retry policy is configured, just after the reconciler execution. +`RetryInfo.getAttemptCount()` is zero after the first reconciliation attempt, since it is not a +result of a retry (regardless of whether a retry policy is configured or not). + +`ErrorStatusUpdateControl` is used to tell the SDK what to do and how to perform the status +update on the primary resource, always performed as a status sub-resource request. Note that +this update request will also produce an event, and will result in a reconciliation if the +controller is not generation aware. + +This feature is only available for the `reconcile` method of the `Reconciler` interface, since +there should not be updates to resource that have been marked for deletion. + +Retry can be skipped in cases of unrecoverable errors: + +```java + ErrorStatusUpdateControl.patchStatus(customResource).withNoRetry(); +``` + +### Correctness and Automatic Retries + +While it is possible to deactivate automatic retries, this is not desirable, unless for very +specific reasons. Errors naturally occur, whether it be transient network errors or conflicts +when a given resource is handled by a `Reconciler` but is modified at the same time by a user in +a different process. Automatic retries handle these cases nicely and will usually result in a +successful reconciliation. + +## Retry and Rescheduling and Event Handling Common Behavior + +Retry, reschedule and standard event processing form a relatively complex system, each of these +functionalities interacting with the others. In the following, we describe the interplay of +these features: + +1. A successful execution resets a retry and the rescheduled executions which were present before + the reconciliation. However, a new rescheduling can be instructed from the reconciliation + outcome (`UpdateControl` or `DeleteControl`). + + For example, if a reconciliation had previously been re-scheduled after some amount of time, but an event triggered + the reconciliation (or cleanup) in the mean time, the scheduled execution would be automatically cancelled, i.e. + re-scheduling a reconciliation does not guarantee that one will occur exactly at that time, it simply guarantees that + one reconciliation will occur at that time at the latest, triggering one if no event from the cluster triggered one. + Of course, it's always possible to re-schedule a new reconciliation at the end of that "automatic" reconciliation. + + Similarly, if a retry was scheduled, any event from the cluster triggering a successful execution in the mean time + would cancel the scheduled retry (because there's now no point in retrying something that already succeeded) + +2. In case an exception happened, a retry is initiated. However, if an event is received + meanwhile, it will be reconciled instantly, and this execution won't count as a retry attempt. +3. If the retry limit is reached (so no more automatic retry would happen), but a new event + received, the reconciliation will still happen, but won't reset the retry, and will still be + marked as the last attempt in the retry info. The point (1) still holds, but in case of an + error, no retry will happen. + +The thing to keep in mind when it comes to retrying or rescheduling is that JOSDK tries to avoid unnecessary work. When +you reschedule an operation, you instruct JOSDK to perform that operation at the latest by the end of the rescheduling +delay. If something occurred on the cluster that triggers that particular operation (reconciliation or cleanup), then +JOSDK considers that there's no point in attempting that operation again at the end of the specified delay since there +is now no point to do so anymore. The same idea also applies to retries. \ No newline at end of file diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md new file mode 100644 index 0000000000..0ede7a21a6 --- /dev/null +++ b/docs/content/en/docs/documentation/eventing.md @@ -0,0 +1,330 @@ +--- +title: Event sources and related topics +weight: 47 +--- + +## Handling Related Events with Event Sources + +See also +this [blog post](https://csviri.medium.com/java-operator-sdk-introduction-to-event-sources-a1aab5af4b7b) +. + +Event sources are a relatively simple yet powerful and extensible concept to trigger controller +executions, usually based on changes to dependent resources. You typically need an event source +when you want your `Reconciler` to be triggered when something occurs to secondary resources +that might affect the state of your primary resource. This is needed because a given +`Reconciler` will only listen by default to events affecting the primary resource type it is +configured for. Event sources act as listen to events affecting these secondary resources so +that a reconciliation of the associated primary resource can be triggered when needed. Note that +these secondary resources need not be Kubernetes resources. Typically, when dealing with +non-Kubernetes objects or services, we can extend our operator to handle webhooks or websockets +or to react to any event coming from a service we interact with. This allows for very efficient +controller implementations because reconciliations are then only triggered when something occurs +on resources affecting our primary resources thus doing away with the need to periodically +reschedule reconciliations. + +![Event Sources architecture diagram](../assets/images/event-sources.png) + +There are few interesting points here: + +The `CustomResourceEventSource` event source is a special one, responsible for handling events +pertaining to changes affecting our primary resources. This `EventSource` is always registered +for every controller automatically by the SDK. It is important to note that events always relate +to a given primary resource. Concurrency is still handled for you, even in the presence of +`EventSource` implementations, and the SDK still guarantees that there is no concurrent execution of +the controller for any given primary resource (though, of course, concurrent/parallel executions +of events pertaining to other primary resources still occur as expected). + +### Caching and Event Sources + +Kubernetes resources are handled in a declarative manner. The same also holds true for event +sources. For example, if we define an event source to watch for changes of a Kubernetes Deployment +object using an `InformerEventSource`, we always receive the whole associated object from the +Kubernetes API. This object might be needed at any point during our reconciliation process and +it's best to retrieve it from the event source directly when possible instead of fetching it +from the Kubernetes API since the event source guarantees that it will provide the latest +version. Not only that, but many event source implementations also cache resources they handle +so that it's possible to retrieve the latest version of resources without needing to make any +calls to the Kubernetes API, thus allowing for very efficient controller implementations. + +Note after an operator starts, caches are already populated by the time the first reconciliation +is processed for the `InformerEventSource` implementation. However, this does not necessarily +hold true for all event source implementations (`PerResourceEventSource` for example). The SDK +provides methods to handle this situation elegantly, allowing you to check if an object is +cached, retrieving it from a provided supplier if not. See +related [method](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java#L146) +. + +### Registering Event Sources + +To register event sources, your `Reconciler` has to override the `prepareEventSources` and return +list of event sources to register. One way to see this in action is +to look at the +[tomcat example](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java) +(irrelevant details omitted): + +```java + +@ControllerConfiguration +public class WebappReconciler + implements Reconciler, Cleaner, EventSourceInitializer { + // ommitted code + + @Override + public Map prepareEventSources(EventSourceContext context) { + InformerEventSourceConfiguration configuration = + InformerEventSourceConfiguration.from(Tomcat.class, Tomcat.class) + .withSecondaryToPrimaryMapper(webappsMatchingTomcatName) + .withPrimaryToSecondaryMapper( + (Webapp primary) -> Set.of(new ResourceID(primary.getSpec().getTomcat(), + primary.getMetadata().getNamespace()))) + .build(); + return EventSourceInitializer + .nameEventSources(new InformerEventSource<>(configuration, context)); + } + +} +``` + +In the example above an `InformerEventSource` is configured and registered. +`InformerEventSource` is one of the bundled `EventSource` implementations that JOSDK provides to +cover common use cases. + +### Managing Relation between Primary and Secondary Resources + +Event sources let your operator know when a secondary resource has changed and that your +operator might need to reconcile this new information. However, in order to do so, the SDK needs +to somehow retrieve the primary resource associated with which ever secondary resource triggered +the event. In the `Tomcat` example above, when an event occurs on a tracked `Deployment`, the +SDK needs to be able to identify which `Tomcat` resource is impacted by that change. + +Seasoned Kubernetes users already know one way to track this parent-child kind of relationship: +using owner references. Indeed, that's how the SDK deals with this situation by default as well, +that is, if your controller properly set owner references on your secondary resources, the SDK +will be able to follow that reference back to your primary resource automatically without you +having to worry about it. + +However, owner references cannot always be used as they are restricted to operating within a +single namespace (i.e. you cannot have an owner reference to a resource in a different namespace) +and are, by essence, limited to Kubernetes resources so you're out of luck if your secondary +resources live outside of a cluster. + +This is why JOSDK provides the `SecondayToPrimaryMapper` interface so that you can provide +alternative ways for the SDK to identify which primary resource needs to be reconciled when +something occurs to your secondary resources. We even provide some of these alternatives in the +[Mappers](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/Mappers.java) +class. + +Note that, while a set of `ResourceID` is returned, this set usually consists only of one +element. It is however possible to return multiple values or even no value at all to cover some +rare corner cases. Returning an empty set means that the mapper considered the secondary +resource event as irrelevant and the SDK will thus not trigger a reconciliation of the primary +resource in that situation. + +Adding a `SecondaryToPrimaryMapper` is typically sufficient when there is a one-to-many relationship +between primary and secondary resources. The secondary resources can be mapped to its primary +owner, and this is enough information to also get these secondary resources from the `Context` +object that's passed to your `Reconciler`. + +There are however cases when this isn't sufficient and you need to provide an explicit mapping +between a primary resource and its associated secondary resources using an implementation of the +`PrimaryToSecondaryMapper` interface. This is typically needed when there are many-to-one or +many-to-many relationships between primary and secondary resources, e.g. when the primary resource +is referencing secondary resources. +See [PrimaryToSecondaryIT](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/primarytosecondary/PrimaryToSecondaryIT.java) +integration test for a sample. + +### Built-in EventSources + +There are multiple event-sources provided out of the box, the following are some more central ones: + +#### `InformerEventSource` + +[InformerEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java) +is probably the most important `EventSource` implementation to know about. When you create an +`InformerEventSource`, JOSDK will automatically create and register a `SharedIndexInformer`, a +fabric8 Kubernetes client class, that will listen for events associated with the resource type +you configured your `InformerEventSource` with. If you want to listen to Kubernetes resource +events, `InformerEventSource` is probably the only thing you need to use. It's highly +configurable so you can tune it to your needs. Take a look at +[InformerEventSourceConfiguration](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java) +and associated classes for more details but some interesting features we can mention here is the +ability to filter events so that you can only get notified for events you care about. A +particularly interesting feature of the `InformerEventSource`, as opposed to using your own +informer-based listening mechanism is that caches are particularly well optimized preventing +reconciliations from being triggered when not needed and allowing efficient operators to be written. + +#### `PerResourcePollingEventSource` + +[PerResourcePollingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java) +is used to poll external APIs, which don't support webhooks or other event notifications. It +extends the abstract +[ExternalResourceCachingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/ExternalResourceCachingEventSource.java) +to support caching. +See [MySQL Schema sample](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java) +for usage. + +#### `PollingEventSource` + +[PollingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java) +is similar to `PerResourceCachingEventSource` except that, contrary to that event source, it +doesn't poll a specific API separately per resource, but periodically and independently of +actually observed primary resources. + +#### Inbound event sources + +[SimpleInboundEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/SimpleInboundEventSource.java) +and +[CachingInboundEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/CachingInboundEventSource.java) +are used to handle incoming events from webhooks and messaging systems. + +#### `ControllerResourceEventSource` + +[ControllerResourceEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSource.java) +is a special `EventSource` implementation that you will never have to deal with directly. It is, +however, at the core of the SDK is automatically added for you: this is the main event source +that listens for changes to your primary resources and triggers your `Reconciler` when needed. +It features smart caching and is really optimized to minimize Kubernetes API accesses and avoid +triggering unduly your `Reconciler`. + +More on the philosophy of the non Kubernetes API related event source see in +issue [#729](https://github.com/java-operator-sdk/java-operator-sdk/issues/729). + + +## InformerEventSource Multi-Cluster Support + +It is possible to handle resources for remote cluster with `InformerEventSource`. To do so, +simply set a client that connects to a remote cluster: + +```java + +InformerEventSourceConfiguration configuration = + InformerEventSourceConfiguration.from(SecondaryResource.class, PrimaryResource.class) + .withKubernetesClient(remoteClusterClient) + .withSecondaryToPrimaryMapper(Mappers.fromDefaultAnnotations()); + +``` + +You will also need to specify a `SecondaryToPrimaryMapper`, since the default one +is based on owner references and won't work across cluster instances. You could, for example, use the provided implementation that relies on annotations added to the secondary resources to identify the associated primary resource. + +See related [integration test](https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/informerremotecluster). + + +## Generation Awareness and Event Filtering + +A best practice when an operator starts up is to reconcile all the associated resources because +changes might have occurred to the resources while the operator was not running. + +When this first reconciliation is done successfully, the next reconciliation is triggered if either +dependent resources are changed or the primary resource `.spec` field is changed. If other fields +like `.metadata` are changed on the primary resource, the reconciliation could be skipped. This +behavior is supported out of the box and reconciliation is by default not triggered if +changes to the primary resource do not increase the `.metadata.generation` field. +Note that changes to `.metada.generation` are automatically handled by Kubernetes. + +To turn off this feature, set `generationAwareEventProcessing` to `false` for the `Reconciler`. + + +## Max Interval Between Reconciliations + +When informers / event sources are properly set up, and the `Reconciler` implementation is +correct, no additional reconciliation triggers should be needed. However, it's +a [common practice](https://github.com/java-operator-sdk/java-operator-sdk/issues/848#issuecomment-1016419966) +to have a failsafe periodic trigger in place, just to make sure resources are nevertheless +reconciled after a certain amount of time. This functionality is in place by default, with a +rather high time interval (currently 10 hours) after which a reconciliation will be +automatically triggered even in the absence of other events. See how to override this using the +standard annotation: + +```java +@ControllerConfiguration(maxReconciliationInterval = @MaxReconciliationInterval( + interval = 50, + timeUnit = TimeUnit.MILLISECONDS)) +public class MyReconciler implements Reconciler {} +``` + +The event is not propagated at a fixed rate, rather it's scheduled after each reconciliation. So the +next reconciliation will occur at most within the specified interval after the last reconciliation. + +This feature can be turned off by setting `maxReconciliationInterval` +to [`Constants.NO_MAX_RECONCILIATION_INTERVAL`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java#L20-L20) +or any non-positive number. + +The automatic retries are not affected by this feature so a reconciliation will be re-triggered +on error, according to the specified retry policy, regardless of this maximum interval setting. + +## Rate Limiting + +It is possible to rate limit reconciliation on a per-resource basis. The rate limit also takes +precedence over retry/re-schedule configurations: for example, even if a retry was scheduled for +the next second but this request would make the resource go over its rate limit, the next +reconciliation will be postponed according to the rate limiting rules. Note that the +reconciliation is never cancelled, it will just be executed as early as possible based on rate +limitations. + +Rate limiting is by default turned **off**, since correct configuration depends on the reconciler +implementation, in particular, on how long a typical reconciliation takes. +(The parallelism of reconciliation itself can be +limited [`ConfigurationService`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L120-L120) +by configuring the `ExecutorService` appropriately.) + +A default rate limiter implementation is provided, see: +[`PeriodRateLimiter`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/rate/PeriodRateLimiter.java#L14-L14) +. +Users can override it by implementing their own +[`RateLimiter`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/rate/RateLimiter.java) +and specifying this custom implementation using the `rateLimiter` field of the +`@ControllerConfiguration` annotation. Similarly to the `Retry` implementations, +`RateLimiter` implementations must provide an accessible, no-arg constructor for instantiation +purposes and can further be automatically configured from your own, provided annotation provided +your `RateLimiter` implementation also implements the `AnnotationConfigurable` interface, +parameterized by your custom annotation type. + +To configure the default rate limiter use the `@RateLimited` annotation on your +`Reconciler` class. The following configuration limits each resource to reconcile at most twice +within a 3 second interval: + +```java + +@RateLimited(maxReconciliations = 2, within = 3, unit = TimeUnit.SECONDS) +@ControllerConfiguration +public class MyReconciler implements Reconciler { + +} +``` + +Thus, if a given resource was reconciled twice in one second, no further reconciliation for this +resource will happen before two seconds have elapsed. Note that, since rate is limited on a +per-resource basis, other resources can still be reconciled at the same time, as long, of course, +that they stay within their own rate limits. + +## Optimizing Caches + +One of the ideas around the operator pattern is that all the relevant resources are cached, thus reconciliation is +usually very fast (especially if no resources are updated in the process) since the operator is then mostly working with +in-memory state. However for large clusters, caching huge amount of primary and secondary resources might consume lots +of memory. JOSDK provides ways to mitigate this issue and optimize the memory usage of controllers. While these features +are working and tested, we need feedback from real production usage. + +### Bounded Caches for Informers + +Limiting caches for informers - thus for Kubernetes resources - is supported by ensuring that resources are in the cache +for a limited time, via a cache eviction of least recently used resources. This means that when resources are created +and frequently reconciled, they stay "hot" in the cache. However, if, over time, a given resource "cools" down, i.e. it +becomes less and less used to the point that it might not be reconciled anymore, it will eventually get evicted from the +cache to free up memory. If such an evicted resource were to become reconciled again, the bounded cache implementation +would then fetch it from the API server and the "hot/cold" cycle would start anew. + +Since all resources need to be reconciled when a controller start, it is not practical to set a maximal cache size as +it's desirable that all resources be cached as soon as possible to make the initial reconciliation process on start as +fast and efficient as possible, avoiding undue load on the API server. It's therefore more interesting to gradually +evict cold resources than try to limit cache sizes. + +See usage of the related implementation using [Caffeine](https://github.com/ben-manes/caffeine) cache in integration +tests +for [primary resources](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/sample/AbstractTestReconciler.java#L29-L29). + +See +also [CaffeineBoundedItemStores](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/main/java/io/javaoperatorsdk/operator/processing/event/source/cache/CaffeineBoundedItemStores.java) +for more details. \ No newline at end of file diff --git a/docs/content/en/docs/documentation/features.md b/docs/content/en/docs/documentation/features.md new file mode 100644 index 0000000000..c39dece4e3 --- /dev/null +++ b/docs/content/en/docs/documentation/features.md @@ -0,0 +1,73 @@ +--- +title: Other Features +weight: 57 +--- + +The Java Operator SDK (JOSDK) is a high level framework and related tooling aimed at +facilitating the implementation of Kubernetes operators. The features are by default following +the best practices in an opinionated way. However, feature flags and other configuration options +are provided to fine tune or turn off these features. + +## Support for Well Known (non-custom) Kubernetes Resources + +A Controller can be registered for a non-custom resource, so well known Kubernetes resources like ( +`Ingress`, `Deployment`,...). + +See +the [integration test](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/deployment) +for reconciling deployments. + +```java +public class DeploymentReconciler + implements Reconciler, TestExecutionInfoProvider { + + @Override + public UpdateControl reconcile( + Deployment resource, Context context) { + // omitted code + } +} +``` + +## Leader Election + +Operators are generally deployed with a single running or active instance. However, it is +possible to deploy multiple instances in such a way that only one, called the "leader", processes the +events. This is achieved via a mechanism called "leader election". While all the instances are +running, and even start their event sources to populate the caches, only the leader will process +the events. This means that should the leader change for any reason, for example because it +crashed, the other instances are already warmed up and ready to pick up where the previous +leader left off should one of them become elected leader. + +See sample configuration in +the [E2E test](https://github.com/java-operator-sdk/java-operator-sdk/blob/8865302ac0346ee31f2d7b348997ec2913d5922b/sample-operators/leader-election/src/main/java/io/javaoperatorsdk/operator/sample/LeaderElectionTestOperator.java#L21-L23) +. + +## Automatic Generation of CRDs + +Note that this feature is provided by the +[Fabric8 Kubernetes Client](https://github.com/fabric8io/kubernetes-client), not JOSDK itself. + +To automatically generate CRD manifests from your annotated Custom Resource classes, you only need +to add the following dependencies to your project: + +```xml + + + io.fabric8 + crd-generator-apt + provided + +``` + +The CRD will be generated in `target/classes/META-INF/fabric8` (or +in `target/test-classes/META-INF/fabric8`, if you use the `test` scope) with the CRD name +suffixed by the generated spec version. For example, a CR using the `java-operator-sdk.io` group +with a `mycrs` plural form will result in 2 files: + +- `mycrs.java-operator-sdk.io-v1.yml` +- `mycrs.java-operator-sdk.io-v1beta1.yml` + +**NOTE:** +> Quarkus users using the `quarkus-operator-sdk` extension do not need to add any extra dependency +> to get their CRD generated as this is handled by the extension itself. diff --git a/docs/content/en/docs/documentation/observability.md b/docs/content/en/docs/documentation/observability.md new file mode 100644 index 0000000000..27a68086d5 --- /dev/null +++ b/docs/content/en/docs/documentation/observability.md @@ -0,0 +1,112 @@ +--- +title: Observability +weight: 55 +--- + +## Runtime Info + +[RuntimeInfo](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/RuntimeInfo.java#L16-L16) +is used mainly to check the actual health of event sources. Based on this information it is easy to implement custom +liveness probes. + +[stopOnInformerErrorDuringStartup](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L168-L168) +setting, where this flag usually needs to be set to false, in order to control the exact liveness properties. + +See also an example implementation in the +[WebPage sample](https://github.com/java-operator-sdk/java-operator-sdk/blob/3e2e7c4c834ef1c409d636156b988125744ca911/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java#L38-L43) + +## Contextual Info for Logging with MDC + +Logging is enhanced with additional contextual information using +[MDC](http://www.slf4j.org/manual.html#mdc). The following attributes are available in most +parts of reconciliation logic and during the execution of the controller: + +| MDC Key | Value added from primary resource | +|:---------------------------|:----------------------------------| +| `resource.apiVersion` | `.apiVersion` | +| `resource.kind` | `.kind` | +| `resource.name` | `.metadata.name` | +| `resource.namespace` | `.metadata.namespace` | +| `resource.resourceVersion` | `.metadata.resourceVersion` | +| `resource.generation` | `.metadata.generation` | +| `resource.uid` | `.metadata.uid` | + +For more information about MDC see this [link](https://www.baeldung.com/mdc-in-log4j-2-logback). + +## Metrics + +JOSDK provides built-in support for metrics reporting on what is happening with your reconcilers in the form of +the `Metrics` interface which can be implemented to connect to your metrics provider of choice, JOSDK calling the +methods as it goes about reconciling resources. By default, a no-operation implementation is provided thus providing a +no-cost sane default. A [micrometer](https://micrometer.io)-based implementation is also provided. + +You can use a different implementation by overriding the default one provided by the default `ConfigurationService`, as +follows: + +```java +Metrics metrics; // initialize your metrics implementation +Operator operator = new Operator(client, o -> o.withMetrics(metrics)); +``` + +### Micrometer implementation + +The micrometer implementation is typically created using one of the provided factory methods which, depending on which +is used, will return either a ready to use instance or a builder allowing users to customized how the implementation +behaves, in particular when it comes to the granularity of collected metrics. It is, for example, possible to collect +metrics on a per-resource basis via tags that are associated with meters. This is the default, historical behavior but +this will change in a future version of JOSDK because this dramatically increases the cardinality of metrics, which +could lead to performance issues. + +To create a `MicrometerMetrics` implementation that behaves how it has historically behaved, you can just create an +instance via: + +```java +MeterRegistry registry; // initialize your registry implementation +Metrics metrics = new MicrometerMetrics(registry); +``` + +Note, however, that this constructor is deprecated and we encourage you to use the factory methods instead, which either +return a fully pre-configured instance or a builder object that will allow you to configure more easily how the instance +will behave. You can, for example, configure whether or not the implementation should collect metrics on a per-resource +basis, whether or not associated meters should be removed when a resource is deleted and how the clean-up is performed. +See the relevant classes documentation for more details. + +For example, the following will create a `MicrometerMetrics` instance configured to collect metrics on a per-resource +basis, deleting the associated meters after 5 seconds when a resource is deleted, using up to 2 threads to do so. + +```java +MicrometerMetrics.newPerResourceCollectingMicrometerMetricsBuilder(registry) + .withCleanUpDelayInSeconds(5) + .withCleaningThreadNumber(2) + .build(); +``` + +### Operator SDK metrics + +The micrometer implementation records the following metrics: + +| Meter name | Type | Tag names | Description | +|-------------------------------------------------------------|----------------|-------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| operator.sdk.reconciliations.executions.`` | gauge | group, version, kind | Number of executions of the named reconciler | +| operator.sdk.reconciliations.queue.size.`` | gauge | group, version, kind | How many resources are queued to get reconciled by named reconciler | +| operator.sdk.``.size | gauge map size | | Gauge tracking the size of a specified map (currently unused but could be used to monitor caches size) | +| operator.sdk.events.received | counter | ``, event, action | Number of received Kubernetes events | +| operator.sdk.events.delete | counter | `` | Number of received Kubernetes delete events | +| operator.sdk.reconciliations.started | counter | ``, reconciliations.retries.last, reconciliations.retries.number | Number of started reconciliations per resource type | +| operator.sdk.reconciliations.failed | counter | ``, exception | Number of failed reconciliations per resource type | +| operator.sdk.reconciliations.success | counter | `` | Number of successful reconciliations per resource type | +| operator.sdk.controllers.execution.reconcile | timer | ``, controller | Time taken for reconciliations per controller | +| operator.sdk.controllers.execution.cleanup | timer | ``, controller | Time taken for cleanups per controller | +| operator.sdk.controllers.execution.reconcile.success | counter | controller, type | Number of successful reconciliations per controller | +| operator.sdk.controllers.execution.reconcile.failure | counter | controller, exception | Number of failed reconciliations per controller | +| operator.sdk.controllers.execution.cleanup.success | counter | controller, type | Number of successful cleanups per controller | +| operator.sdk.controllers.execution.cleanup.failure | counter | controller, exception | Number of failed cleanups per controller | + +As you can see all the recorded metrics start with the `operator.sdk` prefix. ``, in the table above, +refers to resource-specific metadata and depends on the considered metric and how the implementation is configured and +could be summed up as follows: `group?, version, kind, [name, namespace?], scope` where the tags in square +brackets (`[]`) won't be present when per-resource collection is disabled and tags followed by a question mark are +omitted if the associated value is empty. Of note, when in the context of controllers' execution metrics, these tag +names are prefixed with `resource.`. This prefix might be removed in a future version for greater consistency. + + diff --git a/docs/content/en/docs/documentation/reconciler.md b/docs/content/en/docs/documentation/reconciler.md new file mode 100644 index 0000000000..330bc15ac7 --- /dev/null +++ b/docs/content/en/docs/documentation/reconciler.md @@ -0,0 +1,178 @@ +--- +title: Implementing a reconciler +weight: 45 +--- + +## Reconciliation Execution in a Nutshell + +Reconciliation execution is always triggered by an event. Events typically come from a +primary resource, most of the time a custom resource, triggered by changes made to that resource +on the server (e.g. a resource is created, updated or deleted). Reconciler implementations are +associated with a given resource type and listens for such events from the Kubernetes API server +so that they can appropriately react to them. It is, however, possible for secondary sources to +trigger the reconciliation process. This usually occurs via +the [event source](#handling-related-events-with-event-sources) mechanism. + +When an event is received reconciliation is executed, unless a reconciliation is already +underway for this particular resource. In other words, the framework guarantees that no +concurrent reconciliation happens for any given resource. + +Once the reconciliation is done, the framework checks if: + +- an exception was thrown during execution and if yes schedules a retry. +- new events were received during the controller execution, if yes schedule a new reconciliation. +- the reconcilier instructed the SDK to re-schedule a reconciliation at a later date, if yes + schedules a timer event with the specified delay. +- none of the above, the reconciliation is finished. + +In summary, the core of the SDK is implemented as an eventing system, where events trigger +reconciliation requests. + +## Implementing a [`Reconciler`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Reconciler.java) and/or [`Cleaner`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) + +The lifecycle of a Kubernetes resource can be clearly separated into two phases from the +perspective of an operator depending on whether a resource is created or updated, or on the +other hand if it is marked for deletion. + +This separation-related logic is automatically handled by the framework. The framework will always +call the `reconcile` method, unless the custom resource is +[marked from deletion](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work) +. On the other, if the resource is marked from deletion and if the `Reconciler` implements the +`Cleaner` interface, only the `cleanup` method will be called. Implementing the `Cleaner` +interface allows developers to let the SDK know that they are interested in cleaning related +state (e.g. out-of-cluster resources). The SDK will therefore automatically add a finalizer +associated with your `Reconciler` so that the Kubernetes server doesn't delete your resources +before your `Reconciler` gets a chance to clean things up. +See [Finalizer support](#finalizer-support) for more details. + +### Using `UpdateControl` and `DeleteControl` + +These two classes are used to control the outcome or the desired behaviour after the reconciliation. + +The [`UpdateControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/UpdateControl.java) +can instruct the framework to update the status sub-resource of the resource +and/or re-schedule a reconciliation with a desired time delay: + +```java + @Override + public UpdateControl reconcile( + EventSourceTestCustomResource resource, Context context) { + // omitted code + + return UpdateControl.patchStatus(resource).rescheduleAfter(10, TimeUnit.SECONDS); + } +``` + +without an update: + +```java + @Override + public UpdateControl reconcile( + EventSourceTestCustomResource resource, Context context) { + // omitted code + + return UpdateControl.noUpdate().rescheduleAfter(10, TimeUnit.SECONDS); + } +``` + +Note, though, that using `EventSources` should be preferred to rescheduling since the +reconciliation will then be triggered only when needed instead than on a timely basis. + +Those are the typical use cases of resource updates, however in some cases there it can happen that +the controller wants to update the resource itself (for example to add annotations) or not perform +any updates, which is also supported. + +It is also possible to update both the status and the resource with the `patchResourceAndStatus` method. In this case, +the resource is updated first followed by the status, using two separate requests to the Kubernetes API. + +From v5 `UpdateControl` only supports patching the resources, by default +using [Server Side Apply (SSA)](https://kubernetes.io/docs/reference/using-api/server-side-apply/). +It is important to understand how SSA works in Kubernetes. Mainly, resources applied using SSA +should contain only the fields identifying the resource and those the user is interested in (a 'fully specified intent' +in Kubernetes parlance), thus usually using a resource created from scratch, see +[sample](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourcewithssa). +To contrast, see the same sample, this time [without SSA](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourceandstatusnossa/PatchResourceAndStatusNoSSAReconciler.java). + +Non-SSA based patch is still supported. +You can control whether or not to use SSA +using [`ConfigurationServcice.useSSAToPatchPrimaryResource()`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L385-L385) +and the related `ConfigurationServiceOverrider.withUseSSAToPatchPrimaryResource` method. +Related integration test can be +found [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourceandstatusnossa). + +Handling resources directly using the client, instead of delegating these updates operations to JOSDK by returning +an `UpdateControl` at the end of your reconciliation, should work appropriately. However, we do recommend to +use `UpdateControl` instead since JOSDK makes sure that the operations are handled properly, since there are subtleties +to be aware of. For example, if you are using a finalizer, JOSDK makes sure to include it in your fully specified intent +so that it is not unintentionally removed from the resource (which would happen if you omit it, since your controller is +the designated manager for that field and Kubernetes interprets the finalizer being gone from the specified intent as a +request for removal). + +[`DeleteControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DeleteControl.java) +typically instructs the framework to remove the finalizer after the dependent +resource are cleaned up in `cleanup` implementation. + +```java + +public DeleteControl cleanup(MyCustomResource customResource,Context context){ + // omitted code + + return DeleteControl.defaultDelete(); + } + +``` + +However, it is possible to instruct the SDK to not remove the finalizer, this allows to clean up +the resources in a more asynchronous way, mostly for cases when there is a long waiting period +after a delete operation is initiated. Note that in this case you might want to either schedule +a timed event to make sure `cleanup` is executed again or use event sources to get notified +about the state changes of the deleted resource. + +### Finalizer Support + +[Kubernetes finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) +make sure that your `Reconciler` gets a chance to act before a resource is actually deleted +after it's been marked for deletion. Without finalizers, the resource would be deleted directly +by the Kubernetes server. + +Depending on your use case, you might or might not need to use finalizers. In particular, if +your operator doesn't need to clean any state that would not be automatically managed by the +Kubernetes cluster (e.g. external resources), you might not need to use finalizers. You should +use the +Kubernetes [garbage collection](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#owners-dependents) +mechanism as much as possible by setting owner references for your secondary resources so that +the cluster can automatically deleted them for you whenever the associated primary resource is +deleted. Note that setting owner references is the responsibility of the `Reconciler` +implementation, though [dependent resources](https://javaoperatorsdk.io/docs/dependent-resources) +make that process easier. + +If you do need to clean such state, you need to use finalizers so that their +presence will prevent the Kubernetes server from deleting the resource before your operator is +ready to allow it. This allows for clean up to still occur even if your operator was down when +the resources was "deleted" by a user. + +JOSDK makes cleaning resources in this fashion easier by taking care of managing finalizers +automatically for you when needed. The only thing you need to do is let the SDK know that your +operator is interested in cleaning state associated with your primary resources by having it +implement +the [`Cleaner

`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) +interface. If your `Reconciler` doesn't implement the `Cleaner` interface, the SDK will consider +that you don't need to perform any clean-up when resources are deleted and will therefore not +activate finalizer support. In other words, finalizer support is added only if your `Reconciler` +implements the `Cleaner` interface. + +Finalizers are automatically added by the framework as the first step, thus after a resource +is created, but before the first reconciliation. The finalizer is added via a separate +Kubernetes API call. As a result of this update, the finalizer will then be present on the +resource. The reconciliation can then proceed as normal. + +The finalizer that is automatically added will be also removed after the `cleanup` is executed on +the reconciler. This behavior is customizable as explained +[above](#using-updatecontrol-and-deletecontrol) when we addressed the use of +`DeleteControl`. + +You can specify the name of the finalizer to use for your `Reconciler` using the +[`@ControllerConfiguration`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java) +annotation. If you do not specify a finalizer name, one will be automatically generated for you. + +From v5 by default finalizer is added using Served Side Apply. See also UpdateControl in docs. \ No newline at end of file diff --git a/docs/content/en/docs/faq/_index.md b/docs/content/en/docs/faq/_index.md index 67020845c1..9308ce4cfa 100644 --- a/docs/content/en/docs/faq/_index.md +++ b/docs/content/en/docs/faq/_index.md @@ -1,6 +1,6 @@ --- title: FAQ -weight: 80 +weight: 90 --- ### How can I access the events which triggered the Reconciliation? diff --git a/docs/content/en/docs/features/_index.md b/docs/content/en/docs/features/_index.md deleted file mode 100644 index de49abe2b5..0000000000 --- a/docs/content/en/docs/features/_index.md +++ /dev/null @@ -1,853 +0,0 @@ ---- -title: Features -weight: 50 ---- - -# Features - -The Java Operator SDK (JOSDK) is a high level framework and related tooling aimed at -facilitating the implementation of Kubernetes operators. The features are by default following -the best practices in an opinionated way. However, feature flags and other configuration options -are provided to fine tune or turn off these features. - -## Reconciliation Execution in a Nutshell - -Reconciliation execution is always triggered by an event. Events typically come from a -primary resource, most of the time a custom resource, triggered by changes made to that resource -on the server (e.g. a resource is created, updated or deleted). Reconciler implementations are -associated with a given resource type and listens for such events from the Kubernetes API server -so that they can appropriately react to them. It is, however, possible for secondary sources to -trigger the reconciliation process. This usually occurs via -the [event source](#handling-related-events-with-event-sources) mechanism. - -When an event is received reconciliation is executed, unless a reconciliation is already -underway for this particular resource. In other words, the framework guarantees that no -concurrent reconciliation happens for any given resource. - -Once the reconciliation is done, the framework checks if: - -- an exception was thrown during execution and if yes schedules a retry. -- new events were received during the controller execution, if yes schedule a new reconciliation. -- the reconcilier instructed the SDK to re-schedule a reconciliation at a later date, if yes - schedules a timer event with the specified delay. -- none of the above, the reconciliation is finished. - -In summary, the core of the SDK is implemented as an eventing system, where events trigger -reconciliation requests. - -## Implementing a [`Reconciler`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Reconciler.java) and/or [`Cleaner`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) - -The lifecycle of a Kubernetes resource can be clearly separated into two phases from the -perspective of an operator depending on whether a resource is created or updated, or on the -other hand if it is marked for deletion. - -This separation-related logic is automatically handled by the framework. The framework will always -call the `reconcile` method, unless the custom resource is -[marked from deletion](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work) -. On the other, if the resource is marked from deletion and if the `Reconciler` implements the -`Cleaner` interface, only the `cleanup` method will be called. Implementing the `Cleaner` -interface allows developers to let the SDK know that they are interested in cleaning related -state (e.g. out-of-cluster resources). The SDK will therefore automatically add a finalizer -associated with your `Reconciler` so that the Kubernetes server doesn't delete your resources -before your `Reconciler` gets a chance to clean things up. -See [Finalizer support](#finalizer-support) for more details. - -### Using `UpdateControl` and `DeleteControl` - -These two classes are used to control the outcome or the desired behaviour after the reconciliation. - -The [`UpdateControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/UpdateControl.java) -can instruct the framework to update the status sub-resource of the resource -and/or re-schedule a reconciliation with a desired time delay: - -```java - @Override - public UpdateControl reconcile( - EventSourceTestCustomResource resource, Context context) { - // omitted code - - return UpdateControl.patchStatus(resource).rescheduleAfter(10, TimeUnit.SECONDS); - } -``` - -without an update: - -```java - @Override - public UpdateControl reconcile( - EventSourceTestCustomResource resource, Context context) { - // omitted code - - return UpdateControl.noUpdate().rescheduleAfter(10, TimeUnit.SECONDS); - } -``` - -Note, though, that using `EventSources` should be preferred to rescheduling since the -reconciliation will then be triggered only when needed instead than on a timely basis. - -Those are the typical use cases of resource updates, however in some cases there it can happen that -the controller wants to update the resource itself (for example to add annotations) or not perform -any updates, which is also supported. - -It is also possible to update both the status and the resource with the `patchResourceAndStatus` method. In this case, -the resource is updated first followed by the status, using two separate requests to the Kubernetes API. - -From v5 `UpdateControl` only supports patching the resources, by default -using [Server Side Apply (SSA)](https://kubernetes.io/docs/reference/using-api/server-side-apply/). -It is important to understand how SSA works in Kubernetes. Mainly, resources applied using SSA -should contain only the fields identifying the resource and those the user is interested in (a 'fully specified intent' -in Kubernetes parlance), thus usually using a resource created from scratch, see -[sample](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourcewithssa). -To contrast, see the same sample, this time [without SSA](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourceandstatusnossa/PatchResourceAndStatusNoSSAReconciler.java). - -Non-SSA based patch is still supported. -You can control whether or not to use SSA -using [`ConfigurationServcice.useSSAToPatchPrimaryResource()`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L385-L385) -and the related `ConfigurationServiceOverrider.withUseSSAToPatchPrimaryResource` method. -Related integration test can be -found [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/patchresourceandstatusnossa). - -Handling resources directly using the client, instead of delegating these updates operations to JOSDK by returning -an `UpdateControl` at the end of your reconciliation, should work appropriately. However, we do recommend to -use `UpdateControl` instead since JOSDK makes sure that the operations are handled properly, since there are subtleties -to be aware of. For example, if you are using a finalizer, JOSDK makes sure to include it in your fully specified intent -so that it is not unintentionally removed from the resource (which would happen if you omit it, since your controller is -the designated manager for that field and Kubernetes interprets the finalizer being gone from the specified intent as a -request for removal). - -[`DeleteControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DeleteControl.java) -typically instructs the framework to remove the finalizer after the dependent -resource are cleaned up in `cleanup` implementation. - -```java - -public DeleteControl cleanup(MyCustomResource customResource,Context context){ - // omitted code - - return DeleteControl.defaultDelete(); - } - -``` - -However, it is possible to instruct the SDK to not remove the finalizer, this allows to clean up -the resources in a more asynchronous way, mostly for cases when there is a long waiting period -after a delete operation is initiated. Note that in this case you might want to either schedule -a timed event to make sure `cleanup` is executed again or use event sources to get notified -about the state changes of the deleted resource. - -### Finalizer Support - -[Kubernetes finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) -make sure that your `Reconciler` gets a chance to act before a resource is actually deleted -after it's been marked for deletion. Without finalizers, the resource would be deleted directly -by the Kubernetes server. - -Depending on your use case, you might or might not need to use finalizers. In particular, if -your operator doesn't need to clean any state that would not be automatically managed by the -Kubernetes cluster (e.g. external resources), you might not need to use finalizers. You should -use the -Kubernetes [garbage collection](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#owners-dependents) -mechanism as much as possible by setting owner references for your secondary resources so that -the cluster can automatically deleted them for you whenever the associated primary resource is -deleted. Note that setting owner references is the responsibility of the `Reconciler` -implementation, though [dependent resources](https://javaoperatorsdk.io/docs/dependent-resources) -make that process easier. - -If you do need to clean such state, you need to use finalizers so that their -presence will prevent the Kubernetes server from deleting the resource before your operator is -ready to allow it. This allows for clean up to still occur even if your operator was down when -the resources was "deleted" by a user. - -JOSDK makes cleaning resources in this fashion easier by taking care of managing finalizers -automatically for you when needed. The only thing you need to do is let the SDK know that your -operator is interested in cleaning state associated with your primary resources by having it -implement -the [`Cleaner

`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) -interface. If your `Reconciler` doesn't implement the `Cleaner` interface, the SDK will consider -that you don't need to perform any clean-up when resources are deleted and will therefore not -activate finalizer support. In other words, finalizer support is added only if your `Reconciler` -implements the `Cleaner` interface. - -Finalizers are automatically added by the framework as the first step, thus after a resource -is created, but before the first reconciliation. The finalizer is added via a separate -Kubernetes API call. As a result of this update, the finalizer will then be present on the -resource. The reconciliation can then proceed as normal. - -The finalizer that is automatically added will be also removed after the `cleanup` is executed on -the reconciler. This behavior is customizable as explained -[above](#using-updatecontrol-and-deletecontrol) when we addressed the use of -`DeleteControl`. - -You can specify the name of the finalizer to use for your `Reconciler` using the -[`@ControllerConfiguration`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java) -annotation. If you do not specify a finalizer name, one will be automatically generated for you. - -From v5 by default finalizer is added using Served Side Apply. See also UpdateControl in docs. - -## Generation Awareness and Event Filtering - -A best practice when an operator starts up is to reconcile all the associated resources because -changes might have occurred to the resources while the operator was not running. - -When this first reconciliation is done successfully, the next reconciliation is triggered if either -dependent resources are changed or the primary resource `.spec` field is changed. If other fields -like `.metadata` are changed on the primary resource, the reconciliation could be skipped. This -behavior is supported out of the box and reconciliation is by default not triggered if -changes to the primary resource do not increase the `.metadata.generation` field. -Note that changes to `.metada.generation` are automatically handled by Kubernetes. - -To turn off this feature, set `generationAwareEventProcessing` to `false` for the `Reconciler`. - -## Support for Well Known (non-custom) Kubernetes Resources - -A Controller can be registered for a non-custom resource, so well known Kubernetes resources like ( -`Ingress`, `Deployment`,...). - -See -the [integration test](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/deployment) -for reconciling deployments. - -```java -public class DeploymentReconciler - implements Reconciler, TestExecutionInfoProvider { - - @Override - public UpdateControl reconcile( - Deployment resource, Context context) { - // omitted code - } -} -``` - -## Max Interval Between Reconciliations - -When informers / event sources are properly set up, and the `Reconciler` implementation is -correct, no additional reconciliation triggers should be needed. However, it's -a [common practice](https://github.com/java-operator-sdk/java-operator-sdk/issues/848#issuecomment-1016419966) -to have a failsafe periodic trigger in place, just to make sure resources are nevertheless -reconciled after a certain amount of time. This functionality is in place by default, with a -rather high time interval (currently 10 hours) after which a reconciliation will be -automatically triggered even in the absence of other events. See how to override this using the -standard annotation: - -```java -@ControllerConfiguration(maxReconciliationInterval = @MaxReconciliationInterval( - interval = 50, - timeUnit = TimeUnit.MILLISECONDS)) -public class MyReconciler implements Reconciler {} -``` - -The event is not propagated at a fixed rate, rather it's scheduled after each reconciliation. So the -next reconciliation will occur at most within the specified interval after the last reconciliation. - -This feature can be turned off by setting `maxReconciliationInterval` -to [`Constants.NO_MAX_RECONCILIATION_INTERVAL`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java#L20-L20) -or any non-positive number. - -The automatic retries are not affected by this feature so a reconciliation will be re-triggered -on error, according to the specified retry policy, regardless of this maximum interval setting. - -## Automatic Retries on Error - -JOSDK will schedule an automatic retry of the reconciliation whenever an exception is thrown by -your `Reconciler`. The retry is behavior is configurable but a default implementation is provided -covering most of the typical use-cases, see -[GenericRetry](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/retry/GenericRetry.java) -. - -```java - GenericRetry.defaultLimitedExponentialRetry() - .setInitialInterval(5000) - .setIntervalMultiplier(1.5D) - .setMaxAttempts(5); -``` - -You can also configure the default retry behavior using the `@GradualRetry` annotation. - -It is possible to provide a custom implementation using the `retry` field of the -`@ControllerConfiguration` annotation and specifying the class of your custom implementation. -Note that this class will need to provide an accessible no-arg constructor for automated -instantiation. Additionally, your implementation can be automatically configured from an -annotation that you can provide by having your `Retry` implementation implement the -`AnnotationConfigurable` interface, parameterized with your annotation type. See the -`GenericRetry` implementation for more details. - -Information about the current retry state is accessible from -the [Context](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/Context.java) -object. Of note, particularly interesting is the `isLastAttempt` method, which could allow your -`Reconciler` to implement a different behavior based on this status, by setting an error message -in your resource' status, for example, when attempting a last retry. - -Note, though, that reaching the retry limit won't prevent new events to be processed. New -reconciliations will happen for new events as usual. However, if an error also occurs that -would normally trigger a retry, the SDK won't schedule one at this point since the retry limit -is already reached. - -A successful execution resets the retry state. - -### Setting Error Status After Last Retry Attempt - -In order to facilitate error reporting, `Reconciler` can implement the -[ErrorStatusHandler](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ErrorStatusHandler.java) -interface: - -```java -public interface ErrorStatusHandler

{ - - ErrorStatusUpdateControl

updateErrorStatus(P resource, Context

context, Exception e); - -} -``` - -The `updateErrorStatus` method is called in case an exception is thrown from the `Reconciler`. It is -also called even if no retry policy is configured, just after the reconciler execution. -`RetryInfo.getAttemptCount()` is zero after the first reconciliation attempt, since it is not a -result of a retry (regardless of whether a retry policy is configured or not). - -`ErrorStatusUpdateControl` is used to tell the SDK what to do and how to perform the status -update on the primary resource, always performed as a status sub-resource request. Note that -this update request will also produce an event, and will result in a reconciliation if the -controller is not generation aware. - -This feature is only available for the `reconcile` method of the `Reconciler` interface, since -there should not be updates to resource that have been marked for deletion. - -Retry can be skipped in cases of unrecoverable errors: - -```java - ErrorStatusUpdateControl.patchStatus(customResource).withNoRetry(); -``` - -### Correctness and Automatic Retries - -While it is possible to deactivate automatic retries, this is not desirable, unless for very -specific reasons. Errors naturally occur, whether it be transient network errors or conflicts -when a given resource is handled by a `Reconciler` but is modified at the same time by a user in -a different process. Automatic retries handle these cases nicely and will usually result in a -successful reconciliation. - -## Retry and Rescheduling and Event Handling Common Behavior - -Retry, reschedule and standard event processing form a relatively complex system, each of these -functionalities interacting with the others. In the following, we describe the interplay of -these features: - -1. A successful execution resets a retry and the rescheduled executions which were present before - the reconciliation. However, a new rescheduling can be instructed from the reconciliation - outcome (`UpdateControl` or `DeleteControl`). - - For example, if a reconciliation had previously been re-scheduled after some amount of time, but an event triggered - the reconciliation (or cleanup) in the mean time, the scheduled execution would be automatically cancelled, i.e. - re-scheduling a reconciliation does not guarantee that one will occur exactly at that time, it simply guarantees that - one reconciliation will occur at that time at the latest, triggering one if no event from the cluster triggered one. - Of course, it's always possible to re-schedule a new reconciliation at the end of that "automatic" reconciliation. - - Similarly, if a retry was scheduled, any event from the cluster triggering a successful execution in the mean time - would cancel the scheduled retry (because there's now no point in retrying something that already succeeded) - -2. In case an exception happened, a retry is initiated. However, if an event is received - meanwhile, it will be reconciled instantly, and this execution won't count as a retry attempt. -3. If the retry limit is reached (so no more automatic retry would happen), but a new event - received, the reconciliation will still happen, but won't reset the retry, and will still be - marked as the last attempt in the retry info. The point (1) still holds, but in case of an - error, no retry will happen. - -The thing to keep in mind when it comes to retrying or rescheduling is that JOSDK tries to avoid unnecessary work. When -you reschedule an operation, you instruct JOSDK to perform that operation at the latest by the end of the rescheduling -delay. If something occurred on the cluster that triggers that particular operation (reconciliation or cleanup), then -JOSDK considers that there's no point in attempting that operation again at the end of the specified delay since there -is now no point to do so anymore. The same idea also applies to retries. - -## Rate Limiting - -It is possible to rate limit reconciliation on a per-resource basis. The rate limit also takes -precedence over retry/re-schedule configurations: for example, even if a retry was scheduled for -the next second but this request would make the resource go over its rate limit, the next -reconciliation will be postponed according to the rate limiting rules. Note that the -reconciliation is never cancelled, it will just be executed as early as possible based on rate -limitations. - -Rate limiting is by default turned **off**, since correct configuration depends on the reconciler -implementation, in particular, on how long a typical reconciliation takes. -(The parallelism of reconciliation itself can be -limited [`ConfigurationService`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L120-L120) -by configuring the `ExecutorService` appropriately.) - -A default rate limiter implementation is provided, see: -[`PeriodRateLimiter`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/rate/PeriodRateLimiter.java#L14-L14) -. -Users can override it by implementing their own -[`RateLimiter`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ce4d996ee073ebef5715737995fc3d33f4751275/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/rate/RateLimiter.java) -and specifying this custom implementation using the `rateLimiter` field of the -`@ControllerConfiguration` annotation. Similarly to the `Retry` implementations, -`RateLimiter` implementations must provide an accessible, no-arg constructor for instantiation -purposes and can further be automatically configured from your own, provided annotation provided -your `RateLimiter` implementation also implements the `AnnotationConfigurable` interface, -parameterized by your custom annotation type. - -To configure the default rate limiter use the `@RateLimited` annotation on your -`Reconciler` class. The following configuration limits each resource to reconcile at most twice -within a 3 second interval: - -```java - -@RateLimited(maxReconciliations = 2, within = 3, unit = TimeUnit.SECONDS) -@ControllerConfiguration -public class MyReconciler implements Reconciler { - -} -``` - -Thus, if a given resource was reconciled twice in one second, no further reconciliation for this -resource will happen before two seconds have elapsed. Note that, since rate is limited on a -per-resource basis, other resources can still be reconciled at the same time, as long, of course, -that they stay within their own rate limits. - -## Handling Related Events with Event Sources - -See also -this [blog post](https://csviri.medium.com/java-operator-sdk-introduction-to-event-sources-a1aab5af4b7b) -. - -Event sources are a relatively simple yet powerful and extensible concept to trigger controller -executions, usually based on changes to dependent resources. You typically need an event source -when you want your `Reconciler` to be triggered when something occurs to secondary resources -that might affect the state of your primary resource. This is needed because a given -`Reconciler` will only listen by default to events affecting the primary resource type it is -configured for. Event sources act as listen to events affecting these secondary resources so -that a reconciliation of the associated primary resource can be triggered when needed. Note that -these secondary resources need not be Kubernetes resources. Typically, when dealing with -non-Kubernetes objects or services, we can extend our operator to handle webhooks or websockets -or to react to any event coming from a service we interact with. This allows for very efficient -controller implementations because reconciliations are then only triggered when something occurs -on resources affecting our primary resources thus doing away with the need to periodically -reschedule reconciliations. - -![Event Sources architecture diagram](../assets/images/event-sources.png) - -There are few interesting points here: - -The `CustomResourceEventSource` event source is a special one, responsible for handling events -pertaining to changes affecting our primary resources. This `EventSource` is always registered -for every controller automatically by the SDK. It is important to note that events always relate -to a given primary resource. Concurrency is still handled for you, even in the presence of -`EventSource` implementations, and the SDK still guarantees that there is no concurrent execution of -the controller for any given primary resource (though, of course, concurrent/parallel executions -of events pertaining to other primary resources still occur as expected). - -### Caching and Event Sources - -Kubernetes resources are handled in a declarative manner. The same also holds true for event -sources. For example, if we define an event source to watch for changes of a Kubernetes Deployment -object using an `InformerEventSource`, we always receive the whole associated object from the -Kubernetes API. This object might be needed at any point during our reconciliation process and -it's best to retrieve it from the event source directly when possible instead of fetching it -from the Kubernetes API since the event source guarantees that it will provide the latest -version. Not only that, but many event source implementations also cache resources they handle -so that it's possible to retrieve the latest version of resources without needing to make any -calls to the Kubernetes API, thus allowing for very efficient controller implementations. - -Note after an operator starts, caches are already populated by the time the first reconciliation -is processed for the `InformerEventSource` implementation. However, this does not necessarily -hold true for all event source implementations (`PerResourceEventSource` for example). The SDK -provides methods to handle this situation elegantly, allowing you to check if an object is -cached, retrieving it from a provided supplier if not. See -related [method](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java#L146) -. - -### Registering Event Sources - -To register event sources, your `Reconciler` has to override the `prepareEventSources` and return -list of event sources to register. One way to see this in action is -to look at the -[tomcat example](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java) -(irrelevant details omitted): - -```java - -@ControllerConfiguration -public class WebappReconciler - implements Reconciler, Cleaner, EventSourceInitializer { - // ommitted code - - @Override - public Map prepareEventSources(EventSourceContext context) { - InformerEventSourceConfiguration configuration = - InformerEventSourceConfiguration.from(Tomcat.class, Tomcat.class) - .withSecondaryToPrimaryMapper(webappsMatchingTomcatName) - .withPrimaryToSecondaryMapper( - (Webapp primary) -> Set.of(new ResourceID(primary.getSpec().getTomcat(), - primary.getMetadata().getNamespace()))) - .build(); - return EventSourceInitializer - .nameEventSources(new InformerEventSource<>(configuration, context)); - } - -} -``` - -In the example above an `InformerEventSource` is configured and registered. -`InformerEventSource` is one of the bundled `EventSource` implementations that JOSDK provides to -cover common use cases. - -### Managing Relation between Primary and Secondary Resources - -Event sources let your operator know when a secondary resource has changed and that your -operator might need to reconcile this new information. However, in order to do so, the SDK needs -to somehow retrieve the primary resource associated with which ever secondary resource triggered -the event. In the `Tomcat` example above, when an event occurs on a tracked `Deployment`, the -SDK needs to be able to identify which `Tomcat` resource is impacted by that change. - -Seasoned Kubernetes users already know one way to track this parent-child kind of relationship: -using owner references. Indeed, that's how the SDK deals with this situation by default as well, -that is, if your controller properly set owner references on your secondary resources, the SDK -will be able to follow that reference back to your primary resource automatically without you -having to worry about it. - -However, owner references cannot always be used as they are restricted to operating within a -single namespace (i.e. you cannot have an owner reference to a resource in a different namespace) -and are, by essence, limited to Kubernetes resources so you're out of luck if your secondary -resources live outside of a cluster. - -This is why JOSDK provides the `SecondayToPrimaryMapper` interface so that you can provide -alternative ways for the SDK to identify which primary resource needs to be reconciled when -something occurs to your secondary resources. We even provide some of these alternatives in the -[Mappers](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/Mappers.java) -class. - -Note that, while a set of `ResourceID` is returned, this set usually consists only of one -element. It is however possible to return multiple values or even no value at all to cover some -rare corner cases. Returning an empty set means that the mapper considered the secondary -resource event as irrelevant and the SDK will thus not trigger a reconciliation of the primary -resource in that situation. - -Adding a `SecondaryToPrimaryMapper` is typically sufficient when there is a one-to-many relationship -between primary and secondary resources. The secondary resources can be mapped to its primary -owner, and this is enough information to also get these secondary resources from the `Context` -object that's passed to your `Reconciler`. - -There are however cases when this isn't sufficient and you need to provide an explicit mapping -between a primary resource and its associated secondary resources using an implementation of the -`PrimaryToSecondaryMapper` interface. This is typically needed when there are many-to-one or -many-to-many relationships between primary and secondary resources, e.g. when the primary resource -is referencing secondary resources. -See [PrimaryToSecondaryIT](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/primarytosecondary/PrimaryToSecondaryIT.java) -integration test for a sample. - -### Built-in EventSources - -There are multiple event-sources provided out of the box, the following are some more central ones: - -#### `InformerEventSource` - -[InformerEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java) -is probably the most important `EventSource` implementation to know about. When you create an -`InformerEventSource`, JOSDK will automatically create and register a `SharedIndexInformer`, a -fabric8 Kubernetes client class, that will listen for events associated with the resource type -you configured your `InformerEventSource` with. If you want to listen to Kubernetes resource -events, `InformerEventSource` is probably the only thing you need to use. It's highly -configurable so you can tune it to your needs. Take a look at -[InformerEventSourceConfiguration](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java) -and associated classes for more details but some interesting features we can mention here is the -ability to filter events so that you can only get notified for events you care about. A -particularly interesting feature of the `InformerEventSource`, as opposed to using your own -informer-based listening mechanism is that caches are particularly well optimized preventing -reconciliations from being triggered when not needed and allowing efficient operators to be written. - -#### `PerResourcePollingEventSource` - -[PerResourcePollingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java) -is used to poll external APIs, which don't support webhooks or other event notifications. It -extends the abstract -[ExternalResourceCachingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/ExternalResourceCachingEventSource.java) -to support caching. -See [MySQL Schema sample](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaReconciler.java) -for usage. - -#### `PollingEventSource` - -[PollingEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java) -is similar to `PerResourceCachingEventSource` except that, contrary to that event source, it -doesn't poll a specific API separately per resource, but periodically and independently of -actually observed primary resources. - -#### Inbound event sources - -[SimpleInboundEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/SimpleInboundEventSource.java) -and -[CachingInboundEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/CachingInboundEventSource.java) -are used to handle incoming events from webhooks and messaging systems. - -#### `ControllerResourceEventSource` - -[ControllerResourceEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSource.java) -is a special `EventSource` implementation that you will never have to deal with directly. It is, -however, at the core of the SDK is automatically added for you: this is the main event source -that listens for changes to your primary resources and triggers your `Reconciler` when needed. -It features smart caching and is really optimized to minimize Kubernetes API accesses and avoid -triggering unduly your `Reconciler`. - -More on the philosophy of the non Kubernetes API related event source see in -issue [#729](https://github.com/java-operator-sdk/java-operator-sdk/issues/729). - -## Contextual Info for Logging with MDC - -Logging is enhanced with additional contextual information using -[MDC](http://www.slf4j.org/manual.html#mdc). The following attributes are available in most -parts of reconciliation logic and during the execution of the controller: - -| MDC Key | Value added from primary resource | -|:---------------------------|:----------------------------------| -| `resource.apiVersion` | `.apiVersion` | -| `resource.kind` | `.kind` | -| `resource.name` | `.metadata.name` | -| `resource.namespace` | `.metadata.namespace` | -| `resource.resourceVersion` | `.metadata.resourceVersion` | -| `resource.generation` | `.metadata.generation` | -| `resource.uid` | `.metadata.uid` | - -For more information about MDC see this [link](https://www.baeldung.com/mdc-in-log4j-2-logback). - -## InformerEventSource Multi-Cluster Support - -It is possible to handle resources for remote cluster with `InformerEventSource`. To do so, -simply set a client that connects to a remote cluster: - -```java - -InformerEventSourceConfiguration configuration = - InformerEventSourceConfiguration.from(SecondaryResource.class, PrimaryResource.class) - .withKubernetesClient(remoteClusterClient) - .withSecondaryToPrimaryMapper(Mappers.fromDefaultAnnotations()); - -``` - -You will also need to specify a `SecondaryToPrimaryMapper`, since the default one -is based on owner references and won't work across cluster instances. You could, for example, use the provided implementation that relies on annotations added to the secondary resources to identify the associated primary resource. - -See related [integration test](https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/informerremotecluster). - -## Dynamically Changing Target Namespaces - -A controller can be configured to watch a specific set of namespaces in addition of the -namespace in which it is currently deployed or the whole cluster. The framework supports -dynamically changing the list of these namespaces while the operator is running. -When a reconciler is registered, an instance of -[`RegisteredController`](https://github.com/java-operator-sdk/java-operator-sdk/blob/ec37025a15046d8f409c77616110024bf32c3416/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/RegisteredController.java#L5) -is returned, providing access to the methods allowing users to change watched namespaces as the -operator is running. - -A typical scenario would probably involve extracting the list of target namespaces from a -`ConfigMap` or some other input but this part is out of the scope of the framework since this is -use-case specific. For example, reacting to changes to a `ConfigMap` would probably involve -registering an associated `Informer` and then calling the `changeNamespaces` method on -`RegisteredController`. - -```java - -public static void main(String[] args) { - KubernetesClient client = new DefaultKubernetesClient(); - Operator operator = new Operator(client); - RegisteredController registeredController = operator.register(new WebPageReconciler(client)); - operator.installShutdownHook(); - operator.start(); - - // call registeredController further while operator is running -} - -``` - -If watched namespaces change for a controller, it might be desirable to propagate these changes to -`InformerEventSources` associated with the controller. In order to express this, -`InformerEventSource` implementations interested in following such changes need to be -configured appropriately so that the `followControllerNamespaceChanges` method returns `true`: - -```java - -@ControllerConfiguration -public class MyReconciler implements Reconciler { - - @Override - public Map prepareEventSources( - EventSourceContext context) { - - InformerEventSource configMapES = - new InformerEventSource<>(InformerEventSourceConfiguration.from(ConfigMap.class, TestCustomResource.class) - .withNamespacesInheritedFromController(context) - .build(), context); - - return EventSourceUtils.nameEventSources(configMapES); - } - -} -``` - -As seen in the above code snippet, the informer will have the initial namespaces inherited from -controller, but also will adjust the target namespaces if it changes for the controller. - -See also -the [integration test](https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/changenamespace) -for this feature. - -## Leader Election - -Operators are generally deployed with a single running or active instance. However, it is -possible to deploy multiple instances in such a way that only one, called the "leader", processes the -events. This is achieved via a mechanism called "leader election". While all the instances are -running, and even start their event sources to populate the caches, only the leader will process -the events. This means that should the leader change for any reason, for example because it -crashed, the other instances are already warmed up and ready to pick up where the previous -leader left off should one of them become elected leader. - -See sample configuration in -the [E2E test](https://github.com/java-operator-sdk/java-operator-sdk/blob/8865302ac0346ee31f2d7b348997ec2913d5922b/sample-operators/leader-election/src/main/java/io/javaoperatorsdk/operator/sample/LeaderElectionTestOperator.java#L21-L23) -. - -## Runtime Info - -[RuntimeInfo](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/RuntimeInfo.java#L16-L16) -is used mainly to check the actual health of event sources. Based on this information it is easy to implement custom -liveness probes. - -[stopOnInformerErrorDuringStartup](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java#L168-L168) -setting, where this flag usually needs to be set to false, in order to control the exact liveness properties. - -See also an example implementation in the -[WebPage sample](https://github.com/java-operator-sdk/java-operator-sdk/blob/3e2e7c4c834ef1c409d636156b988125744ca911/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java#L38-L43) - -## Automatic Generation of CRDs - -Note that this feature is provided by the -[Fabric8 Kubernetes Client](https://github.com/fabric8io/kubernetes-client), not JOSDK itself. - -To automatically generate CRD manifests from your annotated Custom Resource classes, you only need -to add the following dependencies to your project: - -```xml - - - io.fabric8 - crd-generator-apt - provided - -``` - -The CRD will be generated in `target/classes/META-INF/fabric8` (or -in `target/test-classes/META-INF/fabric8`, if you use the `test` scope) with the CRD name -suffixed by the generated spec version. For example, a CR using the `java-operator-sdk.io` group -with a `mycrs` plural form will result in 2 files: - -- `mycrs.java-operator-sdk.io-v1.yml` -- `mycrs.java-operator-sdk.io-v1beta1.yml` - -**NOTE:** -> Quarkus users using the `quarkus-operator-sdk` extension do not need to add any extra dependency -> to get their CRD generated as this is handled by the extension itself. - -## Metrics - -JOSDK provides built-in support for metrics reporting on what is happening with your reconcilers in the form of -the `Metrics` interface which can be implemented to connect to your metrics provider of choice, JOSDK calling the -methods as it goes about reconciling resources. By default, a no-operation implementation is provided thus providing a -no-cost sane default. A [micrometer](https://micrometer.io)-based implementation is also provided. - -You can use a different implementation by overriding the default one provided by the default `ConfigurationService`, as -follows: - -```java -Metrics metrics; // initialize your metrics implementation -Operator operator = new Operator(client, o -> o.withMetrics(metrics)); -``` - -### Micrometer implementation - -The micrometer implementation is typically created using one of the provided factory methods which, depending on which -is used, will return either a ready to use instance or a builder allowing users to customized how the implementation -behaves, in particular when it comes to the granularity of collected metrics. It is, for example, possible to collect -metrics on a per-resource basis via tags that are associated with meters. This is the default, historical behavior but -this will change in a future version of JOSDK because this dramatically increases the cardinality of metrics, which -could lead to performance issues. - -To create a `MicrometerMetrics` implementation that behaves how it has historically behaved, you can just create an -instance via: - -```java -MeterRegistry registry; // initialize your registry implementation -Metrics metrics = new MicrometerMetrics(registry); -``` - -Note, however, that this constructor is deprecated and we encourage you to use the factory methods instead, which either -return a fully pre-configured instance or a builder object that will allow you to configure more easily how the instance -will behave. You can, for example, configure whether or not the implementation should collect metrics on a per-resource -basis, whether or not associated meters should be removed when a resource is deleted and how the clean-up is performed. -See the relevant classes documentation for more details. - -For example, the following will create a `MicrometerMetrics` instance configured to collect metrics on a per-resource -basis, deleting the associated meters after 5 seconds when a resource is deleted, using up to 2 threads to do so. - -```java -MicrometerMetrics.newPerResourceCollectingMicrometerMetricsBuilder(registry) - .withCleanUpDelayInSeconds(5) - .withCleaningThreadNumber(2) - .build(); -``` - -### Operator SDK metrics - -The micrometer implementation records the following metrics: - -| Meter name | Type | Tag names | Description | -|-------------------------------------------------------------|----------------|-------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| -| operator.sdk.reconciliations.executions.`` | gauge | group, version, kind | Number of executions of the named reconciler | -| operator.sdk.reconciliations.queue.size.`` | gauge | group, version, kind | How many resources are queued to get reconciled by named reconciler | -| operator.sdk.``.size | gauge map size | | Gauge tracking the size of a specified map (currently unused but could be used to monitor caches size) | -| operator.sdk.events.received | counter | ``, event, action | Number of received Kubernetes events | -| operator.sdk.events.delete | counter | `` | Number of received Kubernetes delete events | -| operator.sdk.reconciliations.started | counter | ``, reconciliations.retries.last, reconciliations.retries.number | Number of started reconciliations per resource type | -| operator.sdk.reconciliations.failed | counter | ``, exception | Number of failed reconciliations per resource type | -| operator.sdk.reconciliations.success | counter | `` | Number of successful reconciliations per resource type | -| operator.sdk.controllers.execution.reconcile | timer | ``, controller | Time taken for reconciliations per controller | -| operator.sdk.controllers.execution.cleanup | timer | ``, controller | Time taken for cleanups per controller | -| operator.sdk.controllers.execution.reconcile.success | counter | controller, type | Number of successful reconciliations per controller | -| operator.sdk.controllers.execution.reconcile.failure | counter | controller, exception | Number of failed reconciliations per controller | -| operator.sdk.controllers.execution.cleanup.success | counter | controller, type | Number of successful cleanups per controller | -| operator.sdk.controllers.execution.cleanup.failure | counter | controller, exception | Number of failed cleanups per controller | - -As you can see all the recorded metrics start with the `operator.sdk` prefix. ``, in the table above, -refers to resource-specific metadata and depends on the considered metric and how the implementation is configured and -could be summed up as follows: `group?, version, kind, [name, namespace?], scope` where the tags in square -brackets (`[]`) won't be present when per-resource collection is disabled and tags followed by a question mark are -omitted if the associated value is empty. Of note, when in the context of controllers' execution metrics, these tag -names are prefixed with `resource.`. This prefix might be removed in a future version for greater consistency. - -## Optimizing Caches - -One of the ideas around the operator pattern is that all the relevant resources are cached, thus reconciliation is -usually very fast (especially if no resources are updated in the process) since the operator is then mostly working with -in-memory state. However for large clusters, caching huge amount of primary and secondary resources might consume lots -of memory. JOSDK provides ways to mitigate this issue and optimize the memory usage of controllers. While these features -are working and tested, we need feedback from real production usage. - -### Bounded Caches for Informers - -Limiting caches for informers - thus for Kubernetes resources - is supported by ensuring that resources are in the cache -for a limited time, via a cache eviction of least recently used resources. This means that when resources are created -and frequently reconciled, they stay "hot" in the cache. However, if, over time, a given resource "cools" down, i.e. it -becomes less and less used to the point that it might not be reconciled anymore, it will eventually get evicted from the -cache to free up memory. If such an evicted resource were to become reconciled again, the bounded cache implementation -would then fetch it from the API server and the "hot/cold" cycle would start anew. - -Since all resources need to be reconciled when a controller start, it is not practical to set a maximal cache size as -it's desirable that all resources be cached as soon as possible to make the initial reconciliation process on start as -fast and efficient as possible, avoiding undue load on the API server. It's therefore more interesting to gradually -evict cold resources than try to limit cache sizes. - -See usage of the related implementation using [Caffeine](https://github.com/ben-manes/caffeine) cache in integration -tests -for [primary resources](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/sample/AbstractTestReconciler.java#L29-L29). - -See -also [CaffeineBoundedItemStores](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/main/java/io/javaoperatorsdk/operator/processing/event/source/cache/CaffeineBoundedItemStores.java) -for more details. - - diff --git a/docs/content/en/docs/getting-started/_index.md b/docs/content/en/docs/getting-started/_index.md index e3a3f95788..df8a4b77fe 100644 --- a/docs/content/en/docs/getting-started/_index.md +++ b/docs/content/en/docs/getting-started/_index.md @@ -1,58 +1,4 @@ --- -title: Getting Started - -weight: 20 ---- - -## Introduction & Resources on Operators - -Operators manage both cluster and non-cluster resources on behalf of Kubernetes. This Java -Operator SDK (JOSDK) aims at making it as easy as possible to write Kubernetes operators in Java -using an API that should feel natural to Java developers and without having to worry about many -low-level details that the SDK handles automatically. - -For an introduction on operators, please see this -[blog post](https://blog.container-solutions.com/kubernetes-operators-explained). -or [this talk](https://www.youtube.com/watch?v=CvftaV-xrB4) - -You can read about the common problems JOSDK is solving for you -[here](https://blog.container-solutions.com/a-deep-dive-into-the-java-operator-sdk). - -You can also refer to the -[Writing Kubernetes operators using JOSDK blog series](https://developers.redhat.com/articles/2022/02/15/write-kubernetes-java-java-operator-sdk) -. - -## Generating Project Skeleton - -Project includes a maven plugin to generate a skeleton project: - -```shell -mvn io.javaoperatorsdk:bootstrapper:[version]:create -DprojectGroupId=org.acme -DprojectArtifactId=getting-started -``` - -## Getting Started - -The easiest way to get started with SDK is to start -[minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) and -execute one of our [examples](https://github.com/java-operator-sdk/java-operator-sdk/tree/main/sample-operators). -There is a dedicated page to describe how to [use the samples](/docs/using-samples). - -Here are the main steps to develop the code and deploy the operator to a Kubernetes cluster. -A more detailed and specific version can be found under `samples/mysql-schema/README.md`. - -1. Setup `kubectl` to work with your Kubernetes cluster of choice. -1. Apply Custom Resource Definition -1. Compile the whole project (framework + samples) using `mvn install` in the root directory -1. Run the main class of the sample you picked and check out the sample's README to see what it - does. When run locally the framework will use your Kubernetes client configuration (in `~/. - kube/config`) to establish a connection to the cluster. This is why it was important to set - up `kubectl` up front. -1. You can work in this local development mode to play with the code. -1. Build the Docker image and push it to the registry -1. Apply RBAC configuration -1. Apply deployment configuration -1. Verify if the operator is up and running. Don't run it locally anymore to avoid conflicts in - processing events from the cluster's API server. - - - +title: Getting started +weight: 10 +--- \ No newline at end of file diff --git a/docs/content/en/docs/getting-started/bootstrap-and-samples.md b/docs/content/en/docs/getting-started/bootstrap-and-samples.md new file mode 100644 index 0000000000..597ed3477e --- /dev/null +++ b/docs/content/en/docs/getting-started/bootstrap-and-samples.md @@ -0,0 +1,38 @@ +--- +title: Bootstrapping and samples +weight: 20 +--- + +## Generating Project Skeleton + +Project includes a maven plugin to generate a skeleton project: + +```shell +mvn io.javaoperatorsdk:bootstrapper:[version]:create -DprojectGroupId=org.acme -DprojectArtifactId=getting-started +``` + +You can build this project with maven, +the build will generate also the [CustomResourceDefinition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) +for you. + +## Getting started with samples + +You can find examples under [sample-operators](https://github.com/java-operator-sdk/java-operator-sdk/tree/master/sample-operators) +directory which are intended to demonstrate the usage of different components in different scenarios, but mainly are more real world +examples: + +* *webpage*: Simple example creating an NGINX webserver from a Custom Resource containing HTML code. We provide more + flavors of implementation, both with the low level APIs and higher level abstractions. +* *mysql-schema*: Operator managing schemas in a MySQL database. Shows how to manage non Kubernetes resources. +* *tomcat*: Operator with two controllers, managing Tomcat instances and Webapps running in Tomcat. The intention + with this example to show how to manage multiple related custom resources and/or more controllers. + +The easiest way to run / try out is to run one of the samples on +[minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) or [kind](https://kind.sigs.k8s.io/). +After applying the generated CRD, you can simply run your main class. The controller will automatically +start communicate with you local Kubernetes cluster and reconcile custom resource after you create one. + +See also detailed instructions under [`samples/mysql-schema/README.md`](https://github.com/operator-framework/java-operator-sdk/blob/main/sample-operators/mysql-schema/README.md). + + + diff --git a/docs/content/en/docs/getting-started/intro-to-operators.md b/docs/content/en/docs/getting-started/intro-to-operators.md new file mode 100644 index 0000000000..6247bef288 --- /dev/null +++ b/docs/content/en/docs/getting-started/intro-to-operators.md @@ -0,0 +1,32 @@ +--- +title: Introduction to Kubernetes operators +weight: 15 +--- + +## Introduction & Resources + +Operators manage both cluster and non-cluster resources on behalf of Kubernetes. Java +Operator SDK (JOSDK) aims to make it as easy as possible to implement a Kubernetes operators in Java. +The APIs are designed to feel natural to Java developers. In addition the framework tries to +handle common problem out of the box, so you don't have to worry about generic sub-problems. + +For an introduction on operators, please see this +[blog post](https://blog.container-solutions.com/kubernetes-operators-explained). + +For introductions to JOSDK see [this talk](https://www.youtube.com/watch?v=CvftaV-xrB4). + +You can read about the common problems JOSDK is solving for you +[here](https://blog.container-solutions.com/a-deep-dive-into-the-java-operator-sdk). + +You can also refer to the +[Writing Kubernetes operators using JOSDK blog series](https://developers.redhat.com/articles/2022/02/15/write-kubernetes-java-java-operator-sdk). + + +## Operators in General + - [Implementing Kubernetes Operators in Java talk](https://www.youtube.com/watch?v=CvftaV-xrB4) + - [Introduction of the concept of Kubernetes Operators](https://blog.container-solutions.com/kubernetes-operators-explained) + - [Operator pattern explained in Kubernetes documentation](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) + - [An explanation why Java Operators makes sense](https://blog.container-solutions.com/cloud-native-java-infrastructure-automation-with-kubernetes-operators) + - [What are the problems an operator framework is solving](https://csviri.medium.com/deep-dive-building-a-kubernetes-operator-sdk-for-java-developers-5008218822cb) + - [Writing Kubernetes operators using JOSDK blog series](https://developers.redhat.com/articles/2022/02/15/write-kubernetes-java-java-operator-sdk) + diff --git a/docs/content/en/docs/patterns-and-best-practices/_index.md b/docs/content/en/docs/getting-started/patterns-best-practices.md similarity index 99% rename from docs/content/en/docs/patterns-and-best-practices/_index.md rename to docs/content/en/docs/getting-started/patterns-best-practices.md index a2b3b716b6..575c82af72 100644 --- a/docs/content/en/docs/patterns-and-best-practices/_index.md +++ b/docs/content/en/docs/getting-started/patterns-best-practices.md @@ -1,9 +1,8 @@ --- -title: Patterns and Best Practices +title: Patterns and best practices weight: 25 --- - This document describes patterns and best practices, to build and run operators, and how to implement them in terms of the Java Operator SDK (JOSDK). diff --git a/docs/content/en/docs/glossary/_index.md b/docs/content/en/docs/glossary/_index.md index 187a0bfb4a..1523f68a23 100644 --- a/docs/content/en/docs/glossary/_index.md +++ b/docs/content/en/docs/glossary/_index.md @@ -1,6 +1,6 @@ --- title: Glossary -weight: 40 +weight: 100 --- - **Primary Resource** - the resource that represents the desired state that the controller is diff --git a/docs/content/en/docs/intro-to-operators/_index.md b/docs/content/en/docs/intro-to-operators/_index.md deleted file mode 100644 index 54fc7d82a8..0000000000 --- a/docs/content/en/docs/intro-to-operators/_index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: Introduction to Operators - -weight: 10 ---- - -This page provides a selection of articles that gives an introduction to Kubernetes operators. - -## Operators in General - - [Implementing Kubernetes Operators in Java talk](https://www.youtube.com/watch?v=CvftaV-xrB4) - - [Introduction of the concept of Kubernetes Operators](https://blog.container-solutions.com/kubernetes-operators-explained) - - [Operator pattern explained in Kubernetes documentation](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) - - [An explanation why Java Operators makes sense](https://blog.container-solutions.com/cloud-native-java-infrastructure-automation-with-kubernetes-operators) - - [What are the problems an operator framework is solving](https://csviri.medium.com/deep-dive-building-a-kubernetes-operator-sdk-for-java-developers-5008218822cb) - - [Writing Kubernetes operators using JOSDK blog series](https://developers.redhat.com/articles/2022/02/15/write-kubernetes-java-java-operator-sdk) - diff --git a/docs/content/en/docs/migration/_index.md b/docs/content/en/docs/migration/_index.md index 9d7ca4d7f2..115adab35d 100644 --- a/docs/content/en/docs/migration/_index.md +++ b/docs/content/en/docs/migration/_index.md @@ -1,4 +1,5 @@ --- title: Migrations +weight: 150 --- diff --git a/docs/content/en/docs/using-samples/_index.md b/docs/content/en/docs/using-samples/_index.md deleted file mode 100644 index 62319860ac..0000000000 --- a/docs/content/en/docs/using-samples/_index.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -title: Using sample Operators -weight: 30 ---- - -We have examples under [sample-operators](https://github.com/java-operator-sdk/java-operator-sdk/tree/master/sample-operators) -directory which are intended to demonstrate the usage of different components in different scenarios, but mainly are more real world -examples: - -* *webpage*: Simple example creating an NGINX webserver from a Custom Resource containing HTML code. -* *mysql-schema*: Operator managing schemas in a MySQL database. Shows how to manage non Kubernetes resources. -* *tomcat*: Operator with two controllers, managing Tomcat instances and Webapps running in Tomcat. The intention - with this example to show how to manage multiple related custom resources and/or more controllers. - -# Implementing a Sample Operator - -Add [dependency](https://search.maven.org/search?q=a:operator-framework%20AND%20g:io.javaoperatorsdk) to your project with Maven: - -```xml - - - io.javaoperatorsdk - operator-framework - {see https://search.maven.org/search?q=a:operator-framework%20AND%20g:io.javaoperatorsdk for latest version} - -``` - -Or alternatively with Gradle, which also requires declaring the SDK as an annotation processor to generate the mappings -between controllers and custom resource classes: - -```groovy -dependencies { - implementation "io.javaoperatorsdk:operator-framework:${javaOperatorVersion}" - annotationProcessor "io.javaoperatorsdk:operator-framework:${javaOperatorVersion}" -} -``` - -Once you've added the dependency, define a main method initializing the Operator and registering a controller. - -```java -public class Runner { - - public static void main(String[] args) { - Operator operator = new Operator(); - operator.register(new WebPageReconciler()); - operator.start(); - } -} -``` - -The Controller implements the business logic and describes all the classes needed to handle the CRD. - -```java - -@ControllerConfiguration -public class WebPageReconciler implements Reconciler { - - // Return the changed resource, so it gets updated. See javadoc for details. - @Override - public UpdateControl reconcile(CustomService resource, - Context context) { - // ... your logic ... - return UpdateControl.patchStatus(resource); - } -} -``` - -A sample custom resource POJO representation - -```java - -@Group("sample.javaoperatorsdk") -@Version("v1") -public class WebPage extends CustomResource implements - Namespaced { -} - -public class WebServerSpec { - - private String html; - - public String getHtml() { - return html; - } - - public void setHtml(String html) { - this.html = html; - } -} -``` - -### Deactivating CustomResource implementations validation - -The operator will, by default, query the deployed CRDs to check that the `CustomResource` -implementations match what is known to the cluster. This requires an additional query to the cluster and, sometimes, -elevated privileges for the operator to be able to read the CRDs from the cluster. This validation is mostly meant to -help users new to operator development get started and avoid common mistakes. Advanced users or production deployments -might want to skip this step. This is done by setting the `CHECK_CRD_ENV_KEY` environment variable to `false`. - -### Automatic generation of CRDs - -To automatically generate CRD manifests from your annotated Custom Resource classes, you only need to add the following -dependencies to your project (in the background an annotation processor is used), with Maven: - -```xml - - - io.fabric8 - crd-generator-apt - provided - -``` - -or with Gradle: - -```groovy -dependencies { - annotationProcessor 'io.fabric8:crd-generator-apt:' - ... -} -``` - -The CRD will be generated in `target/classes/META-INF/fabric8` (or in `target/test-classes/META-INF/fabric8`, if you use -the `test` scope) with the CRD name suffixed by the generated spec version. For example, a CR using -the `java-operator-sdk.io` group with a `mycrs` plural form will result in 2 files: - -- `mycrs.java-operator-sdk.io-v1.yml` -- `mycrs.java-operator-sdk.io-v1beta1.yml` - -**NOTE:** -> Quarkus users using the `quarkus-operator-sdk` extension do not need to add any extra dependency to get their CRD generated as this is handled by the extension itself. - -### Quarkus - -A [Quarkus](https://quarkus.io) extension is also provided to ease the development of Quarkus-based operators. - -Add [this dependency](https://search.maven.org/search?q=a:quarkus-operator-sdk) -to your project: - -```xml - - - io.quarkiverse.operatorsdk - quarkus-operator-sdk - {see https://search.maven.org/search?q=a:quarkus-operator-sdk for latest version} - - -``` - -Create an Application, Quarkus will automatically create and inject a `KubernetesClient` ( -or `OpenShiftClient`), `Operator`, `ConfigurationService` and `ResourceController` instances that your application can -use. Below, you can see the minimal code you need to write to get your operator and controllers up and running: - -```java - -@QuarkusMain -public class QuarkusOperator implements QuarkusApplication { - - @Inject - Operator operator; - - public static void main(String... args) { - Quarkus.run(QuarkusOperator.class, args); - } - - @Override - public int run(String... args) throws Exception { - operator.start(); - Quarkus.waitForExit(); - return 0; - } -} -``` - -### Spring Boot - -You can also let Spring Boot wire your application together and automatically register the controllers. - -Add [this dependency](https://search.maven.org/search?q=a:operator-framework-spring-boot-starter%20AND%20g:io.javaoperatorsdk) to your project: - -```xml - - - io.javaoperatorsdk - operator-framework-spring-boot-starter - {see https://search.maven.org/search?q=a:operator-framework-spring-boot-starter%20AND%20g:io.javaoperatorsdk for - latest version} - - -``` - -Create an Application - -```java - -@SpringBootApplication -public class Application { - - public static void main(String[] args) { - SpringApplication.run(Application.class, args); - } -} -``` - -You will also need a `@Configuration` to make sure that your reconciler is registered: - -```java - -@Configuration -public class Config { - - @Bean - public WebPageReconciler customServiceController() { - return new WebPageReconciler(); - } - - @Bean(initMethod = "start", destroyMethod = "stop") - @SuppressWarnings("rawtypes") - public Operator operator(List controllers) { - Operator operator = new Operator(); - controllers.forEach(operator::register); - return operator; - } -} -``` - -#### Spring Boot test support - -Adding the following dependency would let you mock the operator for the tests where loading the spring container is -necessary, but it doesn't need real access to a Kubernetes cluster. - -```xml - - - io.javaoperatorsdk - operator-framework-spring-boot-starter-test - {see https://search.maven.org/search?q=a:operator-framework-spring-boot-starter%20AND%20g:io.javaoperatorsdk for - latest version} - - -``` - -Mock the operator: - -```java - -@SpringBootTest -@EnableMockOperator -public class SpringBootStarterSampleApplicationTest { - - @Test - void contextLoads() { - } -} -``` From b793702bcc41140a2ee9c40f3a542833a41bff03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 24 Mar 2025 13:58:52 +0100 Subject: [PATCH 07/45] docs: fixes on event source page (#2739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../content/en/docs/documentation/eventing.md | 37 ++++++++---------- docs/static/images/event-sources.png | Bin 0 -> 95841 bytes 2 files changed, 17 insertions(+), 20 deletions(-) create mode 100644 docs/static/images/event-sources.png diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index 0ede7a21a6..2591ab19c9 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -23,7 +23,7 @@ controller implementations because reconciliations are then only triggered when on resources affecting our primary resources thus doing away with the need to periodically reschedule reconciliations. -![Event Sources architecture diagram](../assets/images/event-sources.png) +![Event Sources architecture diagram](/images/event-sources.png) There are few interesting points here: @@ -60,29 +60,26 @@ related [method](https://github.com/java-operator-sdk/java-operator-sdk/blob/mai To register event sources, your `Reconciler` has to override the `prepareEventSources` and return list of event sources to register. One way to see this in action is to look at the -[tomcat example](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/WebappReconciler.java) +[WebPage example](https://github.com/operator-framework/java-operator-sdk/blob/main/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageReconciler.java) (irrelevant details omitted): ```java +import java.util.List; + @ControllerConfiguration public class WebappReconciler implements Reconciler, Cleaner, EventSourceInitializer { - // ommitted code - - @Override - public Map prepareEventSources(EventSourceContext context) { - InformerEventSourceConfiguration configuration = - InformerEventSourceConfiguration.from(Tomcat.class, Tomcat.class) - .withSecondaryToPrimaryMapper(webappsMatchingTomcatName) - .withPrimaryToSecondaryMapper( - (Webapp primary) -> Set.of(new ResourceID(primary.getSpec().getTomcat(), - primary.getMetadata().getNamespace()))) + // ommitted code + + @Override + public List> prepareEventSources(EventSourceContext context) { + InformerEventSourceConfiguration configuration = + InformerEventSourceConfiguration.from(Deployment.class, Webapp.class) + .withLabelSelector(SELECTOR) .build(); - return EventSourceInitializer - .nameEventSources(new InformerEventSource<>(configuration, context)); + return List.of(new InformerEventSource<>(configuration, context)); } - } ``` @@ -95,8 +92,8 @@ cover common use cases. Event sources let your operator know when a secondary resource has changed and that your operator might need to reconcile this new information. However, in order to do so, the SDK needs to somehow retrieve the primary resource associated with which ever secondary resource triggered -the event. In the `Tomcat` example above, when an event occurs on a tracked `Deployment`, the -SDK needs to be able to identify which `Tomcat` resource is impacted by that change. +the event. In the `Webapp` example above, when an event occurs on a tracked `Deployment`, the +SDK needs to be able to identify which `Webapp` resource is impacted by that change. Seasoned Kubernetes users already know one way to track this parent-child kind of relationship: using owner references. Indeed, that's how the SDK deals with this situation by default as well, @@ -198,7 +195,7 @@ simply set a client that connects to a remote cluster: ```java -InformerEventSourceConfiguration configuration = +InformerEventSourceConfiguration configuration = InformerEventSourceConfiguration.from(SecondaryResource.class, PrimaryResource.class) .withKubernetesClient(remoteClusterClient) .withSecondaryToPrimaryMapper(Mappers.fromDefaultAnnotations()); @@ -323,8 +320,8 @@ evict cold resources than try to limit cache sizes. See usage of the related implementation using [Caffeine](https://github.com/ben-manes/caffeine) cache in integration tests -for [primary resources](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/sample/AbstractTestReconciler.java#L29-L29). +for [primary resources](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/sample/AbstractTestReconciler.java). See -also [CaffeineBoundedItemStores](https://github.com/java-operator-sdk/java-operator-sdk/blob/902c8a562dfd7f8993a52e03473a7ad4b00f378b/caffeine-bounded-cache-support/src/main/java/io/javaoperatorsdk/operator/processing/event/source/cache/CaffeineBoundedItemStores.java) +also [CaffeineBoundedItemStores](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/caffeine-bounded-cache-support/src/main/java/io/javaoperatorsdk/operator/processing/event/source/cache/CaffeineBoundedItemStores.java) for more details. \ No newline at end of file diff --git a/docs/static/images/event-sources.png b/docs/static/images/event-sources.png new file mode 100644 index 0000000000000000000000000000000000000000..f37e1f72d9e00d9b1837ecf8109e7aa595f804f6 GIT binary patch literal 95841 zcmb6BWmJ^i`v;CHA`S{tBHaQ?cZYz0gmg;?NQ1OAh#(yb5=u!(4BZXV%}5L(B@CU? z4QJmx-~Vr&b6%aZmM>g7bH|Qrf9jf06(w2h`=s}8-MWP>_wt3>ty}2jw{D@C!Z5(U z9NVh9-MaPRmfQ;o4R^!M8O(@B$g5+&gb+#eCq)u1e;+>&C39@awbaklZrpD9RMW}J zL^@v6c^5@09U9Vk=vLepITh*7TQro}euc91x zxTOyN`O-6v7-$+8XwHa^9Ddsotdxv-A_x7^Yrl`g{O|8-1|;S;|Mn!$jQIcWzm2l> zpZQ(yTrZ_HUmm^wF63%mcd^$ug48xjc~>(Zdd}DDJC~Wf z=f9SFBXrRsF_d1Lr3WXd=p4pWcUsZIQT4T@1S6&5hCO-Dt;+FXGW&)v=KYCZW_^xl zc$}B}53bI4a+Jj{c?MENg?)}DE!M)>n_kZO92LbS52hLh|NFH%;OKm(IY<2J zgzj?Z`gHC9dcuJzk}1-u0hi@+eV*-+UbFm#QWm3>qmAO`JreWg+C%@+(x$?KQT-NPlUHS>(F0KyZzw0Oj~u+E z3K&WE-7jkfb_)mXtD@lMef$J{?P47^Mol;-M)mTCg8A6HO=0I0cvIv%mlH>Om*j~0 zTvP3ZLrUTIw!X?POPrsf`D8V+CNlRYSmiYB%)%sKSmf9kR$zZcWaRBw_pkNA$mvR2 znv(EVh0CnR%1?D=QQj2qy`HriRFr}&g{y|pW=Y%dtk+J;doYd_`-8$c6mHX;cJ*6T zR$%fJ>13l6lit-?{c%S{js%gtOT@sTOdivR>FAn3dw{ zSx*n&NttNMlxSs&Z|!fb`xZ3p z^;!jB^I0Cdb48ASHZz^;?G$qRW0lj8V%rBMb}rkltcSsBG*{JXqQv-A^w(C|R#_jL zReoWW{JrbLzZbFRW~fVzs(^<=PuUfO7e7l4rw3DhpL45d!x0xracIS?O>ticUCWA9 z>fJ%8D+$s!Yg;ws^u6D%U3p)(TjM+^$e03_Y;B6`nC@CXH$u@O*|9T?bgC6$8ZF9a z-!&vll_JiJ>Woy7rSy3FPL|y9#q7~UH24bR&K%f0-;O3sqboLw>X#98#WcltyR7yH z_?46Fs&rBN?WIx2cXo(olXkCG8cBAnvUg9n6-yYPZLO9UZ~Qopn#`_3_2DxZE-V|C z1Y4YayfhO(@E#AppD+%dh6%%s@owX% z-;>ZmS28pG!#8Cc8&C{8=#_ces@z%$``6oPO8b?rA^bNR={{@()~|!(uP~W7D5S&p zvA#^Yo9NiI(bk>>H=Z@Si%+^_f<>PtpgF3_I;8tO0YfP}veRkPXw%OYXJmmG|B=iI zY1hYpRk{&Y?ddru{R)5c^|-Ev>9TbzFFn z(2K1!n1}B%|B1+nsAX*=CkIFDgxc*(<~f-`-0FMn_dZO@F>9R~TUU<^l1vOI4Xjm< z>lG$N3xo(&~DEG305eIH;?tA~4E9Yq<{vJPlaQH`i5MQe#_ zAx}%Aj*hBDuFUumN9|%-QyE8j8lp1DLzfEIgHuV1_{hVNM&7!r_!+%58TkaYy0XD< z+>Cw<76aeo#I8q(ypyGEhT0Vb zSt@+w9LZg!?a$c5$K#QQ4xdqy$_M4Yt>uz+UkSQQmbo>Fxnm6CMED-1`y1CW zwfrEQa2wb0$*s-JE6s%BD)>>hn~dz(+|cP3>v(Dui;;nwMOsdI-!X;@#j+$UqD4l> zPV>JhlS6X#qXkrRpLl8i|9q9lSWJixbXa*temXSmaw(cq9O3!6+jSd7$etLzN5*4# zvgAWG_S#a={|?9G+9g)5q6(g3^CV5^CQzYH(07t+YHYyvWrserE>wwKPW;*;Gm;Ok6hCT0Tf@}G@_*lE zQSv5HUa1&SQXsP}k$`Stnt1bg>+)dIh!ui4Hj%bx`T^W9ci2|GT4_Hry<+qi5%E+$inrU`;)R{^pB| zJ+a{JZ=N;z&nMmFli_!bhhr5BVa!$E*zyiJFeovwh=0S%FGLjn@1-5a4cwOA1w9~Q z>pw;<^0yXQ#;iw%3w=Ovr`epG#z7Orn)fGIcYk@nhI@uX!J5Kb<{BLJ7tV)BZDsXr z#h#J8v9K`Kn}HC@JTe~JoGd3B$^FOCAEPJB+h2FSUBB=0u^@K>y5*BRZfh#@#_Ox| z$lpv#Vc6waW6v!$gLKt%lm

?altTsK8sFO9la%yg|h%bTxoA1lz42zXiDLQ07g?$^n2)3e1HSF~@M&9{DvqkmEvh7i9NaU)CRu~N$nPdBcB-qA{Kl&gQ?34d!rFqN-> zGAhh6MsEa+%E)({qDhASn{l}L44U!S-?6y5;6S{qNk%7T&IDV{85I zxzx@%!efo(3PZzs=lrfkeNN_st-!$BYncBJh2E@O;zwto-Ve-RF)AgmJ|9vG58H0x zHt&CFQ{4Ez?&@rVS>YvgxlzF&-I%&p5s`<>N%qU~ePA)N%~f;#H#%Xi z_*e|Qwkl7X?)>`SGEw-XtL6N%&&1+0&7baSd?4T!)hDudp(RJ=1F#AtL87C+x@p8^zgxk@Vz`7zli0qsYGgV_s4kp z!^(@A&epx>sQ>pc0H8U>oI|M8zUh=9l3Z|@Lt*_cZGYWnS>MI{hxV++V6wQT zxn=;1a@fx?T;s70BBRX}E$D44G{8lq9V!7>y#ZR8Q9^LXy$!jN(eK;fkgaHYD4Yf0 z8v;>!GiiAh8j19Iym9|68aahF4h6Mpy8b_oT@={m&?n>+3h7%}h%LcnAd3F`R9-%q z>9D`n&{hD?>eqDHe7)k=_&t5}<4N)?KvGuyiJV}}!PmhO>*HXCm&3}Bz`5bgH9bp+ zO^+l?Gx&dYbe=(?W{^gpE_gJrqF(qQKUk(x8s=LK`-A`QSHLxgU{;VWm}A6hVfC!_ z|Edf$sPa-lSaMVE>g~j67=FVsbit<@J1h?3U%%@y9;;nRol0C@(Y_g)CwjE~tyEwk z=TP^>8t2WL6x1i#H%=jMBqIm!N3Tq`OC$H@r zU7H3%ckG)lwRBC^SnU}{(RGbIF#lC-&N#GT|L>k`s$g59BKfz+VrQv*PiRCh4@RuO z&Wmh;dvQlCiC^tTlUu8PVK_u@Etq*fHSqsd6(UJ_;f~G1t=V8>h!&Q#-nWW<#dkxf z&H#GN4?kdvWmFzK_u6SZ*0MMx4SHf=Zxx;Qf72?||HmxBU|5FCZnQ?39*^;t*=xOf z?)CJ0Bk#Tm#@R~mnBw;YZk(IOgkg8XUJ-)W)AB-z3Qe&ERw%y8G0nM9HE1DUe!1{Hzc>q@7_xoR;3REx==}hO^|>00k=&dy?P8SNQL2 zRFz!VFc^j->@|e`f9ugFk=J&jr1KIMQzw)h!hAlTljbXbeX$}wVwIB;KL%b(#-VKa zyUp^g)dVBoo<8YHEjL?bH&PLIhlk}1iis*INLE^OmV2o?SZEw$LlB{v4$`D9B4djrI-C(ky9cAB!st>}l$AKL4Jwj-sfLX2W9J-}&igB}K$tcMUNVIw~z0-XMKbB*LE+lv(C z&$26=)u_2qqRi{|PH8`rbG&)}&|_%wuqzR4uK%_*1YT~?0>n1%a?I#p8Q%(qIIY|L?+X?^3n5y*w1*UGV8SC(rUJvxy$l ziJU#x`^a&o*5P&M%uNQ879*C|Vs1~3lJ*2q6EXs1H57#!Gb_@@GvsrzNR3`0m>6#pKfsH5 z#t5bif{3UuavxF~X=0OnH~zNjq@7~;em;EbjUI1-`$l0ksqtfNZk9H}u^GeP?MGcA z@he9Qoy{h-prWyg1DG;{Tqgc^67_!Mt)9MP%yt^yddKsfpR83Aw9$o!zwY2Ho|cls zyRiG&tHVp8f5nuJtd7-;>Dll~_U-M8*t@<))l>v;W%Eu&?eQ{eMmwISN}>lNj|~rN z+MN^5^?c@79%?!^U;t&s#C43e^p!25J(OUisDAr@C05V65~rMR;!S|~-FpUkq}gBR zyFg|A*3~PC{s8@!k2*%nvLXw{5YWtAS)eZbZ0gml(Ov!H`D- zM{AEGs`Dp8?7U#t8=^S-NPNsz0lq-KK;7w_d@Ekt1>V1hS@$DW=o~~>lAZs!IHGs0 z0b<`@-8cTEDxXvcVz3_iOi?7J9iU=4o;C(eXpNn=+@s<_6YIpTKNwc%L%tQEA~ez{ zOz~L$CYWeC7cH<5LjK29LxkNx$fJosNg#dTpd)mgx0R*t)6Icj!Gl%KWGo-}aC#OD zo%1aZJK-yKJIy(NvRr+YwKTK4Y|JY^&gW!}Ye=V=p41A+kgM2Y@n5vYMO z#E0PUU2;r^{dVZ*?Ba=sm^j%+xarG7Bq;ash5+|?UjzNe9-W>=J?I(w6VAUrlQNN{ zNC7@{$v``isLw⁡lV>R)->gMBS&9%*jdhWZG&xp7o#*VJyT7w-m8BJkj3O2M0pW z4Axp$?Ev4@;!31zd#RbQ&N8Uqh-g#c=OxmK-6HX!OS(843_3vMDcs|!id5MEG@x5T zh4;~-E5k1`L8rH<(={hO%3`pK`PlnRG56h-&}wkILRSD#${!KlX^6EKKlvMA7lj>Z zmz~5~Oa>de0#mKq}HY|%?k}0ytlaexGFPXKX~Yl^>Ar? zx!V~*TVU*o&W8Rb!a%x&6VxXe;udH1MgoK~KgUOM_%~Qt;t?32mBVd7-A={!`KN5l z^)Wkq#^e&;FMg3--a?(yE*g_$m!<^14s^DaEicIx1tOigAx@vVKVes>%*#$k6L0RS za;!C2GptN!JA!+#yLblR?)}nvM}UUAH$)D8OOc&R#t}&1j`S6|lLK0W8`|Y$i*vQn z;z7FM18(E=vRi0Xs|S^9vLF^`sY&h}HANFl9`l$*=W_r<6bV?b19*>>QiOPebw!A# z25j)P?H9{;_xjjZR+bt&oNV0!mBGR2N($69)am4)$NusXM~oFXWgLTBv)qdb?suLC zTo!6!vEAPQ2eGhIq9$edV1d-lQKA9>qS6n{JH5C6dPye#-H-hcu>GD!sVNgJ9QPxe zKe_}9!l&r{OZPUp^W!>lw4FXnKK-TK z%2!9LM$%EG#@dO3y@x%AeLnhQQ>-8Ae6icL!r9~=^bl}!bac>lk(h_k#b zIEJYz_MbI?OW3bR((J$PX%FBs~BD0OgXUh z1s(h%j!uGT3zlLSe#RewS!Bc6tRSJqFlsAZ*pNjw?(qhCR|sb^tPfU{JG=l0!lS-4 z&-FZyT3nbL7oT`vK*`TlT|3aNvx6ByF4&AF{ZvdGEzcuEp!sF4II ziIMqFK5m7LhuYDHEVxGH^J;Y*;ZB$0pxfIr7b~CsDchkSB=inUDvji95B z_y0&RAbKfeX^&puM1 zJ)l$`XREtg#n_7V5l8ZADs!h6Eji6E>Kc<27irWLFfURQP#$!a?q_Q``zMOi=BHdN zKQb6FUNZhoe?SQ%(n+O`%D+4OGx{8zQDaD@^3ev3hzSL{kW1_Xtj$-48i5*b>S14D z<+uWdzIQ%oqv9j(5Rr?U`I@dP#BTobmt7R|K5cSF($gtPGqMdc z4)5*qfhC(x=dZ_@?>(ab{gJlRUJygHcm*7Cz9X42%sW{Si?*2NtNNMgKI8sMm#C{8 zD^mzCNoaqMY63n=123uBBSES<#!!m)jp>HxGd!!0bUbW?l_!eXj1`DHd>qHh*@-Xl zAdagb;a14IV6&I_G&0T7`Yb6 zBND$vmuOtSn%#MZ>GZB{ExS|9zn6I#Z?L0jf1<`fOj(1)5sS9pcn;zZBBD%{@``ib z`vYXh!slm>5hevifCAPE3ELY5Mga-JlrA?ktg1|?aC-Td9cA0EUGjZ=O?3_HQc6iN z+-b+OQn-qi)X-(LN1z(HI2!pj!H1z&(Z@JDR%ps~chx4l_&Ak_`cw}!M8$R~jcZr= zdQL10@->=@N&3Qj8_l&d&^OFZEyG){cBj!lHWb6!XkTtv5_h05oiUNC3nX!76g~|% z2hvZyAl<)#!mR}Y^UGN^o@LEXdf}vFuUxVGapPWKxx7;01a!t_`J2VEbv$vuNS)1Y zESp!Lf(V<6W=ENZe5suohZbDx8=g6?gNpPc?b+~>h?Owh;O8IUO!47|A5=c{KK)op zJRZiGmiy2|JU5Cq#H63~BYK_mHXtddr5g82MGUAXm#sTgsom3uZNpisED0nXHnSDK z#f-Q7+SXLekuE8FQcQ*Naa4p{qxAxe&Out+bOpMuPu3-{_+N1_xdF+dQMWda@PrP@jc%6*us3|!Gcr|^or2!ERAN%;i+R+Yb45j z{`O2&mIArGJD-U7#+{>@(yQ zQr>v%#tm`YJ39p30Jigq7ynNWlLPV^n(^aw)nteL^2XT9DBAYFS*ZlOHD#}PB!eyS z^rds#Po!9s`Ndj`%I-f?S_i69B9%fe;ruI)R;npP13-=Z+E*%sj|?th!^B^cpMMH_ zhH=k=ya3>Ae(}B{6PISwWG1FW7`BdfgdL%&44(T7_g{Yx0x1Id^lx*-`vE6x&pS$W zax2PiS1FbmJQ0S>x4kHIZq{ zMvT`_^Gnh*)xk(TeB5XEPVk@XHDFE8&F<1f+({=ybZm4S#ocE1XYeQ6Va+ZTZ(tab z=JIvwpHPY-K?M$Fte28h#d7}2#eaKlswdmNA=TY=WZzw0-yxPTS=N2X~k(y82KjHqcpN~cKztCBuCxZCwh;|iB8y` zKLlN2+irMIrH^GL8JT{&v#!$0J$(<1VSXk!ZFklKhI3_C{Y~~Aad=Y={+nl2%~-DR zV>%H@TXkK$PCL@#c?b-pWlrustM}|g<p%vLG`t{bS45fCOqw;FO2oPl={(^)_J|3G7d#vG8kPa?%GSht?ii=g z+U3qlSA!ChcpUdS@VkpNeaRLZc^{Cf*&~TLY3*@f;d#F$NzQJbu0nv7z#K2NLNbw% zaoJ@K7`;<_#7dAtL+Konk1ca7^^pMW-ccxZ=$lUx?fLL|MeXX`y7Z4h_nF}FxgnWO zp?|}+-vIf_IB82lqleW7d6~yo6}$aa!;F)U#6Z9vX!PEnB$^VW5*V*!Mw@ThRPR0 z__?}WY2lKNRw)8`siTodxREbh#X0LYZSF_=hIKAMaiwumJuM$rUgN}w${(_GIoh18 zow7URriI35I0Fc$n%E!Lb4eL^)G3le2)ye2;AM0^~s}L448|(@X}ZHy}uJX z4f{%T-C^6lqLW$M&l4+wAG9RTGHc--?>1#uE%I&>xcA?S-#z``)r9&@O4flxoina& zzPk@JkG`2g+7jk^rlYs9by9)4`huLZDE&rfaBOrRM*|Xnu$mdI^`A31oQ4$Z&){PI z7(YvFF0EY*CvJw5^0~Uj){*Xj(T3UH*|`j7t9u%qDSfQR_L2H#v==H~Bz_`B9HKd{$V0I?KQHNjLE4_1SpyFmOJT?RUV7uK`hyjq=73PX5x(13ep_CYr_6=c9F+I0f)By zIBmn)UG9(MN=12k3+o>XGs-1y`L{h_Z}@y~!4T}WJzoN`WDr0QoIVE!PCkxq)}UCk zXHKqVL6Gk5)f>ZUL-5EW)3wgdC9XhxKBI(c)DvD!V8z^j=AxO{QD;h?tC}gZ^qb*h zekA{_z^had!2#e6`v6m;R&ouVUnCFYZaZfaimBELYvzQ|pfdO~xa#+36*n=0BJcJn}3>P#%1Ky8W2~l3X1jF zr>M@QyS07ndM^!=t;c0bB5ltzm|7-+1)?52X0cSX@Ijj9ngt)9Ebr| zImfPnuZx?nJcWSryjKBaXMUgS%VWE`s~brr5UBU)a=`2CudZKz%hyVv6ho2|j0@Q^ z3Xr0d^yZUY2*Iv3wOSAhvLttU&|G3*p&fv<@1G1sj&I*1|EvXsuLD%Hc-1$e=*W_y zI=fP;QbROhv>*Wt;ms1M8yhcJ3h6%w0wkuYMdcN_xNx}Vrn_y0`374+#hyF=lrpOq z1!lR4U^WF;Mz=ol_n;7xHVNhWDwTX1UxCL;~6)E zDHtUcGs-~o@sDJBl=2#|5C%I!WLhroL9$I^DO!82(I$CRc z3IxQy)Dt<8=W2KX8GPR*J`b$v2_P161LHzVP{lMY3dn2+)6R&?Jfe-Y8vE;vCBS{DHlS%xJN?(xSb$=iPMC{rh=OTL8~ZYhx$f z$i$D}tYd5V*V<_s4G0PkNV<8am3%UT*~ml?55de~?{D-k>d@P39uLQQoH$iP?LYtr zv61(_%ci&#OWWbCZJEx!sXZWqRANr+MA=PPr1|V|q3d3rA0spkb_Jp^Z{vehN>ziM zMT)Bys*?{eZ}M$y#h}o};(6e!K!(@vYR#Hrs9GLt(qqMo`CVO*zr&^y zYfyE?Pe*i7`6W{|15~nX#ol*eG(5ORM0hta^Fq@ zJJZ>L&{(;DW4zdivfvWv=Kb?S%*72czQ9}Vl=HKl%h0nGK9@P}sfnR6#=kQs>-wnA zktI>~rL@^tvY_*x2-+{;wXq^H&oCq0+I=MJ-J^(AQklo_Y*mb=3Pw0SNBVO|jr{c~!f)@_;q*#nxHb1!R~*qzG|j+eNV>LlWgj7~{MxouIu&w9 zm&Qx;=hV;21v&>>3yz}1i|LJ7`u0<4e$Xf1Kpjlrb@r}@_p#L%^tBZ@iE7}-wcyS7 z0FIvq={4e!T^eE_%i-wZfgvCLD+Kq`^WY!BMV^X4nLFISe85&ieUBwMafJSJtV`YK zF1bkjE?kEyn(-xIZzID%%uUJ}GT=nZnizyt<+7mK#j9xAE1W!e+cQeZ39U*+(g^~3 z)iq^4{Bp*k5aJ1zJLxmJipjyN0sfM*T@iVA6W&`+P-Ewm>o?03W|cl>;v;queplQ( zjK&LK;!~)~f$?6IhmPX&AE~e!ar*rph^@RQ1K!CeEkDFT6ryGQhM^46glb4QajOy+aDj*czC zJT5F|LaCEDUo7?f`lL(T>5arEB2AS|MjDyW-=WmeOSx6F6{doQ8|Dkdb|ozsr?n)q z1!qE~jp#NPPZ|~y)jKo`%~4|Kfz~>C+if-zEd5LY8~DZc4b8A8Yu27urz_WMlGryV z!IJt`E6WRMJQNe2;@6RS$v~P2Iq6k2rhK?g)bD^>3`~9#LEyL(y2qhG;9unEBJ~z4 zkN7|YUa^`jdza7{>-M7=0?azxjjSYc;|CjmtwTlFr`pcG5zXwQ(UA5jf9J^?I53B& ztA{w|n>;@dSYC2`BaZnImNt={+n@2;xPb28#H!xQT*8@$kg>o=sI4)D#{}gRPp<$H zNK7zgDbrErbwI5E=lE+hBhuj~SMrMpE16?OcoXqxDoUwZimAojo8+x@NX|B18VS=C zto-V7$B%!x!>ed=gTft0ZK=zq9p&>MI0K0zODKgT=6{#5#=sSyjB!H42anBB^En1z zjslf<9!9!#IA=!g*9@+(mRyOuxWv_=zZd^DQ%>iSLZM-2X@Wiuw#T;>HL!UJXcVzB zC~34e8E3FDV@*^J-4!D`Tw5_2d9vLwrrIV90_7vLJ8u1h93@}FRv=4YBv~|L5|E5i zVs#_-J3mb036=N+st3M`D8KuWFP=Wa)GM#w_<|)H#51_WacIUtRy8gyz_m*#e;(tU zT!Q3IO?eSu5T8p--0TyR7e_)L|;Qa4h3rKQ9t7~u)ysuwA}w~gqS)N)j478BDyHxRFL~5oQbO{%`s}BH^R{yv)V+( zqblI`9}l)xc+DT4>qQv+`Wt`J>5b=?bxh^@UAXR3RF$}#$sjIc!BQc zjGshll}D`w9idD@Yyuo0lZ*Q{ z;%@^LMBGR)TAm4Bwu=AK{l#&8Zb%rxO0GWIlq5YN%>bz6<2rm*UmOcznz!gJg`B1qLa^zsJ;Wr*ISjJGnZ3(8H5CX|jom9|dU!~1AQX}wMQ zpTP@PmkH0?l_f=h9NV$ybAB`sZSngAf2yZx7ASrsKYIHCtCNw^Utt(c1GF)5TVzwKQ#XS@8xaX8)?)#==!=oVkrfjCsjs z_#5ig5;YO!8&UL|Fi6&twp|;`?|~?$WNYbx;dMn#S(1Vmxms1$C-^WY;%MfNaUf?d z;udT2%en5aYL7I3J8@R>n;-&LjXP;N?BZ*kFs3*-O4e1Q+$S`w(ENNft}yI=DTw+o z8QI|?mr$Hbc11ET6O>x{Fs2zN7UVx^wBVuPyGzp(Yq!y{yuDz-n92EL4-D%5t`swO z7SqqA+eP;kkXjAR0<%jeVf!bhQ8$RJ=@`gZR@s3ON|BzvhecfmT8qQg<=&;5*b zzT>y~CPuKIz2k1GkxVJXuk#;inwfF%x@}MMc?GvHtY{tfWw?PZg_$6GS^h3)slqyBXa;WFo1w=t z$3krEIZ1CV=PFI83cW2aL9xAToj`nF98758M5cr=<9q&DBCHfPGU_9+tw3s??meT6 zll~9N#0(0qB)b59SY7)k2f9PvF&O1f0v=`}Pn15;{aOp-$bAUE{ff6h6|sBb*8CH@ zTpsU{Grcks8Ol0mN#tf?c0khVLGlloM8Z4IkK<#J)*nm~GiD30%2ye5f|M2Zt{6v` z;cp!v<)6cG=of3+;DtPg0)qYBl>_4ueEg z8sxh_6g7nIGI|E`w;gh8kQ)IQYolVZ`BSSko(JVZq=u{ywRuHAa(x6kP1tG|V?@5IkuPV;we5H@{hY?);0IDRw%j?l4j-k!~C{{Vz>~}5l z@&?f4xxvBq(q?W02}x(OF%WwghSgw!yy)fn2o6-b*ME2oHxQKE^W? z0O`N&UkOHmqQMMh*Q{DT-i`DCIXIiplF%P{`E4v{EWYCHJD(uB16ZqABndae<%ZEU zoaIKjSjUE@riWcqm#4@fWKTXgM(>Wly4xq+HIqB3O!5T|t9M6>ER|0MXY_X4%sf~E${PjcN;lm;&_b`KL38?F z83Fu=F(smuz*wvDOj7MqIs_!3L6%}Vah&2d@5bWkO(y}^7>ka_y$@dABsukrSld`y znU!8@>lg!fF7H#E@#C~5aFgaC%^Kl4tb~nez+*tnEB2;UC8taJ3A7UA3JXCH8myc& zsEtHq@F8P5hMw9+=x!NpzsZNA?F(}CZ-g5l289@T{Mz=*LxH~^4*mw<@G*(i@1NeR zg_&Y-0mdYd2JSNmRw7gp6p8*yP)Iub$&|0uPx|y&K!rmkF3HmWgpb@H*YlhP>0jQCCv_*VI z@Kh{kFIC1V*qcoCoB0c_kC)6e+~81BZe7>g{}k+sVSug2QO0AfpC0TZz{SKt!rQ|Nw4(GF z)mUm-fQezPCE+$-^wV-0V)GpRXAn(e0;!?R&X8}j1&M1=b{({NC|jg@m_toU`?$DJ z3bzg6?OGTLGwv9ClTc=Cw94Sz{duIKGZt7?l3_C(@JZ zNYz~tvUfGcLZEMF7^KhV634BZOTfyAuLR-*XLZ?YYH^bN#0USp0r)(vp99i;s-gS* zrvpHRvx6x;pBkBc-!UFH^s<9C!v3)&S`oCZf(?1zf!c6%(?OMFTP8=!_4Z{`BakJ3 zYHI1y7uK!AJ;=v+$8_klXZ9~+PN5Y4L&QT+yBkuQFP7NXegr&}2bZ^eI6NqTifDi$ zB*NFR`yd1$pJ3=|MQL;h{Fs+uAe^Juz?YZPx5U6oyV@hFG}oe)ukOz zw!*3b0d=jpEQB&Hr#1NXb>7YE<#Sp6T>-UMVDn8-H#d9W6och+>htPp1Be-|q{nON zJ9dyivIh`6a;o9_n{4K4aHr{lBl7XS4jUy=r0dWsc9(RWOceRIIa%bw2cdkRCj)(8 z>HHxYdnJQdy?6S2v#Ym1OjNUzv^nrpnd+MU3H1t&o1ikN2NZ2`mk@1l)`pazQ%1%u zDAEH(Z=H5XmDa6*ULp&4Y$NII5&wIXiFZDLCP=PokWE`U#1j~*@c~*B|DR_$$-}Tl z7$NKjEVITBDwI3%1PqEVL0;9<@CdYI*c3$n^G%TpJ^?OzgOmm#OD_0 z5<3y9d3h1W=huyKRd6tFwZV!O7YAW0DA%{tONCO+wF?>I7F|%WruKlfe zX2<18k5CXc*_fqbG|Esc)x(x(9xrmrEc~>A;bM`~R%(+d@-pJh(<4#IAC`CzIh%pR z?jhdy7(o@eMK`+8NU*~$#tk5qk#F=K3C-6fQS!9gomWejN}4DTaGa{JPCN!xqS^{s z^7(hEZJ=L45`q(}yzY5%mMVE{crftziw;Nhz=n;|RSSAp7K}T*XCVuK$3w9|#ygvg^?Q7%`YzTH; zv%pu~*U3e0g!WL94|KTAz{GlgL_SWZiAGhIY5fezE#HX&bxh~e}8Z#_x|h(?U5QdwA`(g?s!zp3qg%O z*48ATvzEU?U8+@qmc%E`%^;rcw|Gn%XV5@orQxOeiU|1$Qs{hlnxKHENrb!*yuVjw zi3H`L9w*bGr!QUN$=f1O9I_UE|5HXrktmB%QI2To_TH%wNGSeN{4r^L_XWtHzEq^I zEaK|VNHS8kK6;RH{b6@b@ai*f66jLOI>IZ${qdD?$5B|RK&qTmzHi59=oRPH>4c)7 z=$N*ctYE5EM7ZAI3ZKCi+J2ua7kiy{CF9t*Es134dZOV#{DbiOoM;H5J1gw#6F;yt z;Zy$7b`MNu@PgVeJ1Amx#NzXc81**o419uMD~+p>Spq}@Hzh_szK>X z$92prvG^xer!S75GI{ zHiwu4aK0}#jgyZif|e)%y5kR5zB!rqW<0aQFji9z)GeL+%As!J1LC2J>;dRD5f&;{ zfSn0mzr_V4(wCiWyYIPMxM{mhFDFp@`=~mDGwOA{;*rOXkRuCC%~kvcqNp9eYcDaX zdHupOoZ~u(B|;>6+CJ9zSF*qspXFB6k#x(Dko#Pj}thaFyZ8f*B9?8Aw zK*|vd#OAO6))p^fYO5h|8CYEBG_yL-Xu@*gFVc}}KI@1}Q2>@mxI`5R9p*q24om*d zGk8vikDr}=ohL@46mfK<#EI8J+PMML%6jjM#6%US$0EeK|6Z|GrV4%1-EGhibo4Om z^!mHbCR!^h!r!8)@rLu3x!2`pU+KMt73fygP|xbW$lLl}v=`t#58G5=Scy^QbF%-R zZ`#9{*1z&1aI`SH3WT@G{+7k*8>Iia8^SMqEck3OkO>1Vf}pjGb5PMTo<}Reey6S7 zS-SeWH0!vcY(lIa=FY~+#qQ)&ahoNPGed4ALT%f5dFK9nDNN=m37x^X*SUP_d7M84 z*dVj;VOh;4+d^%AK$gs#N6lUnN=2lPn$_XWL`EQp(Xd@JXPeghx4@eMKCjWw+;sx_ zCKaBD=QrQUKc)vBGSA!Y&ld=R>urU?Xl4AQ4-ADxPCgSg8fI zq&d4E1Fws(=nuidI+OnULoj5UL$B4s0Rwc+ctrBHT;WpXxFLuMtG^r_a;~q-1r34{PiUrX%Ln$l&!F}ON-6W% z29H|D;mKKcgmvPhk@!!r@`XhgUAy2>K;7Tza*`OFfhb7Y^iTHq;yYDeByeimK|Y#l zX6yEA(8vdP7AXy>#4yBO0ClbA0XcsjaL|oHo$MgXnhoi$z{zUlY|k#sUdhzN%mi3d`6nqf_$Hav)dS?%$ zXE{^fl=C{-BG4ay$~g6?TKs&o{GE6Z=&_sB119ahc-e^|>NIlnkD>Q&hyFL|d5Qcy z@Zy+{7@Hwahf(vO&k%_I&aN$>uVc?eX;2_k&n!;UX`Ei^3Tn(xV4Z@bK(c`Ekx;OX z;p^6@y0YB})QiSv2k28R?LsQ}9R?7$^_yku3^&H2R#Rjm?&SkrXMB-hd!(G2HP`fV z)v_ z__sGm|McG}apO`w0soPE5Uyz%7ZS6cxH|@&dMy^Q}r?i?7W2s2WH+ zJkr#HAo!J!fY;F``E%8D>F#z-f;GLDzbhczwKC6(;;n%~D$Y$H!&-YkehN=3#bsFT z)a@;aUWW*2bMz#op~cnd^PgI-;?N>$6Q5=P39shhlVd|`JIE{+TlssDiQEeVblUOO z8l^MnV_LEf$bFuBRTk09Na~M=yqjF2D*aru1tLrkJ~pE6+?;2W7=>E0^donFOW`b) zz-=|;f5$C@miU*Do%Sv#*-4&M+}HbIo4`EO2<3Fx4(dnpy*zfZ^w*OoW{94rsD1~u z?jG~W69h)zmq7d`RbEv-rN?sen5Zbxqlt?PGrFfw$!6kv-vjBO%E|>ZLm1a|XBzz4 zg`oS>DJUl}1p-2e(Kz$%b5|?&r|v_k;5P>NBJ-(m{|(KAEWs`^R1YO!-TVK9@b7ck zt&irNUIF23NMJizV?^M{mTIbpst3y38-T?Hevo07$mP=lZ$|qm=rR~&996W?n@X|kjBet&-&pcPE6mR-^Vi__9?VUGo6!7Xa5A>G+ z$aYWZ`S?~P9dh9cYw|_NY4gom=u3S49l*p?WR16@+-g$vOS=jXHU>AZ%nS<%zouYO zcl(wUHr&Jf1QJ5a7oPaH5NYCmUuJ^5NR4sh((r0_&jYPyLJ<-q5ApUWz)u65a=%dd zl)FG161L7PmLReX6nHCVEU*^N8YpM)hy;x%o?l%3J*GLK=)Q_0)bKX1LD+OE8@hEa znt0WGXSo)6mr<3MVb zsVubdmofczSD=Hxk1=VaYl*GvjE2&0_bW@0d(lg8Lo#Gb@GlcMM}8m;GT7w<*p^bJOk^+Puw%2WOKpbqvnQq; zs^4UZszBLhzg|Qa`vZ{`=69?28bBmr88^qaf2BR`->36HSu?sfv^rr+x>dHxqY06R zDobsqwFthr$dcf-z;s(-B!A%ZtogtMfP<5$L&GRYyZny@x+2HFoA%fc?(om<<3%{6 zjfWqTHGhTAGwsksvF-~DinvWqFjPgF_Bxurp0r2flbl|^!vBC5`0qO|M(I|dOF`1g zU)SYBvnfo1K7^Zr3bOrZ8thq*IZU6S6r;ghebnJXBEAEDdl&VN<A-AiRSN->=Li7%&Z*tP3!Wf ze211|^!<H3Ta12#lmhIJ-4VOLW;tW)m0~omaBxw z@elGz;M^6{9#P#@b=Cnz~WBL7>Lhj|kj>%rC_<{G)q`VoyODYt2%J!Q%aCC{S4#hC_wysb@!l)hL* zPX$at^-|iEIC90LpAPOC~^w@Df-<_?(@N#7vvA3FuQ_Kgk4d+ zlAU@P;lu_yB&MN`QN+%n$65_t3qAzJUZ<%Gk&}^4C?gr!WS&$qvLYiRWOq<@ghJU_QC2qJ*WLT`eS9Ck z$FIlx(I4J9=f1D|y07c?daf((+Odw>{JJTxER8371{rH?kvpEsGyWD0XGr2gzJyqK zCdLjw-MT!mb2qh7YTl@$mi#g1p57>=3f$*wc>L!@@^7L+A+)B0FlBj*eC1_hJ4q;y zrR^i>RxVz5xVP4&Zf%AZx^cD8ctKDZjS}g?jGaWqS&pfjk!aIg@FYy7zbj1Q%k3t( ziEL%o&)OWx)W#XkdI%Tf`1_GuwHvA|D&HGIRSUH4orp5^-mJg*y&cak&pl(Fwxw4j zBqzT5oF(;dRVAfoXRo=I%U|)KTi^3+GyRU5UvUlS|(rmA*+i6PjaGpEoX`tJh@teg`u*AgM)3^vL@AJmQ(x9^c zphCr4Eq5SU+9EK3)>Qn3hCX=0{j#=NfpcH;&iUK0uHkT+vRg&;{QZZOX|UBLe7VV0 zA@t$O>;3r)>a8r+mj_>z-8GH~`pUG2s@?gVr6E{*qTd{E72^~>cgIh(*7@47ewAd4 z$jJ8U^tz`Yai>F#en?6P&QmXXLZ~5L-|$nQd4UEhSZj!3v_oMXik6)Zx(pj!CL@8b zG)(`_3n=xT4Lh=z1Se_At?m|73v=7ctX+YW1wk50y_2od-e#m?#daTC`R31WBZhs7 zeD|IF`%!z8{XL9626t#wRA-q-HN2Y{>Ec%E^eP#Wi*2))1kG(an*^N)2IB=wn^Fyb ze(rt}FRTQNRfXT=l{SDi#|k)&c?RjM>AzEgue~7D;KPe;{WfWrq>+nf&etw>nuh3; zC`Wr$@!zGI)WbCN-s{WKyc=S$YWaEoxOS_t*{dK8;kJzI{%E0;i`4SOo9mW?`ERqX zaGL0NipLa((qL8d+oetSKO@}{UxTF<6oi^blYI8ywh=i@=s>luSs}VBpNd8?`1-S2H}Y(3gU7{SpfxWuu#IT=Dj- z#p$Sy3dkm04R2%DHMKsuEE5J^k3yDAP2sw_o{`)$4k#O*0zq~Xq zwBNv4DN-EJGf26&_50=Tcl~onUV+TyL*g^nLFeiu9uq_dNc8=}e8MhFwSFklw(}Of zAg)0ys(GBd%YhfUJav#dxu;3HVH;R2e^OR}zSuG#?OcfeFI2*De+h_Kek)IMi%W(49oUgZ!N>Ohl}2kPW} zbhqi9BE)E^Wz*p3nqAgo7Zv56bMhxE5O6v6@=yHLWl;FS!11UH=M@b>Y|j0c+_BPJZy(}UEj&ND51!&?K#~lA8vNGcJ^>zO?e%iYL%+{^u*9urM9LrS zdrWs)KIjB?bWnYVwH4-$`8Eq=>h!x8?{+rU?e-$s5r{Octts*hIUts%if|V;_B#iv z4o+|-ujCKbL`7ZpAb}@`h?wjZz)-iIA~y?RK40D=d5ecU4Ku3GE;*SZqSR5;fM6gB zq5Xs!3ZtQgcdGb&%{MK$5n!J64Eci<0R0fUzdg=Irdk2dnao*8^V{d?SHKHxJutoZjkt={L5 zS8me7A0$+-zyiSM56VtqFN^=a<~(jSC(f+`X@2~?J}h2Drp8%gk^f#l6R$R=JFgB8 z_VU0WT=~T}0$Fy5DbqWwqB80rp{1aht9m5x=wKk8Q~CVBt=U(``X13wCBYesfG!)5 zaZ~Z-&IkKqZ}t1_Uti0R^pBl;elGSo&guyxQ|k{PwB(gyFj)>TyaxaK z>UanIIm^R)V{Yw!m(5nLFoG`N^)sBc5t1y$Y&`^NN>6dU>E{2N!pdo+m~Hshb@q6n z;_57u5m8u{?7s;$#!=%yp!OEOv~{jvkOub#v;hmS@c!TTiW4-Z+d6ZIcI_w}f*$#N z+03aA+`-_H0`FacH>`I&!si-ToS~79QJnhk|5R?vwe)_@R)d*cPkv<52vpg6Jm3D! z|K30CI#`hGkrDxbyRJZ!{r~rA?-4^osr83`kOmn1cjX4$5F+4!!~h@y?@2>QHS51< z9Kg=ml@(cc8#abwB$!kCximU%+)+wdy`OfPs-r`wJG9YgOqM9 zc#Qa+o}!*&WZ8MmSxhfs*OwsyI-#q`6X<_SJBSIv_^A;E4MFzRUAR6m_;>oL6I2xD zrvWcmLZH@)@qoQ=-(lMnzG;iEL_!dL3(hwENia=BCZVRvIr>18`NkfrKZrr`I~B~< zlz$r-k}BkNxq+n^I7oXyot)@!!e-T_|7P8u>s#iaK?svdIr>b$%{SH`g1MUFm1H=- z>Zf%CW!`8Px5_QHBxNmEN>kg9CoP0(U#`_B4-5Wax4l{aCv6O%SNrc?!`g@r`E|MD zzoGesm}FqB@s|PIaoztti$=n$^e->0={qvQ3Jc6Zp|SuBVdqvL6iSy2tvVACi-GLg zM{cKFn!AH~HXNIok%5ueTA;}N*9H}28(B( zVhxvn=sBb7z!P~Oxvsmv%6lD{r#v5Tx6eXa$n&@l@^E#2jfJru#vraFAYPp{Qld0; z*K4VNgB*F}PUC>=4E7UEL4j#}@aZq$)42%8$DOv@S;xJ(`XZ}7U3m5i@eLtZ?ArVf zzLp}m8aX6mbOCS)JA;zJ%J-9qaKbVV*~Fm|Kp}Y70+LsvY=hFaHM)~JoB&nRaM8c* zqS^@O?IHTKe~B$v-V z@nP*JJWd@1GnBjJhDsR%8Oz$SKS&PA|E9F*vDr-|K?(`y8URU-@e5Gzd=c)5OanZk zV)O&*MxV_K8mt&X=I%H|hL{?l5HPhKrunS$gVQZSEu@b!0q3it!3i&y6&tCQ{m7}m zI17&@F%0rjA@ikH0w7)qmmhUk|9(R-j>n%kz91XU0J5c~S|-fpo2D&0c^TB}5e+Br zu~`p+<@3YNpU^XLVvgnij)WY(kZ?N*4($1=T0Jk*pUT5?DnM`s+Ij~aQtX=C6 zLP`e@(mo(!nXPp;(X4~4d!-WjS~laLKu8oSPw{L_6A=2=0Ia5&VQckm%|%$Ir++S)?fQl5Yc=sc5CT-->B0Y3Twb$jWc7Nz^hRZUB4GIDZ=Gy}I6IRzLW( zk8jBQUbflu9@Km3Qj_rp{{bL~#|ykF@`Q#y z4Sⅇ0ma)!K`6K&eiR5Fay(g>n5J3Bi$+TrF|MGMUa)oad4?3*UO8UjEF#I9|YpzvASAG)ag_WI;%w2{s|$0J%0nfAAxp-(xx* zA`!jRiqA#1ECgY%1K*1clS@#GW|E-14$*_+6l%{q67k)d+vVOFd+BmaJvcAK&0=wHdW1J(T@?M8< z6m1#kde`^gb~qmH?<_$3(b?PUHeKH_jV!`fHpIF`cZuG=K+g`3^W58uX zJs64B_T7tKLnN>eumsLELcEI1?J`4#Y%Xldc7$q=qPXFh;R`9tc4BoP)X9PEvQWmi zU<^zh_Mp+|{Jm*}R5-#vp$h4;9r5#}Fy9=PL8xbHG8)R{%o}Pqno@^Ady)2mvL(_1 zugakmN}sG1J${z_1fo*9M%@BYSn^}JY2;U{D9iI7Mb4hIBj+c_%y*bbdM$pr6x>gj zp~8EP{4zvhCVil*T|EtnguQppE9zYKk&UYX8mb>r4&>4KltCe3v+kFx+T`A^&Nu7C zeQe4X6RW}6axsMy&mnQ#hNCRzS38*Uzg#f0e)&y=O((t#cILd8OC$MZ>><}#6}Q{h z04*^7aUwlxZ$H}WGa_SwdOdre@$X}2_h`~i$CmY*UgvBwgLI>OwWhyzt9)xb9Z>yDWiPf_sG9tg{y&z;|xzV~hi!O3ZHNW?%J!bAg zbkKwVIew+THM$7qlZs=gUwSDA7a~0mX%^_;*Q|Q4rA7KI_0E*aB(4BZ#T* z{a3KCAPMC5MN!T+w%Z@%`oBl*`9VTd_g@QG1%1E4%nKGfDJx$ef zzlmSZlRKKob8?ud)%^_#x;k8tmp($Sl+#ROL)P^=RLd-|Ha>1~0&~+Vz(%o@S~+j1j;n! zm9T5BN}H5HHw?N79{Yj6UU(O=wSH?z!4 zT}CLQPT_Ayf~q@$@a|wh%Jt*1<0z))fALkf9M`R};`S@*6x*B{(9UGm^_Qvp)n@?d5yey%uyWxG zEI9r~ty)Y-Bgr{vAUZ2pu`f{H2}s#8gaqHKk4;LgB$UMjErwn7tY1ha-qEvIe7-DQ zHI?AjKpu!vPEt;qGPCC!dB;X=>=xYJDXIHW4NJfLDzy}E8;zEI)RmcxU&D(kBP*$C zZzq2sn%O2A-6nC0fC@>s6aGS(6&f8v5ka=!dhV3sv8}Rj6_Ozv$*_4MK#SC~PPZ`c zwGuK|(VPr3$bymZu^d?}ZE4@%ZdLh4%|l(6MVb&o9~;L@H9F}KX435bC_4&p42hc8 zZWcw^>~k(!SCSUlSel(J>uAxZaEqSi z+Heh^C1NIB4YSb`)gw_ByM0;Y5pB^!Cu*D06fEm(mfUPYxMp5QW>u3HR7r;~iseTq zuoEMlvE|B899Nl2fRsrN@-` zIsQ^j)IRQeXh=!_Yd>xxSn{+;#IMxA{lrsQMaiYo$5Bz#GH3JbfP%M|UBPOmpwz`t zQ)j%t*{1RuO9r)`X(7cXlbq>@CS{;Es_y5U8Uv?v0gBd{+Tf$*&^5x?ajb@QGBLTD zN#x|~$5x@XjJn1o=S+Mn7Y@AhY>32`O zHJD1%6@4zhsaEP=^_U;m@j~AzX6X_6wRFEPq+{1U(QWClzFZcFDuPOhTy8rfNQ87X~CmDyhTbs9DB6lZGUX{yWEoAR;S-Vy@ZStIb_=Mt3I{Kug zabx1o1y9lC`J&j$&kf<7qnajN7u)b`oPpm~r*7Z3Y0&?yx45*V*_mx$iH4EYDz^*8 zlkTWVlYJ$ZFwwOhR2JKW;#UfT;^g8W=e$9a#Jn2OT>q$cfPw2CJyzwp*A05PCQI#f zZlWN(Mz})tGdGqH{zL5L5V5dR%(F@g*!Olp%Hu-611DD=+wtl}PbX53^AyKZ(Q73U zF{pG$wqGpjmX_*CUDx$Fsr%IM=y@DL4E2s*9IIX7Gd;~WWLp!i7Vk^o$4E+cAR>T{ zXglsKvEWwbAAE;&EV0J^3k(p@@gF>n!@?G;GUOt5ew&1 zgNn_6G?tJ$7n-%>6#Z%6MS1*trE_@=?tfdATm;-%%W}vh-J1elXGWVe#X22e9_W*9)35U>87J7MF_xTfg7YLM9HvYU{(+}W3>JQR#k4W6!l1sH?ujU^+n9iaZ-fSQk|TLo2BBS zY5!rD9h2a7mg_7b;LUE$b8aKE>OoHF&urA8)ce$%5}MpWe=q+8XLQ<7I}%uPDrazKEz8J$ULo3%*>Jzc zBeqlU`z@1ulj&bk(r1=Km$(&6jQ?8>&xl$!^!*`Sgyg5|o!DQb$Wn;cm`Rc0I+I(L zfW3F6NQ?RuW0B1##f+uqmeqw$4jiWv$B#cjXHuo7PIK~GVS>XypaRKR$dv{a_Bq+t zxaVtd?bFXopDUIs*`0^uH)z$UmB^b!G}%uw&+gt7GA`CSeri-77b4p1ay;`Uj==ZG z&*o|JDLVJOndj7A)QZ$|w_ABjuVt}Gvlp#izThthGw;NXyOYTlh)Z%jlnl6N=roU8 zO5#JG#AVUOXO-$ZS~jDC2o#Bxx6aH(&GSlWP2J@@6`uOkTj}iGAa&L}(K3R6LHpb|YOzytPi1xEf!V%il1owji9h||du5dH(s@u4G zk;DdlR1@1rgykpm2%BJ2@U*nz-pXA37M-;i-G@pQvaek7&9af})tXG}BWPP+g^Gl}f4KRwrD)(EP}eMmVh_ z$nMd(G!+?M)UanPt+s2q%GJmuN8*g>4pV-eN-O4wK;K)_ff31<3DN!0k#Fy2ITfr( z^Yxs_6poCX;ft!p-TZx7?W=|DlTeQ9uC}tr1&rK2zm4~p9rJ2&1NDr2&l}0{{6V_G=jd0$HWH;Ri>ylbn_IP& z&WzHMv$&C-sxZy3E5Flwqu7H)jBj-h-Ye~_jKyp16Jz-gUSzAJn^6iFpKxPFl_C5; zs;BhuGN^DO5t}#0%bGOFGSSp4ueKk}b6vnNc(c~|`JI}#YS@7NHj0z0XP3lLX{Che zz+%OkxboMXjJk~ju07-}wNK>H{$~o)a^Wh0l{_Ek`pXLYhTzJ7eHJ)5^=S%N2Zd*}w#?g+9^`>4(~-6K^|ibWZJHuCtQMK12=`Hq<}Io!yMZ`QG_xM*%QdAaJgk&z z84Rc4){kFCvU($AS+WzZ51`iMZe;sEy;e16#-l~^!5*$kn-jLbi`tH+!lof3i$?l@c4OVFG9uSr#);xGu#}fmCIdGoZj2xw{a2J=$&3`$l zETTl^m|Mh6+BfcKAyaFjAu|)VNq1?G7gj{qA?}{D8~y-oYSZ|>ezN6e`t4=JkW#e; z*91EM_p2POJUd(2sQucXpDNf%AHGA1!Cc`P(ZHvl#d*aQ)fc)tzKf-~AhjUQE=ckc zdj-!$^|O6@^4^7=bQ3tWCeNbx?Q!gu6cySi0jw<&Q1eOf6uNNmAhIrQK;%jA)1OGZ zp2SV7l~}tp)$}f+4;7*?2^!8 z&nB4?mCxQtJ%xe871UFV3GG~8uXL7anU-NyO_7YiBzCtd*AD{$Z)n+JgbQQop3Iuo zyCrlreJpWI*V=2&CN9*B!oUoapDcV@6gNRwiD-7<&V2PX2f(gT(j7Z`?@*Ycv#7U+p&3yu8%_&Hne#wWIsek(DCIK zJcFL<)0fFF!`B+BOp!OhCX9;QB+bidb*fdMjlP^?Dckr=b03nIwDBz9_}+RZvqxZjJU3{O55tkKL&uZ*FqY7pwNA14aO7eq z3@g*_{PZCFJbjF87)n(vtUfuj3U`H_9so2&kGLcuB*pTT?cMBZGeYMGfH<0a0zFCS zTyKE+9z?#W8tM4=cC09{c7dZdM*>nMus|jqRFxs~rfoxRRknDX{ARdrmy$HBK3HIY zL~j7Lm*7>Eunn0HiDZ=~7G*jr*de5uMt9;l!~Pr1PVm?10cLZh6+(adTx$OPH1}P^ zRB}=Nc8u*Hl4gXeaHD#~{!qwT1Xm`T^#w?`#MyI-lSh#2^Vfs(iwittN?8<+3AP;> z@{Jp69zdzK0wR*AvkxlX@4q#R{_m&$_A(;_T&e{N0FEZn52pOJ^gFbSyCgMuQe-18 z4y|TXa$evKx(gk>4mJfIh1Ty6;*UUtZOD_Oq?^y^Ns}y;22?W=5M#+2 zj=rax#)j}Uxl$AL!b<`_miA@WyTosr6-f@8iDOK@>?)j-#{o}Z-U^m66F-1KbE-}B zx)L66Eo~pfH6CU2;J&>c#b*W)iu|(jo!kHSHPJ0{d%teh0sAJf(hB5Gl1z4`Oar_d z(F<%oE_C^F$?$H{ z`!JN#l(sZh;I=B|{W0?UR&(l9e@thn_OBIq?lr#@cJ!A# z`|Iu4&NY(@>r18t_x2lMD^c!qs)xRQ++Hft1q>t1A}7TF(VIf#>z3|$+RsvOveBc8 zIe+btI~dCJc4i}c$Y_E%^w3~@!opEs0w?u|eRX~VF4=1Rm^DRIMDy1)8{ku|NS=n1 zb@YNQ3TODR`7g8^#=pSY)e6p=XKVq1`_t?<|4bs`)U6QWZ5YiYo(twx;{J-&uG;2` zRR%LxMYL^4l0+YJ9Li17Ucm8u0HTOQ^1JH?Cl8IM;8?okLJl~pahWl5N2vnuwRUhO!4Lsdh z$a|^(;0RpUk;us<---7vOHA&D@vZt^zq1W?$Qwqch#23&W>LCu+O**Nd1NX|=Kof+ z)In?lpAsA(i~t3}KSdQWIF`FJvIk%o&2t<2_=J1i@LYudiT}OzhStCkv?6hgxJHHQ z5$FRG?SCQpnf(w=ra!ytig->E$EAYEK|^rx@5MjK87)e^9hw*bhG&Mxoo<(YOW~-9 zS>(hxfrG#A8nq*u!uNZ}@fGPd3Hbh$5cbOvYw^@m*R{Mgv+8&ri6ls$pbb_z=t& zN!N>M=QqC46R|!D&AMj9J+KrqGX!pbVAqI94b-ZT9btBI0VMuV^}WF@b8txX->9;SgU5Bo96X-+vfuEX6K^B>`^lWH(;C*{%$U|9ia!tQKuHn_0gJE4gOkSZ z>$cDC(xP>@m4e<=rU(J+YV+2h*}-<9KS!VsJ!FQ4jJG^XELl7r;erDYaDOEge_AV| zgar=M2yjD=3-ko}0FRJ(LwY>kJ|txG8_b6Eenj)o58z}Pj`7OQOkxy>8s70}L)MJ+ zDuFHv_S8$yzs4QtfT=c#VKzGAt;d4I zZ)j)rzZo>(E1o+^i}u8=8(=fo2cD?0eG@IC67?dOhM1yG+ay>OCPh6Ww+J^2+#4NF zxE`u-j_$>2j5t>Y=Y3jsYv6{oTF&jU0@_;O5nLnPvIHby`Z#@CQs25^AfFaKeuQ$X z52!DLMauPGRsXiT3icm0CO6pK2s8za?P_q2<6Xu5PF1C4|iO~Ek7`dmvswWE32s4~b z4|c%fE9fMyOJ>7=Os{L|gQ$g=FUrP7>$f}HAde1j6o$#9aH6MG;*YOd#|r@)GPUSr zS0How97Vn2*D^BO45r`^2lb%{#P*PO?xaKOeQ{j%|L05T>+fXuL2^9RIfMc&14i>K zkO(fj3g1qvbc$R>l}^db`}q<$lvRT;ri(N(~G$Qi#3+6 zbi@rw)(K}0^>OEes7@o8`0jbF_naxr;{1aHhtW*MGh1re_N6mBl4;TC7phA1WmaY3 z8B}V|t+7Y6?HUKw&Se$GzQ{=?Jk`EgoHk)+Ks;mKe>a@-hh~LY7uU8_aTv5{f0cXj z^a^U>4^Bm$kHX-@`Z94@r%?7!KL*`Xy_I0-R1YV=pmDATAb;>|{(_ye6Vn zmTnqROcR)#?@c?GLvRl{i>v*aR%B5x!+U61>0n4}f}dQ5lm}4=lY5_pxJUyczLhHkNj1@v=3;@QgJ=cpFY~X0r`ZO0j(D@mDNDMMIy$g zIEJp>XoXi#U-D_E8AP+oH%LRV7loYd+5M&K?k9qCmT27kBW9M>=dNl@aF_)J<9?p} zLBUt|PT?q^o$9{Ao*RW^Tw+#=fQq!w#o?xuD_~O!NKB!j4QeFky4%jfgR-%_@Yb5_ z4y{qkhNDRj<^5mdh4XR~=@Y>;^wbYlYVM?cKL0M=r?F}`oe9(*x}p(HC^YoR2z;P*Q2ez zBUu?F6uc?og=s2b?h%TYxyVL2Z~VD+vKb4BpgP1>DwO?5ozf4xdLklRG%o^HwG{+R zm?FqT6Xr>k{WS&4Y^jvqS5~EvM#~=^kln?YM_L z-A?tIQO=6pfUP$B)%}~1bCDmFl_d117{9z}6I6WN1ZVR#3NuvLU5DB;_fxHHv}nNsP4ib7RS}%F#*X&mds9S+741Vh*=Y zwZf)VUQ3r>v%7-3ZG#TFvwA8l{Q(86#ra!^zkgTocC@|R?yeKMlak(_>9BqL=m{B1 zT_2)0tQ`v^5t%g+V-!C-A=i{f+IgOA3$1SQI*CVRQZ7WAr}tr18`$9x;e;6D11AQ@StI zC8AWivC~V-X5-KfEG+rS2J_m~vKDDj8@Zn2F4oZ^Hyi9nV0zlUDd(_Jrj=;@(n4f>}fqAFXs2v-Zd;qsE2R{<9sGh`n-LK+a{L52E>ui3|v?SUg=RM|a35sOw zR9&td@;;7vhf0T)VRsMW%=!rQ;k&?yy;*YwA>$3JTdt z5W|vs5;nf`n9>WAL!FTbQj6tjtt;zQY!iNfl9hOMiUA&<5Ku^u=g^`>Jb$Lh} zEkTH=CO^s1@;oKomz3LDCCY#9l4Y3~&Dx1ssPi~nST#vrE2(Ut-`Sy*U>|rffB%v@ z4BIVdjO^bzD6*6tB zIjE)D@fLI7!O~iN-4TJ=%piI{MZ!rkhYoi*y%1IxX?uWjk`f04{0 zt!rOP3g(;-ta<}l0)B(PzjU_aGo#6PyDkz8dKc#nmpqwnijrJuKzz(7t*!vliZ{~a zy(f70FK|d;8AsQ6Ye)XJc?xy4v_-pwW4fIieFI{0aRaaYy8E)tmsJj)fgC$m-Pz~F zYG6&A!1BZCHt#f4a=7>7bM(#MY08nhJY~G(g#va6TJYNl-av$ZI?p`qcn!vj#n%@#(ZwG~)}2t~mD7VtQq4(&-? zKi1`76gs(Y+i4{8ZnXcUqxVmmXMPRfo$7sUl<3j^5Bruwdl*c9!9Vl9=zlh zTebag?AXI&>MDu`G*KB}!MyLa`-Y?b^Knh9z6v(+&_)Q(d0d5M6bMK9AHWZ zv=R$Sp#@^~E5!Lr_1B_N(jXv6Jaqt47hR-uY1(@=_TGg*L4GS>v(WPNGY=`8^g}gL zx`=wv^#YWXJBy|Nvxcw=Ypreg3s&e(3A)N>(uJ8;R)I*kd919Z3*$tpvTtcONEOE> zC1s=FNz3$t%GjL6{UfLSC1tzegZB4PRA@rf{<2HpEf#PEe>%Se#iW4K%>|ntUG5SO z0t#cn44|fx5(c^^s;gbWG3ai-NDQ(kRSn^HeSv)#u#|Z-iz=Wl3lD}@$mJZY+uDtp zV^xqu!oahHYL!Lj4mVLexzyG59Qv;XN;qUtXGk|hIOtx48d3=FUjJC;Ce;8Z0{6SK zHk!}QUt=6aQp*>%T#HZGgi+9!3qE|tNkj_~EfX8^tO{)R-?9OU7R&$dB+#7MhDNPy zYAQ!ScHfMk{o(9PeQKX7BsDG)aZL*!LAx8k=}-@l-GXBkHiB`$5vEbDI3bRsy6^~Y zT<8$ak9@a0{Sh*I^rKQokJ&kih-Prz?X`&x>W4w-C?O+or9Dfc^s|{BdC$LHFY}A6_1Aq^rLI7Bk@4*tT(=5c3{3!$hyk zkCt88s?QGyRF9XE{{W~f5^CYnsrR{hK5^dSb`DHcK-MKejm_|P4ofNt_ z)`}`Bb~}=QOAB|2^=?Cg-T_G+;ld#TvQNUQxi#?E?a!>TDRqs2JWKNk7%JkRd+X5K z(?fKEPNYuOH}W9NE5->VkCJ4 z@@4Hd^eqFIcGFIiBwNDr_nl3_u^=#ppP)HZKd;C3+0Y3{9LV=$w^;w-d%b$9~ z$@9}T;6IU;a0r3`W8G->0ObW#FYf_IsaW*v)~D4af1nB$<^+Nr=h(6_N!Awj*w#A3Y_3$AP`f{yuy}isAtOyFTp7u_l=v`?a_8IWf?~#)QOhq!2mxlrAo&pi`ZfirE zr#P|@uq=-M5!nGyQh#5SP;|e@ug16&W8%h6_w@zUuUCjX+&_s3E{8a^7s*>aNGW>+ z0QuxJ(ZN2DAcB~dzX+n?vw%RZ_A=%OL9h;gnVxQT3xuP)HU`70P5vQ@$6i*ozXjwp zFYX-Dce0Kkxdg&=ca?R`yX_pcL}_4Q)Y^+@Y~?s5REQkiiMgZ5{hJnZo2HM|xCIeN z#i*wT>@MhQp)f&`3BO%Wr0R?%G#zNea)1)o;w5;*4CX15Ut>CRWBXX+>)vc=;O;ah4lbaNh0uxWEG!*pxE+g!te>R^r z2g8Gt#CP|>Cb$ghu;0|Uo>{pu#CdZf<$BImP8?*2=hhH1o^_qq*gqi?uwsVC zWuB2!UyiwS%@0LJcAost;H_q>IBp#x!;e4;!A|PxsK!E~WG{yP&3WX6jToKo> zsEl3yf(h>Plc<%)Z4!xf+Wz3(6KJ{;90o1b)f(1g$A=-9R%5j6G+JXL`8(dC37*vp zZqqxHxa+Rij9$7>frE57%^qiv zR{4nMzRT>DL)fZG3>$sD67)9~3K06_{;AhoG{=wQb~F>?A|CN}+IQI7$dHONPu}>r zo_;Sv@>DrHR`?&Kh{#~Q(Yb}3IQrs>s(G>8aOZV&t6>SDsXrA^w?hKnGo2Tg_zE^hefEao-7B!vA?K4x^ch(>{Ci=dFuBjR0r$C? zVPO$jM-6sOLoGPT#Q#gv{XV>f64rC?SBb=9h^*j;>_=R1` zOhD?#pRd_96WDOx$H132$;Q|km4q7F)! z@TaQGRYEc77onJwR6q(zM6rp4b|u7Wa{f->&n0+kieNE)4s0f$CV$kx>h70}&TXGy z>iG3)W8lH^&R1AxC?=!oh?|%PWPHfqfHkBwf-fCka#1^k$cg1ko7J8Yt0U@i)_diU zgCt|m$5xk*9CI(^R5zHaU3C_xt_V!;U=pFoDB^(5HY-BlTQHZYSK^xj5Us>PgwKU)ne%fP~sCm^XJO{q3?`R9CnBS)jI<#3;yP#tQZV3~WDV zJ{^zFsQ4lu7*XKfr+6aA@zT9tU(bqd1N1C<0pQf<;HVMEorY{r@UT~N)fXa%IZh${ zJ69WQkp^9myYBDX_F@*+JN_R~=kbtmt*K7NpMA5;gFAgT0#~`#w`Qe&>lZop)oMe0a2RKXH<0ie0q<;Cg|HlVuO(nMJ8lbw7RU2F6vtYcf6CU>pt zRVT)v#75ly22FgrF9-p?TJ$SJx_AEeW-RW*3EJpET; zOnUJM5qPZ{BHFZ&H-H*Tn)4A?| z%t6Kh4C4KslBk@Pb#W*M$<#SG%jkw1^l&o(1;~>18tyg_ELT#{=Jm+TcFzUoTx<9j zg|(K)XX3|u(L}>mttc$L;t$M>CO`2{IK@Zi^lRSy-t*Q>~`U(fvBS4yY z^@!iqIXJuh4E!vKG?1D@${O`&l-Q568)3q|$)io2Ij{ELNDn;RnF4}RV~XVwZ=NWwb!5~Nh52?7MEqApj~Yd*^JQ>+~f;zh&NM|Qzow9))+tJ>?CG49`~ z#gH}j4mUfu6@Q)leO3Z$H>9o|z~9q0OmaWgGUa|3dhe}QK`$L#57HhH|K-0Kz4)K~ zchN#E9Q$43y;``LYKK{8+x<963QLV8x5o|))xMQ)QS{}Dd{m}CG)3F5Wt=8FcsQ;% z(cLR`#J*^+MqUw_tcn#2>a#T^_Cn4Z$U9>ZP#=w6CUrC<960azTpIfZ??+-M?|0R20-s;Es%$3>I^8EOQk0;#5vXLy* z0T~%iKtL3|xzU`Fy1N9sGhQjR zCj)@S87#e#0k)Ki-Ry-=Db1DeGn(m^HB3XbK?u8o(YH%!$0@O1?*fJ<4uu=!@ZS8D zlWHUEz9ck}{;@aeNYW2lx(9_^3e5zvRx_GsDgs? z<0W7BqqZRFC0jSzD@iE60BZ43B>669R;h(7sD2zkTK)7gI+gUdViC61zNp};2x6^S z*UyHisSr{3w%WrQ00+6`)=-Ex4*R;d4IKK+rh7!u&>MT*NF# ziGxJixYg&Hg{zK=lqM^-{v6#)1<&kj@T#gBUHvhm$PkEfoZ{Jk{!~_nawfCDzuK6I zz)2y{^Z>NQQ?!kk_AdWl%C3&zr_o`9uzkM3I;nbJA3I)_%at!#wu#Two=b|dOA7k- z2Y)&qv)-`^8X|5?qAVDW7ko^T%Vb5K932if23qqD@WTmvp7$q2F>h}xc-;+S*tt^j z^tZfO;L}y=MJ6qLTMb}o1GgE9En#4WrfAErrq5puj{+Mde--4CNF=(wQ+yVVRlC8x z@_GSwwuz~Q=u199WMAjok9!NmrVaH6Is8Ma3msy z`xqy><}r6Pc9SQi#EcD~zw}-C65Gs%L$F-Ivo$dTzp=) zUIj5$@YJClU%a%=rT=eq8ZuZxK(OFl*;BdYi1@OaqXzu*)k~M+iw((K0r3Qyuh#`g2n=@5MdSMu~v^?Jw+n5GCj) zzzQk>0W`NbdyWlP`U>Kv3qXz)Vgw0%BXVTfYXRp@tz47LMD{=%ENjHy>)3s_#ezI~ z}Eyax^2 z)t!TWNTiPfSsKsa>CvtJ^+9N@WY28Th+3FglL_Hk_h_Y*2^Bk{T$DXlR(u}V-A|^PJ8Q@9SdvZD{U5H*JD%!4{QLIEI`$sN9w(fX zBzudDlFgkiA04h{z_J`+dH@-+lja|J~zJ`S^_M{Tk0} z9ou*1MJ9_;1usUnb`Y_?@8B7>UbR)ZMgyMb zY~`FHaaYG~{QHy1G~5%>kI>eYJAKxMA)d^t1V#BE(mg2uSr>e-GM+S<08a2Lq7}yJ zXtNY9G&yt_O8xi5jg|ItK5Lhxz_r))Yjc2aLX4dB`7c#CfFoezoVqbrcY1yR*LqBP zev`M8BuPp#ru6`bD<)X^^5IpOEgPO#%)X6&c#-5<9Y_)sZ-%2&3cu~x>N>8+Nb+vPO3Uj||v>XOFgXpJVnwbnZ zdesv}aD>rO84ln{bNv;`ROI6r#vW16`XC3WWB57EC$hIfITy7pDnkDR8X3gguGZp+ zE%|Y#6!PhtnbJ7LI5;kvU-&J)0ZpoIEUBjtqZQZIOuj@G8gNQ8@pp~fZ%<~KFOttl z;y|)ke_h61cSg9}8>rC6Fm3L(akb=pj=Lt9N>ZVWb@_10t zTB1$w(?nYiN0+dJ;xCzMsT2sZN3|tJ?u(L7l1E(810aaHJb#`-n1gx9?*!f3lP(60 z<}a2?0eoj1P}7=FZKR@eU%g zRZ~{YU9vL;oU78}zF^A@8CKXK)g5T@B@ukgQ<*QjnZTNqwrp_{C?z!RRr;DxAnP_Q z=H52$zpZZja;%et^PR4aZu!PrBqd(>Dlo>bI@j(;2UbI{8EXRpd|I%JVaaerC|Qec zh#*E!hIQOA*_&RBjj;Fq!3P~`O9KK+nUkeXWfN}J?I>)VXC{L(YS!z~}I0Ky9^P<)M%wsKr0+c3$xtUYo zvxLA(|1+P4N#!EA&~-Vv$Ire`Wkj>j+dRtV;HHs%g`eYv#GZbBmb6XWBZZ`riq)^JLHC z%TO3t>y4PuIZK(jL&$O*{e!^2x4)tYHK`of;}&wL!^KQs*FfD!-A`;+9l&XiXT&Y9 zs;UCJ(pg6{;RJ%MY+O;eOU$R%{@)w3Z2H5glI4(d%3@~NVxIXo9xz{ zwRwyNp35SQZ=_lQA(f%-bLuozZ1}%iv}i2%(cxIV*i=h$M_|jTGRzd#hB5{b*tL>YOuh>3d$`lSg!~TgUd%98L#*Mp+_HT6lC73=Jt+>SCpRp;Gyz+_306{TS#2qjFh4 z8kg=VC#es{WYfIP`vgz00*ZiDXJXdstDjkeO4?i?*QD&y(jjq+vR#B?y#J`bzG8-} zjbWy3eoOV;>|LcG$^;I9s?^YNtvhMzLGjPan0wgCPfw}pW!x?EGLdW2kTiSI;yWpp zJhOFqXQ8og`)&oL?gu>PN4iHm`EBv;CW2w)f;pbWCMS+V8dIPY!0o$aH4+XvmdE4d zwt!L!<%gPGX@JEBB6gH9T!)yDDC}CHHm->i`#QEd!;CSZzMo8|oXMzzPLH_jBLzE7 z41A8}4H&KK&2ZP;UEiCLAF}!}kCV0j!v=%L{>pwIoXt{OltWCikFlWJ_^=8}TzAE` zI=E9G>9oq}7R&ox-p+{^idLz#C8hTDM28hqrfnHGcQk;5O7O zO}XcHr_NN{%vhl9MH9u>tqEN77Ju*9#K`H$UCe+Wc5jl0Bd;0-Xu%2~7i)1$n>=iV zIFZ~r15SL;p^erP6E~RVZt?8^ImMzzG~SWeh~yS6^-5tavkVOgzJUOWF|z9$x!K~H zxlan!ba_#3Kw!Rh@cx{`PbcxUCV`DlvmHQ%%7yi12~Uaf zj%j8+N&Pd@;a-|+_&?#1b$$pf`2@QN!+B6+(p%>Bx}a)Ar^bGB76s^!7FNefcnl)%}a@tk=h7C zXPDiMF$RGJFU@K;68yk#6khVfL7%Hui;Z;B`YpfuJ-@?-uSEgmTGr~U2@`aiE0BYp z9X*f#1CGY=r}Fz(uJ68Oy;ahUHQohE3L`*93r>O*{E|=Gxw(htf3U;sZ^I$PpKc=P zzcrA360M|$XJ+x~gdsV7e^O1APIbc`F0*B(_e+S1Wtvd*C=9T`5fR`E;0`ec-vG*O zR}4%=8R$qqKzKK`{4KorWiZzDp}oF#`^|MKW&&tW*y2=5-=lu{!h~XNwjaTu_glvB zX_L%y9NG;0+{%baEwPsI3>y{$(z%8I`Oo~%C&?~39^P=?)sO;Lj?Aa2a}&YPI|9&# zrlBf!N0P_QG;}11Cr|{Qw&;u89P0QfL}f;yFIWJhwoB`>t;&r{o8u0>Y;YZTH(yIZ zgbLkNRKNnn#V@NAA#DpFoVZl_4)s6@dZbzX9UFw|FA^lBYH~tU)CuZT=Xxd_m0Z?S z^~VGnQoUgP*pnuuW|27Ff|sToPp!sbBJgSAVn(%IG~a%nQn~{i3eCZiqF|w~DE8OD zliA;#+H$+3rh?Rg2UMp*UF5WYKDF z8|q!#9+dlSk(sdxZjT>0%~ zOpOp~E}w2Jl&{P;&40t>^xv6$W26f)d=o4pD(|jfP@V;8U?*%xN|>;0ClX{@`azWm zd`cqq-uySKug8sw9eED^zn|c<#X#)cxxRD}gS9{a9TPNO>{YS@l`+lSAE0zDd+mV}+zqI@sbn0dHuV3%Su$fXT%{7c;02sL!Kgz2AH(a$VD@@2nFV-o^}*(&f2s{V!xmZPI1dTe-{ z<#q+uf|`EXhWNOC}-_luAM}zaoFy!!B@O*nVhWgU_AP>Uux=;R$~sb` z4&NFONgDV2XM3@G3igNZOnA#2&v|ZsxHjw<@He9Rmy}?hf*>2GY8$fid-PFPUOwhc=Ix zPa~gL#hYH%IKlK7OhYF-90?dyxvA!@u#-rF?%+-o&pd3ibUCs&H zuyZT?L65U%yjam(y4#~y=GQ$4m+wwK0IIp!hhS8Z zr!26a#;!ncS*NGF6ARC}3Eo;QOW+KmJIiIJ3ZXI{Y$sBr0=i6Iy_lg(Ui~15!2n<& zc#4^3#KVLP3qX3Jqy~ILLFjBucKEshTWjGTrA1^|m;sMM6QM+x;j#C@3{NEyETDpI zRfRO+;S!0ksU%=s0rtdOHi4mplwp0?Ru4kMUE7!d%A3r&`OcOqLr!lyGOmGf_KjO7 z>19UhS}eUtN`b1IBP^)b7Q>OEJeJ*9(QvPR}oS} z-D?oWh)3tgqXd1QY)SXROSbpF*pT-j$hTz9^kL_H$r1KLpd7<2fG)EHJ|Wt(ZVw^h z7_7B-{#+h&hk$%=6~ujYChcKeDT(`@veGf#75|_B|9)ttEub*H_Kvt0@5gP%|314& zNd$-^Lv-4;n>H%&=>V+nrvnn}wDPM!K;j*R(jgiQ4bHfJ~LT7iU^ zL7t!VYoka6v2FQUlFlOy`k%n@NESA&6m(wCimAZvNkka3Vq*p= z-TWE;U%hDJj6V`@I?M}v1*a!ALvWp#Y`D$E>_RQfKv+q0SW(6oWMs59|6pwL3`&K6^$z<%%~;gY%-30L(a_16u25D|lat1`Ue zy%f=`+U7llh@}8YVEgkPYC1fg6WJG@5*1`!Q<$e|UwS#C8*FPEMYsUePW=}kc2)*U zBZJ=D=B)Dh(*D2!NU$Q`XxbE?sn1_IMqU@AkD6prlc=0XV$N-K1=r^G7-eOPO zRP;l}&{si4qmYktB|;DGad`K)4qWv%tg%#6^h2_F3p8&p=cGBYUL<~j&gxN-lk|W0 zAgO0>9pIT?H8kCmb{c*?rFEZ)62Y)KRRERG6qNbBnM3NQHNegAn)z(Fc>f#!XF_oG z*1(rvcSpbm-S+HT<@Fz043!UXd&&TEK<5xkRPf@I<*^HuCNnx+{2gwk*h%4rtxZtM z9u6NC_K#sxj}c<+(pr<|4@l^Zc)M+|%XA#?&n?`3&B4wf4adxN-XQs&1(Mido zhL-2nO}kU_62FUi##NZZT|Qs*Aa*3Sl1E1!TI$6%l9uI9n4GX0FR z@og)F=9?=RDT@O6g7!>TH^&Qnq>#p|?H?#Jr;T<#vW=%!VJZNrSkls2+f;ENCC*c1 z_R{TZhH){C-@>iwGvP~tH`tE17x@)Uo$zOtNBjg_ut*Y=1tkFuhS_Xe z@b0PAUc_`1-=hdcY0HT8DKONT&`gGWUX5%8Y~>T;fSb>9x6ybujem;7x+$ z!<$CvOOjR9X(-NTB3*rzrMs)$DX~9jo>(G5U6^Cf~n#1MRiI*RxEces1x;?)i$~DZ8F5P!0 zvWahX2|W()u9!r?`$^86?Ju(o3K2mJZ!R@45i*f6E>BV1)U-2PpVhc#&LfEa)bQgKdEZKqj zh6W@?ND8AKsNkdEimrz=)Hj|E+P%XSaHb5^<2Iu6Q7F`?U++k#?Ux<20eI;@Y>)@f zCL*WRPN_Mo&`Ytru^!v#yS0h@H-?*dP8wM9sMfT!B`PkK_TNTT5^iO_N+o5k9`7oo z61f>+7=hsT=9Y02xEA*;|Ibhb~_lon$oE}Uz+85}2->|Uk1aq)6D zeBq%XE|6}m=tLNw$Y(DraA&Qdu-HGTt;v zn+vYt#Ym_4FJ^b~-WaEtU6;=ILOjm!yhDQqF?KH?@y9R0Tgtzdy>y&m+#eyQtanYxCOGZnXSL`G7dV=HIr-{`4D}6yAuQ2)ZA6u`)F&#J4iCg)61Ak+)ivH0X zlrV3*M455s`2a^S=cf-KxNA*HT&P9>sUEnS?+n!%T&&`{BiP}?58{$w_h!W5|6&f z54e+&6+KLQbFf~!ZS(ESy%E1e(d>JFMW9C4S{m`QkMFBV42Lk5!&C(46k_}&LACbw zqG|;fk%bq9?JEi$!Qgt#GgCW;KyVO#F#A-g&?E?DM!OV5?StA>)0X2`ZU3+W2rW+Q zEDbd2uRX;C)+KKkqA^sUfiUrwa?SeZgkKyu za;}_bi%K#Yut_&6+d@bsw1y+b2czAfH~6H|jGgim&hDNK#L8LLeF^iJ7(F)f8ws5v z^%YesFMU}DLgpD}sL-}l{)Z2&iT7XHM-*WW*`~rU6Q34>v!4Y^kl22F1wF9H_ z**_~ypLrN`VNx~~gJ75>!S=?xby`)}C)XyQaVJX9j7!1#j?)4X3q3R}i@HpwkdojL=j>K9pob?vQW= zvAL5p39JK8m8^}rQQ9_!P99ay%3s?tDi)^Dr~O^;BYRFq*O%mDflIoFi-1;Q=Q{=p z6wF)xckD9m3tWENd}sLl?&+RWk%jsi6iy~WU?twEkbt_zVtviFs@1r&_CO231l#%h z&rzJCwFEoFm%zDN412ai03d^7h9=AwL0&_GpKr1^DQIwSB>wW*_abK#Av;Z`5%;6l zHhS_#4O|6WwN}&({zn^hucA*($frrT5^c-=6;)tkn6(9k+bp5j9mq_)2-&cHlpT5Z zqfG!%8T8o_1OZnyb3=!tkR$pvj*?H&VEZQD1O>jb;1qw(4^b65m!`^9aL`EvoaxM~ zs%XQs1~#Pe+z_OD>6g;H97JLHK&|bG*j$gyG9*1)Fw7}omZ>sAhN>{DN~UH}Wrag@ zeO?*X?m-_XsS=$38XOb4fiRT6@sJ@j569gDj4-E58C@zk*!4SYl2PdHI=20P4mo6; z4AQprfSyu(A3TlSi%DYsEy(}>iiT#XmiU8gxTtr1%_w5HLm@t-*9!w5vxlhvB_&`c z61kYi?izrl1)Q#ko%fm`7qAxcUS`_`U(+9GA6IT3aF)8UKD2bEkuB|Dq+x0V~%el%^o@>BCksnH0zdfR4x%Cg zn-UK*xT;Y$!vSD7MA{5~eXY5IjKI=&)w{|uENO9#^CYJYPV@|QK|kd7jr@%ud!|zn zZ6tSi%>t>Pm#V91ZR_V-!Whx5efxH3g`aL>9%ACJ+%J4PR#;?ugk!cpz6k$3D3f>$ zM*jeBtzYPRd|3FBI*VD7pX>A2Cvp^)G(Zu2_5mm9)brtjT}=YhPMb zPh&G323)553iTafQE0sZ$|xtoa7F@()_4>XybufW8LNSOg1lrR06ukonN1sT4H&W7 zL?=ry^~7|>v>x&~(t*SdeU>EP$5T(8l_xfbbRuwA09QSwh%$_sn`E`C5KlycDC?=Y z_Y)Rh4-Od{DKQml8W$(b*N%roBCOk+Sa+#R$6?BEfWHHlho;x(pGyIW-uy6(fjo_= zl;~>LQayvd6_?bV)4?qGGGisk#Yz~(TJc2_60D9&Uwyr7%E0QAZGrdXi#+nKE{qNT zSQK3tmFT<7K!$LjP^8WH=Dy-Nw-^QLDBcewY#&`>H@jYL#5*skknMe5UF$x25;UWJ ztQe5d-Ms7p6Gf69uE}5DyR5Q-7nIyoSqiENP_kWhy<(dh zVY*n3b*XjRc_YPh&A3Z;;4_xQ`|;<#2eoon>*zN7s5}?s;-y4 zQOAguz3Xwq&Si@ zz9Vrx=Z~C~X#;Tv4~eyYq(VgG1dwWS%?|#HJe4jEX2sbUeX;89o(jakE55iHSi2$J z6xQf!Vpq}UyAABrWQqx)eB0&(0jdK^)7-kP1K1XgknUBFN33%%aZQ{t8Bpy~F;4?p z*M<4hVc)C8m$TC1w(c?5^2xWhN@S^8{raIw%u*9^rjVx8Ty8JfCFRgzlAu_C*PQC1 z1)GrHZO#U(LuQD}KY1Ld5@nFU74nT*p={iAw*=J#l$pj_w6AI?WPpI~_!<;P0B&)^ zKyD+#267JZd5qBdJO`a z-!4<%2yEr733JML?E3@i;ZuWGQ?R#lHqGVZ^ZiF+kg^zSB&*E&{6wJE(z_eSG@P*C zLvP;9%gS_R<=TuCsye=sW7)NiaO$~Fy!p!vhq&_YkW{{wnbXpTJq}Yq>M*JDT*_9; zb0e2k7NXswI!)@@F&pU0%kiP#xtCAfTt`yM`71z>b?EtE%&oDjhH zjwuK>G&+4gu;AJu(HeJrrZCZuIe1T5{a2ryk3whTVX>^nfKo_k08Yv~$x6jf(!cMS z9eCi0&HnD`tqN?q;9iL%Y{%{;SW<(FTknQzaaYFFa=exc^mkuPe_GtWiQU6X^U;l> z^Lu0me_k+=h?f_$72Hk9sNEwZl(di@G^xu^v^bE?J?m_sD6lBf{Q6si~b1P`A- zLYx2v9J%^KHmg^24>sH-9(+uWzfXrVvuNyzztac;xz2HmaZyXMHaG;KQ7pfIX;>3e7YploB2haMyiy%l>KDOxFe_Ri>c)xZ`q8C2P?*}+PC}g`rR`O_{?~f%ZHHFNw|Wlg zGo&$626^A4xQ#{SYntiJG+L*opwmTBpW|U~6yP}#rPe+}ZobpY_(Y$vR{8XFvFs76 z4YTD$Q)UhKyPr$`$VKG}Vc-*!s^AdZ!VtsinAW)@#n-LGI}{>9iQAatskw4Jd%jas zN8=jYL@?K^bGxcY&cW{Gqaq71i;-0qNU@8dMN(+?LM@sW*^e#r%w#(0v>;+sg2+43 zKA{CyEE+eq9sj)xx|d-m$n!n-p2TVom%i;7PTU#}gNxBDbU?o(R{gOUGMtST_rK0c zxB+aA={@8VMQ`HED5Gx$>47H+6YN9J5cOr>_nvNPY$uV|)6c!eoI>?9tJ_mF@tp<% zrnYBUZ0%tS-YM;D+}3TIf|*Sa+e!&hP5C0E1oM9AM>xs7^6kh@)|m#Ec2j=e`8LB- zosw=?Ww17h3>>O}i75Bf))s}(927fxk>Fa@UC(V)8PtP{DH_3_Hj(ppe`@ep) zAE}oXvU^a~=G%)F(l{r+>^KO&<<2PzijZkktG{fBY%wV*$N0w#L!>BOSu5M_MwxSk zp_dqt-6kA*G9!n_zNT~^!OZFGiyF(QE+y>adYce*%k~n>8RQdOS6!xLSI*1DYaEUmhzf1oLO4k3pTF*sN}{bl~sEZ)tRTT_kNCIlg>l0C(r3tODKgi z*aLn%+wDH}oJV{7;%IL-VMqI?I?C4HCBh?bKY3uL`}6Lm^hBS4<(m>aMT;gQqO-O+ z*)Y9OLb@S-A}u4XUpZkoY47$^I1Nq4J!M^@xRMjk1t~F1MsTE>;M3FTjJ3E5FDd@1 z^etP(_iI#P0Hpd%c@D9QDImvHt=oe>hYNzJgUT5vB+BUP0qT;uoriokH&r!?$hW4M zoajkUf|-0C>(ZP2kA1KGr*08NtbA*-`O|)DgPgbVkqc2U(qKg^PcYcb;2V{7iZevG zdfSGRfNXJQBfAp)!YtS<_?c>`$W6C=5>c4)3O;EE`7Prhz&gpuk3(DT55(6*&>eC< z6-?sN%4X)>#K&DZRx3J7m`%M4Z-sC|gxr0U=u}_WGH)A@)FNM{W!FN_pbz%r1*|7D zW}*M6!n?B|YrMjk#)9{Wvg8-kevn7hG$#-9J)pnQFM0yI?nkzv@eRIap$1R8t zB4RA78l^OV*@#xW$&dlJ4%N7?idmdjqEl$@%t<+CNX9D{CLmqfg1%TP{CcFS@wSCa zc$ZD#LakKgZ*fAB03yQdy`KQWHrw*G$yC+rkrqs#qAALqXfyKsHx{F|6)u@3uiMmC z_G*T5Dr}8$c9f&qsxXN96=cE~#l+jMmJ;!@ucUd|lY=_#C|8-1^B%e~K#;4lFrN^( zpt>fU=PxmO@8XTxJg%_B&zbylvt3w$Gsf7+IEw4RDO#XL0Uu-Xoj~ZRGQks+E^9&( z)e!wZ_%-nmk1;bA**h!=FE_QZgBK*v66XmL)u%DWQA*2s37&Lnsgof}bS%&AkN6d( zhKm?J<8l$s#jhmrT47!+_3Uw#zF<*^y9jXN8}mm;fjCKM*_xJ1U8TXtfc+!C=L{=P zwwvFzcVBjJ5ou9S2s7p0IlGSR5%U0+PHgu%NK#e zxZ|_*R5Ji?+j21a;@xIK1UIT=4d2%Oegr3xDa|Ys4mSz(LY&D)yk}9zkRkS9yfh1d z&7X~gJdW|`ja4$JG^k2bttGCIVx|{!l*X$Rh7$Q#n`9uUxS`%k`kywg32=n1^Z-cZuu)=F7N22_fg8}A{2~)>r zoz`Y%Lm(4jxD-Jx7GwK&NYaSILp#IfE$G3LxW4GIZ2X?6?$=B8D|OjAotAb@;q`B^ zSxtp8KM(Ew$$^Rr_1kYhI=*~LbIyhHh^SKHsNxEDh;jzzoOZn|`&wSipFZEc-qh^u zd0?{$@%G3Zl}A&8HvxJ`zhwOW@t1AcZ5~)dND!kR*GnjZ+o~32f5$n_`_OzpC~+we zx{(=P>!x$KC&llqIQi9|VPnQU(Jfe&tQHVSj?blNx;@%I$JPo>vIxZTMc>^5}C0B>Zkpdo!4@?BU`U-C}aE9ljN5OXAh5#`1Uk2Zl|Sousi6*qC2Gj^?-*}rgtY$1=iS; zpgx3=S}bQ=vV>mf@e@GJ0vCZ_tusaV6l%rw{d!=r^z*VW5~5$)70W+ZUp%db7P-|s z5j0NG^S-i|QIXiIrAz`AJDur+yBgtvU^oif4KIfRUmas}B zb)w^cWhHxVMM_KSoGiU`7vJORIcHzGShQ0qs^tF53_rxB5=aN7hgk_)RymUzx z@<7e_o-69D=Sr?IimYrytrB%H`L|@8e%7WY5=Z^DQtq-ex4-@Z(iTgDnTfWKo1s2- zysB7NmEzncZ8G-^9_4qEE$1frT>om@c0xH|8`|p0X|wg!wflHEh{mX*mg2Zc&)mj4 zMgikJ;rX!V7spBvW}}Ryo)h)WQ+J)sZ(EPJqC}T~?7GbqaR(l^4=vPp$!E^RKLssq zp=0487&|dM(HpG75nk=_tN%>(qnrz~yghUjv-J&t;CG8aqEzg+3qu`K)@Zg>yLt(7 z|Etbxi!3kE@eZqhtO!qWH{=q^irMn)kM>>G^iZ+%G}AqEq8sjJz0vO#Ip6}N_<5;U zlF9?Zp9aX;W(1~G5DeUEvfgIk&eHnd7nEOqY;st8dq^5C@%EF?mi2j88&@bLrJ{&P zoX*Fx2!=ct(pyM%qX2O!O!|#YC?C-sq$qis`+5JzR>Av9kPIWEbGJ5OLs~0Y1$9d| z(bEBQ;b^mSCKV4k7A7u!_xSbwUXQ`lqL0y;(LJO)j6adx!Khl(M-Ou*+jGn;*_HS+nzj z^J33@o}4)OG;Fyi)Suw4(U$4WL?iW;)DmPLR*rCz{2%`f~R^xDoyy4 z1fSm`Dc123q)ZAK?;1>ByF_T`9YEKSlyT=LIF0e7?;PX)& z;j>FOu8Qq(BYvtNn2md;%t)!o1>Ji(Jt8jOsj^v4uG%TsXQDnzhxiF3UBse`4 zzg?j|5Iby91L>@dK36-=g(YZ^S2&#eS-EvPLERUe4CUYA-pvq9pOHM~@otAhCKP5L z$>-o|XD}%-kfN(shssPaYy1diTXhxM@e;fr2Y(CM<`!;9i#bVGmgJYPaVnB0lf((< z`{c5&cHM$^HrA5@E+?-aidlkR7T?#xI!5gZ8Bq^4)B6z>Dq?HR)qk)Kcojl<_eK3@ z_-P{98oN}@w^eIH`gh=0_Olmq6~{sEBz@Q|4&U ze+IXv1?;!0^>x{m&p?_f!vY=fyQpWRyt&^pxl=LfBW9p8Tm);H2z&(B7N?exT&sQw zdMlY~h%^65JCH^&n*+bD*1}chi+v4~h5#ihXrwABi~i?n#7rbt6wBmr zjUteUK=uf1VUiIUU(l0#q28wHAgT6n04E(|^-x^NRH7~7Z1kLHPs(Hv%Z?rf1{<

w6YgBolNSs`?i1tgmi<{^DVnZiffk2`%VS_O_ zvBQ@RXWyn54;T7S;?eUcUYfjRI}+BPAu8-K#^_s^P}6s1nR)_@@XS!MttecfJ^$+5 z$BREf$rZZ7%`(Yj*B#1$uh^L^2${(7xd4X5M+ouvwubI}!dK)6t+nGNCQo62-m`=<(OF9zhd zjKlp9_X>>=9}Ezu)$(}U2;$N!g6lfezh3v-uZ(qqu2;(#wixRCDa>g}n+jfb^w3s9 zcW71g=TGYcf4+L5Y<7S9sqqwry6;Q9rvbc3i8sG@tZ4SS=?2&swFzD{?WUkJ%XG~d z8??@DoJvrv--m_?wQ^Oyy8o6lEaahlTp)Vz&=u0} z7LGb5AHo~VPNbfBvpn)%)kpNWveGCPC7 z>3Km0Zltbkpcw?bEli~)Hu@RXy1Un%)Lt2&`V2?4B$82}kb2uErzS>9$V+D(M?xz7 zew+o`D`cH+uRDQ~Ocah3S}FhYYt`_b(44VaPn@k6&b`TNo1FohBH_`i9h#|lGHP}1 zFzR7J4HamuD)l>FIn%p#H-on1ddI=XCsYp z9xFb-anrAnYvFOwv^<61pn$E`S;8>7?$#Z6zuiv)r=)a51BzYDZ`EWq=N)dCk>b1g zo(H#>W=3+jzMfi5vlK!(v?HtEJ`2%Xj`sJPj@r!7?f*gwn`GMWmxT#9H^-wv+5i$u zT>+_t44{V`-k+{T#n`{!VLeOQ#(&$J?_IQXow{>>o!CP%O;qN~rp zzAzGz{gTb%`k4Bjc3`C||8+pl;zRYUUqkR?!Q%?X#<-S$KT!35f4uHI3ga4-o6c~C zER|;d7>I@S)J;l{?27k<=lioM!^#r5Lb5-xPBHY?{S&x~^S*uJo2f(sRe6GevE&^N zPAv7GfrKK8o{~0km#Cn#lUzC|s;>T)ifzmdZDG3+E*|z~Nrwv~J%q6M(|CGec1h#$--{G+&1dY_ST@R_L9fbyIwpG?_d8-jN$~h5?Z$b@quwU&d{W0` z%JbqSjcN<4wp`jlk=2Of+-^-oFJy_y5n@m_eZ%jjO+qA!+YUEh_BIjI*RVy>R(PNL z?455uv`+or0YymTsGs)633j@lDW+;y7w%49)J)$ADGl#^z53BHfj^4;mv#5S_tCJ! zdsh8@M+5vHVI$e#E`(Inexo+06=>XPfbqvv7`O2g>XtchE5CFGpTmk-n_a9j5N5x$aRO4Ik3 z#R9VIY=R6^1E$){KQH*-3-PL(($GHaLSjqZAP3)w z{~a0;_C7^9sEql(Rp%%j*ZWgc7{*AD4x9T^);v8vf1Cd!cbBV77VIIbq=RDq$I!mu z(p^2~H{#YxY1NpgM_#-wO;`Xsmxk{!)#$Kh<) z+MV&_qs?k~G^w&GwMe|$y{~C^7XlR;qTJEQedQ)Gq(SskB@U^NU@0-8I1|RS&cr1Rl@fF;2!Sd66&3DO{&oDL> zNp8lJO!Jr~hVLCtKKm=*_RO`2=~p9&^JK+F*VN?jqnh`(duLjCMkklWREoBw zzRg@IO5NRLat!jguG~Qqs<-Lw{812pjiDu|_$IW)^>zq#2~%O%ux*07unIYD;W6b9 z|Bk=znXgl*E8DNyZNi%R=vuzNQksbY4YxPByM%vGDRcfHKJB8#9p!LFMgIySvAA$O z+5P2i>U3B~v8uHD3M0}yWTa;EI#B$!Fy{;V!h-Iyy-o6cWOY}QwUIzw=MTTdYs~$= zsZ^6hx8~g7VyHcmKkSXZc2RSF4N)Yf;`emmH*Xgu*Cteptl;tK!AX7HsP2@#v(5DV z!XoFsgc#$<{mI^Gch8pPw|!sI2B9)>dyDV8zU5QX)XnIaCbp$re=2SsyH4A`_BwAg z%Bgf#ErCHX>ukFQ`oHN zQx*EIWb;KpxZ`5ujGNI;#sf+!N&}s7B#GQA^sbK)#whXbYQnBKMLSS9J8|v&6>KkN z(?rPK&_|(^>8Djq{xU z3yyy|U@`YWy)W=eCbII4SD>7e*zUkBxQOL)(@2fIG`lbsAaNKkDRnn3&U@+o+D9hv zrp#^R2o}0_edt$mEwp<5=btmYD{jV2nf*x4`KF--O;>6yMepb=7Y-1WU4=N_QZ z7=9V{aZH?Ro}M0I2gRI6|BAR7!{N&b9p%EDY?Zc)zd$_@4@>}& zEPUS%X@Qi)G~-5yVZ`yWB%UREOLk>cDKtb>xj`M^TmP!9y9x{J_L?E}5H4mK4J#i@ zQl2;3Pjvli%-06<-Vrk|Sujw^U%1gVXQ{^S1m!p@cmYrQ#aNGlK|jR#jo|N?DtSR(X_J8G>I1hOloFm>#UK(a>6uVfkYbN1 zeH^eq-57M^r2f%&*XECBQ!am~e2#X%wGve*A5mfL5c&f5;UUj}w@pqZzxD_N{vc1RRIYbzpQ6H={DRI)A0B z?67AoMiz9xh$gVMePyR@W2cEX%tMqz%t67RwP4yZtlteQ{av$P!Bs82v;%q>qZ3Mx z>GNOl+c{1@vVIWQdSH>7#iw%KE^9Y*hkr6D3wzHOEg={de}=zulYQDa_MoC_U}jfd z_)8fhM2P~P`+_O|I}sB87_~}J&?6G_z|-(7igDmVYkqFJj-V74aCX6{IQTD}uP-!4 z?~1L~ymc!9rkJ$9-63+WjRI1;{3UT*s^_z6*yw@O>#>k9r#!WY;Ny6&#sHea*Oi%~V64mw!CT`g~((s|Vr08V%mWBep!ldwu&c%na3O!q07{*!e!I9x<) zh0$3VOafbE$IQp`AR$k)@I~_V+k77|PHct<)UtdoEW_H!-8z@~AXm z^oU>3-@1mboVa%lFd&y8hB6~6woRa(yS!_?R6{zF@75*{c6aT4@mr7G?{q`(%UV0U zl5ORTD1EwKU61MmcoflApX^H(UORAoK&hV8(e}=pC$Z1BN-TsP%l$cPG?7)Xb-g&* zyAP=7DcAmCOx|fn)aq>1hLvX7l?;U+GzLC^!gGWJ=Y_-;;A6!~IDmaDTYI0t0}uhx z15%)M)RBUT@pt>eKtV5#(Ie*u|BwI+k8gZ5bsq>MGH=}%sI$5;lK+4In&7Nv`}PoA zcW%DnOMwUo9?_h3eq91f+C7YcZZ}%qr~OXAc8bfiV8@BNT<>ul1d{WC|y$D zbD014u6Mm_&8#)E#yK3$-uu~4{31`NAd`a}r>#oOf>FV3n}S-W@#wP}uTc;D98}elC29nK;~Iy07<6hCN)TZgpOeBj^i%AEK@Q+3g`b zhX{@NAbzjtP2T?IBIV#m!*t|BgRKbP6oCFhC$1K2V0kbq1nM3*JS!A8aZetgCe?N_ zP;xj1D$t;vN`M2JO9HKIlq^p2%!?IJBl)!7 zPFGPi(cS^AnrVS1#L7$ms=`Bb{5MdcW3FQ@{msNL14=jM7fS%Wn!Qey**+{K0vbq; z09kD{#`4T~zyix@=ym!dma)P|HzI7&?c^l~uocLfu%yI@NV?tZ|Mmu(#UYT`xD3&w zAs-5=xKh9b9OFj)MNowRjSufgcil!KrQHCYz}56xHsFfI!_)1X00<>PG=p|QegKJR zFE^Ru6CQe0BVUJ!`XF_FRFOviA}9`CV)EUcGUG-zN#aR}Sp50<>GCc086DiZpyT8X zIJ47WEkufUj-*#WUj0eGmzIgsP?Zowf!YmdGPc>nb276YPywFv zW~A6J+J~h93fm4_qSAxo12UObeSE+A6lcr~{N)!P+q%Z^sXN^0ZCJ7m+ z%!7L)rVI2Y)I}v$v!U%bv;`9pk2q??2FJAwMWtw>xiwPjj|9NzfUX{ihFahWJd5u@=! zkYR%7X5PMu?BJLn*w=K=jt=*gF&|v!yQctQ6V@>sYIq4%jPRXS5s2A`z+4X4n8c(O zSaGz4b>n&Td*Yq65KR(6h0Y`yKVLqkT*%GU;p1hf;Xk91)4;Muin5Re$Xbn*+b45* zQklVtKiM7t`}3;0jSHQwp&+<B|) z2G{S_(h$FAE9r2Tznt@JJUfhgISFt(H85zBs<}+N;+2vPHgOowPnrV6K4V zP~l9Z1YbBUMh>aca2Bvzr?0vKDLgnUQk+ZwY9I$Ca?HlHG;)vEU)6E&CFY91O9=YX z_~grBL3d{RjgjeyPXMlksFF#vYL^ws3g;HBaJ!|2r=FX5g{NheBJ*0dN&Mfk_QEGi zgO2s5&nrYFz?uufVO3l!GEwcE7U>`iy*1mEHB7F>+=TYp7{|B=u0&Op3sOWwG~8W3XbhQEpGOKU_c88Epx3+}2U#OH z2walbypiewG1e(JGy$mPA4TJ8xJKnP&e4@^-gR3Umm(7xsL{NH_yG!d@2+w%9j&!7 zFWR8FLP^=Pi?-qWVF4`vVZ`(2&Rk17<=Q{MovI8Hq2J3&W5V`hwUrW$Lg(Dy$be_U z-*i|Ix>TqKHm`iEw@{LY)6c>G>zxyz{*|2OAcm2Hx#6cDOWgS5m;8>$CgdS6by*7| zk{UX23nHrMH?r=}KFUEJpKYK~{+SqttdfsOJN41JIf6}3#x7SZQ zJy0KLPLB`F%9#)zQ%Y_U z(_0SmmohsPC6}auuMUf#K(!0QS-GJCeI&9s0*N(2_wkC6DKJu}0AfbQUw7af&U#x< zHtTwiHH6->CFHB^DLx1Hwg@Vw&J)RQ%!@UDmg-`3cnM7m8@@dOOx14pV%N_ zE&gh^dr@=PO$NWgwb5;9pRw`k@Z$D@vf z3?CO4+8>Bl=R(rQ0{FZ?hTS8rq*1ZTgt# zYP0qGJ4SN_5#auY1Z}$WaCn7Iv0123HO(1lV8JKlatdh9VJrq0`m$WyK!LUSR4Kx5 z(2H72HP71^D>q|RjwAu3N=T9H?h1UA@y`COgKam6ajmXcGDbSf=s6d-!Hq#i#A)H| zT5Tc1;H6`-pzX25Iu37J05LhNM8sL2ltEO0&y>FUTcp(XaBrR8v4tBzZd%VZL$1T< z)i;v%hl5=D7V(N~dCjCh@$~FvdyGnyQa3Q|n0KZk&>0Lvpyx^x;iHrGTZ0E1mQEs%?MU&shcIylx2Cbv#aOzYWAlfkD;*=ZWdt7K>G_lnIs8Ue>l8ALM3t6wE_WB``Z2jm_1 zbwT36%EpEW6#GMt+q8Z~Y-v?VmVpPzWlLyk`IbVi&u>Yxcv*>XX>sug39l5cuha&| zu7%r19j|@eE>BZih63)ZEdt?GrI`_lh%d7pYE5$=jQN#x-FEt#uW4S>aN1V9NC$I= zoy3u?$5eu-X1@ss(FCVUH{tvRijZd$%{fVemUrEOVbBB=RGPEzLE+~r2vA(td;0SU z=;aZr4~T=LMx)q#3gub0Kv@MgdRjCV;EuY0(Klsf^3JP?6thXtXi0;@=Sgwm!EX_e)B zm)Wd}403VDu1BlBm(w^>)!omj?9#~LS~H+j;YY&(RMdM^OthD4Aab z+*cIWDLuWdle7HJ@YyGmm2 z3C!>&xDP0#oOl2Jfjt;yAVpvbVk6;lNg|Zho!{D;F_bmctLFYREM(=*bL||zv(M0& zkT2~u?~?n+g_yr{Y?c0kmp=PFP@3fVmoZ>u&`HHvnMs0F!W3{gLK-P*7Tp^!Usg>P z=8dOhxzO<#^y1ygI38iWSvaOBu$I(mTDV))4yy*!KR^)M$Eg$3*39JTxlG|-<+^ym z6pisXGp^eiA7s_sE7eR~F8fSQKrW#%i4KetiFh z{NSb2=m!MqQk~yI=Tkin)(-+r&T@`FucmZYUQ1Cz;icsh5%CA2wvdi8-6)i1xgk(GjdQZMOsDH}EBCljic)1tLFj^>& zFCiI$TCtPpFaKlzg{4(EkM)=&pxM+K+=bRCRie%}gqY4(uDVWF?%qkfC(QOQaX5Im zrI(_azn!9wybo7@aD0&N_e(*rdO|f&{^g8!pG1$?K7{#3pkX#asJ~F|Fu_DGMBUM0 zDi_;Qu`U;5_^NE&bm@+3@gJVr0?Or^mQKwd-2PmgMeqh3Z!IZvTrMmfJ)I zaxIfl^!L|VS9(l4BINdp{%|s5OH8E;tBjdcIjtR^)cF}2D5-dy-dDSwah&7GAUtr6 z=Z-|VP|aqA6}?-XV71G1bI4a6w8GV*c}9a)*X&77m3hXC8+j%6Jk5k@C9Z?_ELkG2 zFx4*0qwhhz;rp&Kp(V^e;j%jg0lK|sFkd&zh@^P|%@=J&qT0-PTaqlxC==Nv6uIxd(-e(O@Z z9~eOkFP+*_(al>{Iw9@ghW1W3IYx3&#EV36NCf2 z7!{4MH*enb!VSI~{&t7~9d>GY4|-g`DX=Wo7wFNW=h^NV@BZWay!;}pZa6*zYm`Zw z`n>bSm(HqA$qv;PcYp0_NhI;O16jSNSb|s}XLAs0?(D~e2zrs$aYdtFpFsVeO_#9G z&0F9}AV+l(s)#nTs7HD~5J+~?-R~DIM^#!r3&ea}~02wMS5)6HxJ+NKcBL z3OG+5pK!JPws@ZDug*mEYLj1`NPI=K=`pYQYUYJR>rF_huAyx%CHqh|#G`+}*MvB; z)BKclCDmK7zx>>|tS@#FN>eGaSB4;t=MHrDrrr$sCg>1eyomhl1o+vAozL=}PoSms z-JK%yj5p|WI=R!-;N-C+G4SKbsE**K!-ANPDA^Wp6ltbO9&}~`NAYPZXE*dWV(F*b zOKNFlRd7131?mNV5ZxVZ?!a839|5i}R@3%64392MaL-*R$;)beTLI2Mw9F3R>%Aty zlt^HW?8aDlhp`HWDE`+VPO4zq8^j@~u{oHh9ZPS)fl+x$pEiYPwR%UYrHEqq_Ccm% zI=b~`irSt`0dgTaA**160?Z=!n z;~NQNvh-{{s=?Jev=*X3$mOtlFa&k4x-L|O|3I!zZe_ft3EFV|59M&eaxGAaJ;`vG zs9h6jL~@E)yQ>Nv36*9}MRp5kQ&H|FeGh8v$I2_i?PD6nec{z<|(DW8)ScB(1B{{cp8~P}um@@K7#7VcHMC7)7+)1fE0$7DgJztv+5ypW>;*km*C&ca zPcN1f3BF_kt=}p~5dIL}ShyeU4jGsX7oUX>$bMvi``Q$EJR$qC9mzkKgWaD)yx`F` zxll7f!rYv^4OL>7ht&vb4Jou!oZC5_IL@U~cBb9>xCbRU)zB*zGxI-&I30f_uAzU= zPUD#@tHCEh|KgJ$vCmAlX%c*PiE51`>rM&ipo*&O0~MkFP7t(}5@A~)x90*hI1ov% zZ?GuWo-NepNeDR4-h}U9M$Myzl3-lT^nt+C|2+n_z6fG_u_nPn6Za3Y1o77XYxj%Z zM-!@dT<&I>?boh%WQ89+^>YJyK|v?=|3h`V;TlA#ur=|{+D(f+XoWS#JupO#Io^gPLo(w9RBaW3Pb+K$W#LoALfS^>jeyl2wO zrJ3z!R@zFB%sO;$v)g*5r=iNoF!W$!{k8X_Vcl-BZAH$0VgF=^YFgzyQfVy|mH8h2 ztL*;U!5%;MVVGE1}ujhl; z@?Ei7@F!^9)zT)_hg=&psxb!9QK^dFJPHDmXX>CGX<<#=$05)ey+03KxK2`xzxR;m zXg!~%Q2Wzks(n8Wi)8Q8xjH$G3x8!SsJwy9I^#niZ2Q=&zQ$h*x>XKsyD`-&A`asW z!7LNNqh0(3N1aO*39#G=r|}=@8Qg}f+G_3R?o*EQXI#(*}&t zO2LqY5bC?-XGhpb+JA=R?5Y0g z=91Q)D$euN6mB1>uOipzY|n0B8LpY#+-;LNqIlJnfMG?t4^Grif11L{XM}y8uU*3C zx-`hE2OPN~kq1;<==KgMgy{6gpC@P^@Rf@g3JL3-7y6R;oRC}_n|0x-*`LueC&q|H z$g!@AkYKh5Mj+-jLJu@q*K$=7aFiRGI3?_SE~%hdE^n((5%lO=QAFc7wDLS7CC932 zYgRLTM|CQw++pyWAehr5?Dtf=?d6fK@C?my_dzeyYbXbLVLK}ez~hj>JxwU#wv5Bx zn_RI)+Yo-1USnZPX+-3~b<;d$H7UnrOsMx}#uXevC~hM2t>)Xisnds?5J}b{lIT=C z))xhVN|RIWVXU%q(KI*SnYQmSAAhAezDhjyil*n_ZGOeUtl${i9d@cY)Ux~69P2k@ zWmNs0B8-JHY-*Yw`+tZYk`=e-Ue^-a&7gC$FAe11@oti|6EkmebK`rRu&!3|_36=u zw#u-u_T0h_c#D_!SPI>QWSYZ7_e{=o;#|*%g-XXe7{c@NLT^+PFZ_ktPE;W)$F_;< zw@^2{E#1ctnzb8=v>m^ZeeN z*yFZmeKAuJ36A4P-!{emMdGTWF?mS)oTY7#57eZCY-s6oxAM>~X}Yx!;N)^#Ie zlI5vybJ|1lS?}yWCNC*{ZeN%xBkZfouM;Z6mL8So z9UoBY{ViFPx6akW=0**14;Pu5HW9DKMJq`s#;Dv38bR`H8g+ zU5aQesV-Jft{&M`D8F*A7jw@`qs$= z;(ZpvKL0j3F8qLb*A#Gj2q9DQ**qgJIxS(+0NiTo)(Co<8r}zMoK$aDr5l>;dSA=- zt!iy3>Gvel45I2$O{njk@E2`&cyBpy#g&{$F{BOTMWlU*J_k7rigd#_EL$DIpCr(b z|M{H1v2=JXy})DKIKK9ff5L5?=g@UusBT;0(C?i^W-eh?d5^11(01G}{s|`w`Llh2 z8ENkGUmwG&jm9sEdsVDi+ZSve+!)=?yV`HseN7@Q?^;*!<}N)2_p8m}Kf7U4ZrrA7 zm?4#xmOS&0_!ccnWhM`^O*-^lVkusYlw8%Hpes8CWpbElm(%y0hU=EyDWcM8 z{0$UJ|x;R*)rNu9}|+)r&we(waad^AG0%^Ngvlzi?;6Hjx(%{ zwOq;(&Z0?@bm~&y%cw}`0Vk;DmCZaa*1?PN9|daDL21GmO2M_*~ztS zo^`+<+HNIr97hxoQZS|r!k_?`;v`dQrYE?mHQFhzCxQ${lMAf-+^fOZqbqRAdsT8q ztU(IT`fA8=8W<52O>rr(&T!{@&+jFTlUvs`I;^Y+8uo3N#DC15`^>+=%YU}RPkMp2 zv;8Uc>{xlxpABstChLETD;HH*5B+THOy_RTyqd`{PKiCrug!ltpaNUT6cA_ZTI(JI-0pB z8O^?~#I(lt5BjuO>VCh5tJQsJ%|jAj>1d{kFG&{F)EOLzmHjz%uIrwlXZo!9&8w(p zt<9f??uiC=`^*!kQHj0CTeEFxy6y(|+2_UmpweiBWYC!@(YsA`mR;B?zVFE9D+K!h z3#9~x>40ly$!edEmus(=>|f${mz4h0W0mEc?xB)SV^PY8NmFybxn4b`ZJL&`*ZEha z9XGE>eN@%iNbGPl-eLLUsrRp}-lJvjH;dG!+lK4W(zdsYg!D_~JbILqd~GrX+zgg| z{QGKK=qqR;+M$16u8Kx8-PmNh7SHg=)RWA*FxgQa)dIe0Y@SYiCC{~UpHRKQ)g1vA1soi9!>gb8`x7V zWf5&)M4~sJGK0*85oKa^p~-62(HxeXOLDA3r29J1lr8a6BUeAId3y3&L5Mf$X-P46 zVgJLPqoy$IIaH5mwL+kkS)Oy;N(FC+r6y*B)OTW4L~#9QELKhA=OdNEw9>BFpDW+u z_blb9ePs^nhVPNbPYi^9eQJBFM^6DGuWO)+K9S!+U5a>~WQ(gdz^j16}}% zQ?E}st#j?SNvX`Dx2hCJ4UM_x7)@6GE&8}}-ai%$bUN2I3Ua#cd~%dBJ!qx%jErFp z-xeEutCnu{@=|)~6z7Ng5yCQm82kT18Tp|T+ng*?loR2i>?ysG$QYH#XsM!)nOUaCh>g( z0~cEeTYFf=FAGdQH&0V0zyo;RS;-sjqHCz>Opz5jB1&UiGAbrGIi5bvJcX7lDJ)fP z>U2v0sd@wb{u&5v9TTMw39Y(6x9m1c1)_!N$X(@Z4-=GFAk&6gQmJ2lB5UdUoeHfrH{VzrHMk5=ZiYH*$qF7tv*HI!v7H(f0?yQiInRno}-Doc6Pc;{yel%X(O^zkN zqH#{%KUit&#CF_coz!xuoY6$GXH#s0=4q*zo9LKpGdcu=UcJazx+BCDSNZ1Keopj5 z|J!dTzG%!G;9C8=J%6X;blqF}(lF0EW)v;f440+lJLP6GQmkhBaKe}t9x6W6>*^f) zn1Q_R$6HT+4;{QAHUD$PVej{ng{x!ixh-GH$>ln0OW{RRR~y|nH%;A;?YK*4EOT}yOd}Xvx3TGzs=m~+B{ug z_!*l|lD}O2;GBj1acS513$&HfA&Mmfg+NeHeOl8uQ0F;L5k&;!HJ&bFa39`y-KcWT zfes%n=|CIb)_J+|-b`)CXCZ!k`VYFDO%vtZmUCGNrTzBaq7LIHp72Wl&YJRf zVi2z_(Ln-TTeZW+K2Azui&E~1m0IuFkG(DF9VMV%my_=NLeud@&|{t!6oi-{ zeVM*V(m13gsn{5X@;u)oL+@tS#N0*8Y2Fk`q4^EnM)JgLn6;_!9sOutg@si_C}dE^xY_@DDEPggV+ougj;`MEDvlkPR2VO7qg zgBic@>d*P}#oLxe=a#jIjA;ULR`?S(D-T*sEzE!Pujyu8Jn*xZ$)2j`o$7HkxQ9%bCgRQ5TS@GL31>6>1w(_%|nRNci3?-*mw=)ONjr6NUwfK4+)bfxX8?y4o z-nahh)Wj^$nZ0M(T|IBSDsIWV43n(SKu>q|u>6eQ!pU z3p#O8-PGYy^}|%`DB5{oRKB*?86R)4=v0lrU2!nr=4W!YW&(dKPbLZey_+2qb=4bP z1Pc{v8>aik-fx$PFWpm;?(v=%RgG5ZYL8oCifrk>a~wM=^UUsD%UOD$!dY4?l^f@c``3i8^BGH^&_2BPa+A)K zjUP+=P*(L~{*Nyh^?e!I zXxI{lrl{7QForul7x1WZIJ;^NMb?PdGrzvqH$PMzoRzY(+mAf*y^i_7z{Fgpq!V@D z=Y3W7T*3NB#ss5hgsTY3j$pL-*f};{8MAl0f_K|*>!zo7j?V3jtv|B)v+@Jdi)AKM zrWTPoZLI0f=Fp^X9$UBv!i)meW^33hAwxBchCi~Tk)k*GDMv{%>nLjSN4IIHYJZ^B9L63nt%1;2}k%C-6*EPJ_fXZa#`k>a>1U*{I%@JeTJ0 zNg@sgRo1~R&3Q^!@|E97Kh*DcezZ4qj`VuxJwO^QO`bJs;CXjNP~r!XamyXVIS2Lf<8i8oM{v%Kw*H{h^L?) z6Ocu{E<*tQSwmnb7QK{GOsgAlMD}a9(^L;eGK|E3j|=oaV-aoX4{j$Q=2e8X}g;XE>MaT zw$M~r+kv@!f1S`3=rb-<_PQ+sZYV+2$)w7L9a#*803#|HX7aH5;w zI?Y{G(EOQ}@E$9-pURHCOZ0Ty!_q=vPbqK)QTA`*e4Fy3;DVeGN2z6``0rfDy}4I! z|M%+(p$N1!zt(_gdLy)W)9qlF$-zuSvUK%c0w`0_3K-o8HzfpnnWv)Qh1&@KK;x(D zxT_TCEPzo-b(V7_sT{hByxDm?bccxz;(*+Z-+1lJSKxHO@av`}gTZGv`(eN1!zQSU zri%e?7KDJsz&I*1TQp${>tTVp6al+i{M?Tvu-ZW&?12k6wGtoikN417tL zb4AcRt_HMPeg*bcEZdy#%yS>moV-59mXoIVng{bCElQa{omTQ%l-REXN9e7;`QI-% zK%p@$z_jVR%^EO)_*664SdAqC$P|@%RQOaH8OLL2CC9Z_5jfY*(R54Lp`r8Gn5FLP zyYM~yUS7?=-ylB&B}Uib<0wf2^!xKW?l5qf;VB3@*=J9}ndNXh-=+7LDG}z1A`DTW zXRL52ug4RXbrzXcI+a>|5TdM{b5TY{!ObTeGL3GK(ge zR$xX?E92|^|F3mwz-vXt9q1q1CS14G)Wa5DMfOyn6@&Y$WcL=YBe2^b$Wn@E&rb&g zMAX;xw;sGv=tlykkK+HB>Gi82Ycv&SpuWD9^Q`wN3a1- z??^X+wZ=lG|8yC$OLFmwxjsrxCXV11Kf&yUpf4etMbm~_#s!pRGhk>n1tX&{*|Ef* zl`s!79fu6T`2$}VFSl-T6YdEWg&zVMI&;RzWDv0xp@073I~feE z;KW;MU5!GYg%a{%3PbQ1*0c;9ubuoSCVRretYz3 zc)Z*ISYYy9ub1dhxV#KIGKwXd@UFZ4A3&WN1K&e>QPtay@A6ORUR}}i`ct+h(&UHq z7D5SCj)qk`iznhn-=Z>wC#C;;HtwMSqM}*o_Gh#*JOlqi8FIEGZ;cKMHb6^hoDh0( z$_HldOoErwU;>6f))O zLi|$5qz6K)VpED%RA6;Q^+8Wv4N?oF#th`;ZE;9Y!m;Qhe;IX z3Iek61OvKv*uvw{d@3q}bVrO*rgxijpcV$*fvD?b-)sf(D5!36 zg${NQ)jD&*A`_?+{Jww)HKkDP!KkK#@FC)3zFB_8za_xKG^Y?TS#c4}GScT_McUPwt#5U{u#87VM9`E#-UF*>bHc`4 z27Cc>Iy<^<;1lQwXCc^0cy#Hx4QEf#ByF5H-rmt#F&Y8%ZDaj&ye2>;r60`HC5X_- z!O3lE1oxCiKrJQNeHXuvu?2?XC-X%|WwZi-qTpxzCRw7$xnMngkx3gtX+HL;@~J|o zP^e5=9s}yty%EcF#i<2|?ol1Nu>#+wU{=!vwnfgVR&38k*vF@9JhwupP_G%DP|{D| z)H%=Kcq8RhTR*SzKln6?NFp#GmMZa-3=9Z5KYim?LcR%_E9l7^3cRq4;MUPH+a~C3 z(ZaZ6q5N?-Fgh5{@)|mDp3e?S2%3K-SqzBg|3YS9u*hkRMDzlr01leyPMV67-I1x0 z>)@XEExd}rU<&^!0HTIwoXZtwlK_SB*lQ!Y_!u#$Mkd_m`5z0T;3`ET7hZ!8|H-e8 znwCSkKa3MbKBvZ+W=;hT|LQRZ8SkFzQrP`e$jDr8eiZ3w2UD^*=Kwy6b#r+{up2k5 znTt0fttL8!568mwa&^K(E3KlFUBK?Wa+TcHKXe^J)%q#`*fkY-*pR|I$AgnQY7G1)7AFC> z7eE@jdR(p(Ct-wc62aL~-9TO*ciX=LOk*(xV8wP3z?l?&hT!p+@$Q{9y90rL(+-^O z&QuAJBc%Zr(F0gZ<{9(nhn4`*0MJ-pQbPK>gztbb)ytx8P$lRaPDpsjaW4Jj*Rhm7 z-2BF2hTsI&p4xKxo_zEcBz6Qc`|hH7vRI z_aGORQ!Yt=R!B6GkXEWvT|ZeQJbF!BWf{N-1VGlDcIS6qNjDJ;cnAS->W&~*lRT7B zBDMgK31`wfCpQcb7Mi*T!RT{CfDF1&Za)@3_U7V!3~>+7lTE@tr<0uIIzE-4p;$?< zAe9_fsBf~X?#TzWDS~@I001Ncj+6C{0J2mtwgHUO6x{WpOP79&IM04Tpv1e$;>3cN zFYLGgj>PE@0Yg+hT}caj7i6T#10VJCQ-=yog1#?~$E+4VVMKmbV(%pVqndhdCzS!t z@s3CeQ}X1$Skaf}?tshJt!SOXxhtb>4q+Nj3JUF^&nGV#;Mi$cXtv@J)&bdr(#ci0 z90?H10lmi;AJjo;Q)v6|uwCBNSijM#?+0@%B(re(^l1gzEpW^5#+);1r6-R105ExO zB3&e)rANsex&Mg&*8%Dol);RTM~bf!89jKT8gguZ=TPY++S@AwOfn2gz$v*>v z$W#&a%4?(l}v=iu$lb)2fDb#ChwR2;4UNYIoBUPLquT~uLo_;2e3UIO=O1fx4- z;g28wY;DdPJa|N)?sxK3y2eNNiRY3H+Fiwve~0(C#fXbnA&7?u{&XAT z6;=v{Zh554(e6Uacfd=CZW!zAa5iJw7%0JvQe6P(q?wq>#BcBxjxbpH4mN1Yssxi{ChD6otd=r8t# z+=Y<)t0fVYoRriEz}cB>k&XEtwd43#8l7CqSa0owht}Xg6YuL zd_h_6e1Ui!kBi7NIoR{_qjvwS4I;p$MgVj*MlwoFUo#1!4uiuf&Uf+v#xKqR69C6; z0tNVFUhXn~j+3|(RjS0WVKO)s%v>A$=rbzQNw#zS?kNN^7l<-%v|!t$x^A&|2|UmL zRv-~sK{+$ugql=$UqAHeE2uyQCeI4=796?j&7w1L_BdMXClxVBU4xK?U>@GDfKh zaX0hOnXx#D&wUiK^}nI}J`70`f<8Kj?0Pv3!oYWM8D1mOMuPI=Ps}>IfQ$O?90hV* zXwCzcl=L!8KtFlL2Dt*d{lBGXysU}Pu_EL0ArM?gh9aMys5079ebi`vfqZx9yF}3B zog-&ra2uKPOU@hk47pq1aIca$0s})T?t)Q5B+u$Qm#dNLJ{>Xz4$0Q#DcYnD*0@gM zA3(&tdr&wG+YAaO#o`Tqa?zl}m(-UJ)y zBEA7wZ9A<~^Bd?D)?7g@LY=QvkMt;J{tNRWEYvkvjmH1F8FHnh&3S-u;J5o(^-!K9 zXyG!7-Qnwtt7ljCGMjlcR=r92<4Y*Z*CIrH1q(*_XC$)IrBl5N#~mi9aZUlJOCsVj z*S~4m6RzW8-iUAqY|?()inE*58I|UrE57|}j1@EKSG3>tw`+i?LD=nQS*g^LqA;Wi zBnUSXk!F9a%_*m;G6%FHWN|YJgj^STdq26%uL5H0?S~Gm1987F&!gm$cy%d_fonKz zWTM(oZlw++V@DszoTuQPB?5%itzBrEAe`|g$T((-k(4(NFj5CghQ~yA?%e4^n!b?L zP$Tprq_|a{kszW^?~&Y4Ek!5^lE513iwMVU0awBGpvI`dUjPii1fKr=v5F8)Aw=;= z0@YQ_3yP$q5d`Rgm_)5Y#nY-5U-T0)vWZRv5|c1qYW=Md;s-sr!$nqxV;3RmPP)`l zcuCM=HhiL7YX8$lw?x)YIM01r%A*JjJ)O2gAU@_?x?RVy-6#7V2bB*gzc$R<)bFZv z{DrLdt5El8^X@$~|4U;PVTbW*_rI1sX^cCuX+6MId7asMeI2k}&DAd~*PY-YwMAJk+Wd&WT`R85 zO045n9VA{BWW^8pzN-)QRSUEx<|UT%!VWxwHactqE>7Z9@h3$3RjwgSL;&g`L#{>9 z<-(>$u=S#1mPBmx7Jy*xmAzNV#mb zkrb(qv~}qQBgJL=&g5P*EP*5RP+ChVDD5ODi97GdbadhqD4gLJGAAUT6AmO^@5Vf8 ziIvg zUI;`Gj|P+khvh%Ks;F#F?ww4r@V^*`cK{2NDi`sCUe)gBmlW4wkR{TgGsy8$_u|FO z6ohFrM^S=6l?ovTUWGs0f;|K~RjQgi93SEqt4z*S{dQGO_Dbejn2~Hi$~mbs%?K(f zVntOy%Dp8Z1k+1kfY_=+DWiO`jZADddf3szPN?DN-W@znYMF#%;`B|g{I*=kI zKpebIWCozP$CZ#VGy#QopwlWLnKtGsSvHd)LskW1L=u16z|wm<^PKMmOS@1XDTPC6HB&!Mr5Yt zPlN&IU&Vr}fA5%F_kjvPtVy?@p8T|fUIkLrZALfKU#qFD*dF;-kUg7v8_Dco`IvktXT3JUm?9LL6SDlha(X z-TFJDbny;H9+edhRlx852x5~#Fast|oAEcHmVZ-Fz25H5x&qbO;_uL=fbjF3PmFj%^zG<88AD|(BOpfCL|329 zh!%%$F-%y$O{ETML(mGQm6wDeo3hwO+7vof)zKrzp=F!L7PExUPZw==3igly&b9S! zYX_={wU>lkHhkmx{NM5N&l=wryLa^8_jgW6Q1prWw*nl%OMW!mt}lreriSi-BT)Jh4(BZ=+DnR8*^JNru)Wy-HKlG6@x`uQ${-va{{NF z>%Y+aZ7RlTb9{rpXWf;`XnxhaFrmrMcUV$P87Fx+a`x1l;be~zQf~gcCr>UVL=>~lu-1zljBif~6mi%eYtnN+Vo;&IiZFt&cLb+Lcj32ecRJKW zmml7DD%2FfgxxA9O6`G7E<3!|JNc{iLw4Ha86dwTfk;gG&2fp<1dEx+j_g2*_o)>Qss zD~nBfY|B0qI$cbw-Jaf79C^GOvw77 z6@2rULK76d&FRYHu59U?whg$(Ntq!p4O1Uc6T;KDd;EaZvgS+QB5-mdbdl4d3fPvd zb-P#gyY{679sYc(=#S<(Ha3xQC)YyCx_FmHF%{+8Wo%LTdusXjRL42hy15O4OBV@V znGV?(b(4N?4$}_e;INn)RqkZKmS%)LO5U92G0XJ1)uWt~|3gFkP%e94E&qD9^-Gg& zKPGQR^8_^cQ~KXum(w>jF_-Yo;-Zk34&-32O;ZflF*!T)?wf%susFz`;IFr=N``@< zK&;#%&qwQf{roG&o0j&;Etze%I5E62re8y}F^P^hXi^Skx+aul&xX zdh-SoVT1d>*~F%$*d%jAlpb)|&%Nniq+$8#XOoYeJ`VpnVo&HS*Ya!EFK zvk*xG(J6_>2tqkGZ*PkbwaJcphv43sp_nv6E9$JFb^BxVOH^h9WQfW8%!Ltzylkk% zHtOR$inRP>9_l0wA?T!gHa`>BEs6!XRrXs0B>llrLzrH4d>CViKAyCU!D_MT=8*qHnt4K9ym!*Oxq#zn;n>gW z<D+J*SnD*8ngb#gvr~cRn_LMm>@rM8Jo+tz~;qHxwxT>srh4Y=~ zLTo?#jbyW(Y~=Q^kEUn~q>SmqYp2`#evX_KOZ9@k&d*nnSlr$W`an)+$Z0K-K$zfg zCVa6pXM>OxszH+lAPS{?Vpt0nub7}2yXJH;E5{D&GR-^{XLjc7u-`&-%SJ0ckC9lc zIa-h6JP9;*ia7|Np=5cS9CNG^P<0Fp$m!nOXC_M+1;ssMy2}p{77miZe>!w0=fH+y z8mFM2AF`Jy=3-F^6bX~_pC5ZZG*f8&j$~t_XpqCw|Hp&YSxZ1Zf7+5}Ebzf+0eSoD1Y_?%r_H;3z6}hNJ{W%xdZC2&LwopRp}+6%csjqg&lKthmEXAJ z+iwkVuE+1nutE{{y9U-^YM66NgrPc|_U^54o=z@`?H8&YN+(U5Tqhl*(92e4 z9x5z1bBvA&9=uCXGN3)eajb(e`dmEg?aPvrQ&C#a3;h$GPa3BQP%jZ06ZQpH3CsQp zIjO zdXBOOcYL8UImMActEq&u%WiN(+zKU+kkJT{?tw7HDl%_sQyBX~{1WIFxM>>mkS}c* z7XwiXY|eVJvoY4|sO?&0*b=e&%PoTVBRW55u}{}Rp@l29bf20W!6mkf`LahgH8|zg0FBi)W>K%<8k?&AI5L z{{kj|EqKPOGr@@ULt)9KZ8RBDe-^FRh+mGD6eu!dUbYjPRR+$)omflupYaxF|gbdzV1z-ycan z&BQ+Y>&}i&DU|BilQFoD(BtbUh=$qsw~O8&gN(|9?~e>4>a5RS<|$069wWh7;UB!J zX4zg1nF%AjJCH+Rj4Gt%!-_oMynKoU|4dtOy>b-^=@n+z8?g@bWH+^=KO*$Qqu^`5 zVXDf5rv>c&i+{fe${ujq25jC8KD?}hj>jCJL@jkI9+M)<2mSCZY{#ST9nX9|3P;%dxfMU34!~$al>p8pbsUSpR6(&fn3{VE}FbO z$lm*NOTk7%z6o^qVAH_(QN6P*c`y-TA5Ve)P|uw)&cFrHNzYEa z9CS-CcvmMOnXO2;Vw3*7woF+G8 z&Z|)NwP2Gmol%D|WQ{a?M!5|8XPfgumP$Cx(tNF0Kr)iUCGhfgL#@``B!c@m1$P5= z>gBK}JeGz7>510S_NcL8et9lVWyvTM%bp=8%ayVIZG+rsM=v`3D2^?)I&n(Wi3d4mQ` zPW%6{_1^JR$M64eNgVswdmLNliR^joqG%E+WE5qkj;wHS>?Esml$`8M z*(5vn_15R}{e6G;eg9dHbDZ~hjqCMX7Y=R5oSi`xjXEdVba3BZevqkNsKvLc^pxZ0lG%tAZ2$FX1`K@}z5P%w9qV;+YKe()>LBDVBSg!8{)Ch(C^D z@!a`{^)eMu#Ka;QJXqe9gNlT&mCGcp?WxL(KB@PqTy<4Zs;DGoHg`VsFhYqq7IYQn z&(^Kz+qB5;IdKRT>8$kUX>{=71RSQrO^ewbSc^DLL_E>gy^ZvRhSZgiRX7nET(Ar_ zP&{&$CFRFB)X#3k>a6chGw+|L5)Um?Fg!u{Z+OzqAI}Wy=hPOVY6%u6P@;gebogkb zhSS#fEVW}Y7>5)n1eyEMY7#VWv>x>`WZ>{iT}=k!Is$?nY?(wAqAp%&blH9^nergP zfyjbl!kfr*4G<3c*T3Xy(8#qggLkV4i^^z~x+_TMrmAV%b@| z16v(|bMfFD44)0lEmhg@5Mm-#1$YGkue{|J@QOU-6w z!1B`bimsnIkBz$a6Z`Or8U~pELkpkPNgiL{-sw)}k-MFKocw0}m!T&IEy(M9tE{yXX5sW`qm%>iO9-+A#)Dax^EEsixZ*t#gzfdl|%(c*O z&kdKW3yer~AT1pY8^T;+6i>f5$>v65i#jE#dchEK&eo0nM1BRgM-tuZqJv+F*SYib9n5Ri1{T=X;Bk8*m&dyU%j8dEPWkhXhWTRz9%$9moJFZVi(-%WVLG_& zRID8`&q!(P26-}Ncipf!Dj!1*y{^{R6^F7}n(K;D=?{f;RleT+5cILdOifR;LpJ05 zTnNWGR0Qp|kTikCQNDSV{vaw&wgIO#F4WJMlHt{8lGJ{?#oKmURFh-dHgHpbuHz6+ z$fvpV_L9N)&iM225*8b8YuP{ifJnAfbM&=%F!D8i_m$F0J3)T2BlPgNq4;=T;cUtc zm6Kx;lalqU{tnka>taSEF1+Dol-05$2`vz}%4?OHW< zqpVUP|3l9FD)Phkk9%|aLYspRR#-Ygn8qG zbNt1tS7%9-4yKlZnqPEch~c+&IDgz%UHPoKx^?uFT4fLV$Qh%|_bM_^<}AVtlVvYh zEoKc8C@*~X`7qn4XV}5rOE)HALwu+tkTSX(y3r6##3zkjx&w7>Gv+xDL-V~2gNzod z6TL@lZXz4B3@62W`7?KCH=5ZIY}C@p5^rw zUtwy};Dlv3(vo2hmf^Q7J8fy~6jgUkQlsyOsi{=qh+z!sOZT>u_n&4np`6W*4Tr{N z9trgk4{}p?>RIm|=CcYHqg&2i%GGh#zJJAt9b3Zjz4&_1>r!4aa>2w#7gG%vKN}92 z79+B-a7%5x9+e(Bl0kRrDZ-8G(>x9%k#u#)GO9H zo7hSYd{8&HUL19m%vt)Jk1227t_ASeNs>uReE*O5{un0 z(%p5ALUxv~tmB$7>?~&}56O<^vRkO=v=m~O=Q3@C?tVpW7@b1PN>?^61&<##{9%`*MR0+87H=T?~+e8AL8%uRa9?jh(##>PtZi*4{uF1dsM}m3JEneCq8f zL5(l&oD9wMv}d@YnOZ;|C-uU5s$^d6bI5MNVzS;3ujK4}zH0SmFZ$+)&`%hG+ zB>Q|dl z??R4O%}HGvyOnw&X*fCVly-n$2I`}+iS!G;u-}vISP)H!a1FoOz#fxpZj};fSCt37 z-4X)#MG1{$iQUIMtCTC;fKtW2dby+iL{zFJnuHf~H$}oiUP~dBTXwF%FSC*>gah74 zw6E0Qv#ck7lUd!`jAri*wd$08eM50gWkTNka!LwP3Km~#w>x4U#@T(K0#0g3rX58X zC2AUP%*aPvtWlG4`3|6caka~A(KX?RN&?*O?0*UD8lQE-&LCdIK}uDK>g)Z^Hfz)uWdTQCh{>>RjAXsRUgSK0h<_-8n`yL> zaSa0LTs^sgd>gqE5b&~x1LG)5El<=AaJK!>n7g?;t+eVqmf+b~L}lA*UqnHcPDM0@ zL*upc5fyNj9`_^*C)1@-=>WNTQ&ymMToaOBsxPiyaF66P_ZYfo%>h}5NJdFOzhoz~ zeZ8C@vsd6}hjN(XXUDHhh3ghiu>BK9Q3p7~{Kv7aF)u!sAvl}3vQXKb&)TkflG3=8 z2#GL9BQLTkSVAPEfWg5arP*s_s3U3(cYncYn8g{en8yREG`Ti2qZagjZu?Pn;)FNq z8E1e*^0NBdJ5KFqwt=Mk>|u`dc_yW9{Dx+=2AxvH7z4IoHeF^@Lfv>I^^@BCFMzhG z-E6+qrM3I}j0wAKOjejlh%XX=+daARSA9o;0TBA|0#u2ae9OCrZ)NA3rM%r{(2YB0A2Q4CvbT*sYo~IkhCI z$Gpf6oMFxeYuA7~uCKR+pK9okc{U({&KpMA%6il{$qisR>Z z8kF}y9emMe$5xj>rG>wWk$VgIo_CL9-luysaUwZ}SQ)k7+(c~KTemOYU^e{J9W#x zxaP}wbWw-uZ0%0`G)m3tT)nKW>m&EI?PGKp!nBcLf<_@ileWf`p6zWvCnd|1r#r=~AFj&r{2nwQ+(eRvI7 z1l6<=Ll^R_fA%w76fKBbSG%+D?Ez_@f8h4pyVB#-kP95Nq|;0ExdU3c;|D%wpEVYa zw8X0dD=T>u4$UgB>~X}_;LCN6pC3)u({T&p8RG4!3u4Sl!^G`4*D#K$9|^rZupZ1d z<|;>jp^eLb^oXnEN-bHS$%B-{w=u@sh;$wk8MG)$w0u^9!Q9gQ8!JW-r^`x7ROmHg zDKaW8xx02<3BPm&iNV5`ptI}1l#={d-YIr{;TmzomN3&1={pfrD64ABQ1evBpIZBm zi^v<1Lg(gaPWj3M5hYwhA3#^Q9Gn~~jgzrEU#{rXx37LNKVs5u=P(bUakCZS=yUfX zEMc8Hu~R7)X(>LMYe&38p<+cq_vk~y^#Is({fBbHD9F@sYHtqQB<0t-jYq{z-o5Vp ztvOkaxnRc2;?7KD+&bI1&VrPr{TtRPNylZIn&|y$R}BRuzx1+tc-Y47NtC&&NqZkm zn95lcQ=?Q;bRg_JmA&=wn8yckL~4rB;GQ+mx?pOkFY5*qenQagEr^t^@CB9W_#j=< z=_tz|#KW&n?}MIt{&nkgQ(5db_$>CKB{@h!R-DjTfUvS_4Y_dEs{q`3^O^7H=>>R4 z;g=gM8gGKf=Vt3eiRFdD`ed%ksCXRw=YK&-O}aWsX(#sWSA4%{s$B#e zM^n8a`LRXv1o6l{772)ImySvCu070N3fQ2pw=3+g%Qqo;I%J)hsvcM%&PCPhzvugV zr7ym3Ne962)eUNYwFee{M?N5)7u+{?zh0=_PQ$5eYXPG-DewEom7mq9BUc!n#egbf zkx}_NXnUF~AbYa|*1T!ZR1I7?FRypya;baPEa=L3%_rgh%#9hNPCeh3856gH7rFMm z4emglcq3cRbw+NCo`WZt49Wra-s+)EfPrLISK|6(?BaXp(LamB{ELvas%-j5VR`He zq9?qx78@ZdEEf(6jvfstQL9anIRGcoeE#;O@<3oQ2H;oypB<1?45t|Yy*0!6t747k zj2oot@OYwv+mN+KE5zjQYJw^?d(c4wLBTC`K<)$IcrU4Z+c47wFWvmIg52I6L#T{&w>3 z`XUbzWu8h(``VBz-=SQ;HuI%^eyB?Ahy;Yk9Y5k2!=us-**Rad^kC>Vf*N&Zusjpq z!_x-`GSn>~wYB=qt5+WD!+|wckOy+)LhwNQ{scS{Cnqar_x*`Z$N=r{ zlf&41DWL2wG>l@n>PMA2NUK14L>)uXJ*DRM^&2ES`sraDPk^ zTPd+oewLtqEbs+m!u~8A2c`OoZJ1lt+WBnxbJ;1!uC|~TIdwuq(d7};7%p~LP&^J% z(%%9Vs%+c!Fwu+4p`p_!L0Y^cPe4yU3;&0K=SZQ8EIuKO_#LP+Vv*aU%?<+kYwpY? zk$7>(0u~4Xss2?ZOwX^$v)I8Y`V6Y~6P1>w<@WX4T?FMvelVK;7VWhN#o?1Xw}O2K z2~v9Uo>x7Fkj4fdE>jKfzq{8_kgZg9GF`O{Tpf7qiFh2c4j3DJ>}X_`H*dFNtp|T! z83-0wcz)IEon^>qC)h(GyMi#f2n6fn^|X+mz_%zvy|BGec8`+7cn_rdli%|$Svm2~ z$p0H~yh~LK*f+Z2^qN#VurrSf!bPc5fj6NXLe|Ql zt!f!qoti76n#fmxVgIz(r+9NfS@u!E-E3q+5OQox+OPNKrB{IYZ(wDzZU2uR!bY)N zKuE)X%cE!^D9_Vi$VS|mB>32XmQZN{soefhuN;6tu{Ox8AKDuPPiPoP)I4wp^xfWI zExQB7H8REnD=UPTqDR#6K_F5pvS}~8j|*i(mSo&nF;T~KFdf|lx^MwdMPd}p>1YVP zNj}*w*x(FR{!MU_E$5|;>JXfIBg8UhlFz3@;{o;(JM?XFmxv4+Iu}>h5ycbPE=|uc zG?tj#H(H*ug++zh+TYgD-(`%{5H^-Q1`@v@_Cj`9n>n!HaV~anSMb+${sD$%clBVw zcsrm_t16pWgzx)ltDXGIdRV7mvz$mf2nj!8O!>jh) z*TLGo_Ap%BmEg|}YYc%KAufIIU#UT1?^dtSWjv9VOQGfR_!df@Jm7{rxe2|e*=kAIYWGz ze#x-Nt3#!E`hqI-pm<%mE6?=2Su+7r`QE4jy!4k~(a7(slWs`k>?cNJj3DNf_`A+fd%h;_ttA+9NUP8 zlHZ2=I_Ou+LCU5!bV*K~K?Z`s`7e@SJuGqRH{fE79fD%fx0^#_(|KsS9v6 zXaiCH2{A_~Y(7g{HL$fsB%Dnl40(k^#>sCC%Y7zZd`ZGH-NX;8evt z=$3cl-uw*EhRTEYfBw9OPd`8PZ3C(Isq1#7((m0Xs$NvQ^Ah=eb$l0|*21DlrNoI^ z2xO`xwaeP#2Uy#n+t2_gS0@l}8pyTj7yX@jW%+~7AcG!vR&;sYX)0tV=VD~y2RMat zGZ%ptzowxgRsg2!Kv~gxtGL6=Y1-@_kd)00hcgFMhQpar>RnR}C~IELV?KQNpm~<6 zQ-(l)zFgakcv$=myDU1EQ}tHPx=i_4q@RaO4s+Ehi`4ciE$@Ls|Z`M`~M{I3LG?_tMwJ1etGq#MMf0r)847^!f^`u%|W=v5+!6I1#5 zJG{Zn^_=|ulIFqK#@$k|u2u(v@#s;_O}u2NfL(V$pIxw(xJ9EGWkk=b1bynI(_~|B!Y&Yl2v=^;bkznHQ?tIuVHu!k^U|`JkUeoHaDpaNE$`Sm z<+?XE-)@xCDQy%A{YmJ=)qS?2?6z6tpZ#5s*kro1o63JZVqi4f3@YX`&@(#uBze@< zJu0cJXp>)GBdCbRC>E2Q^~K@El2laCl`A%;L=;g}d>tSEqk%-A{AJK$QREoIE-LOc z>R(v{`eeT{GY)~XQLCrJ&MCOARYTSBtU;KZTTislxO+>}e{16?>N1K30}g@+2FZ!! z!+*Yrg5)J#Z-ebo(y6_hxut>Z!X?x-Ow*n zfaehE;tJrRLfP)%;D83 z=dBg)|#4k|@UT~g!=P7S8yit1#a z!kzl}3o}9s#VVFfs2s-U_$8!bny_PI-r2e}tCz2xf1J|pBa~9R6KNLL?159Xq19&5 z=FySE!?9(jnA9f8LWQ(DWV(pHn!|mHtJpk_H{3{f4BLt+lw1DWKm0d|>1N8qafja^ zY2KA*8%wBWW!~}d${LUaj$>t~f_Ro4K z(RMk;o`;q;dS6K^n;Q7|O+QeAvN&$l%LeytsW6-Wck&AHoTxz+Km#`74r+o-BZtHA zdTQ-Ls2nGlD%Cd#Gham2w8l$)*KZE9Yf`#CeVaBMJF4m-4K;jH`HS)9xKlk0X$kYf z9n;l^>coGljXAv{k|) zqY1b5IB|5dG!3kLr&HCBkzzMOrqMWxxIfPfh5MCFCM3P1pG{=(G|{P;H|H>tM0jULQ#Qwh4!Y zjV)&_2Nf|s2T@X&jrxO}1b3s#NmA1>2X21S5>Mm34MVNt%c+eOb5H@SuVeh%*_%>3 z$;SlvqLbdlRQDa*XWr0)(BuhPh7n`!ZkjyGMh@uemZ+d3O_D-XqJL2syk?^urvc&P z8>-q6D!EZllpvc7N7zGK!~tm?lc}EI5;3kk6}PwAeD<#>IXiOz+5}yKkE1lTc$ZFD z(4;3;cZgA*)Nyst^PG8;dQJ;vV*!zjP-y9^oa|%H*yJ$rRcn<+w@YUy>(qOiT1RmH zFM#QBjq2NSw{2#Go8OFczU5qFKvS^w36zkiD|A{F9(J%a}7XH4xC{EgKPryEol^El|8$0V3mL^Dkq%`zNW`Z zbC^bF1k{5m><6S=XTcX_9C3LmQ+vlx&nupn~)?K z%|CC@Ks~z$1EuPt!DA!O*e4Eag3ba{W?@|h4O*>?)StXfN*+B6GHdexsKUqqxx|l{ z?l19OvgyOrBc`Snjke(d?j*cc<}&4je&#gJ~JcQ>um@z}~+X4XuL9*?XcA=vd^k8u;9v!r2zE#&}!l85JX^5z}Wz|kh zV;SD${V2xh5?*@wF36>QbPg;slQWVZsH0@eA?<;uzvoqRzSn8lU^<88dxZfj<$zks zenS_Al`z?{T5vBnj4JEJLUN4HC;=_?YpR#BAr{@#-jLBH8R1b>U|$nKH9-E2znIDf z;)H6_<&KL_+znpiW<0{iz?g~-M^iHPm@MJC>Z2l7vfNpzwQkSjMRH@*ZAOn7xE2mv zneZTUiJHvHRv!;_Cz!Q*I~>z2?$-$IW!ijg^vDhKXimOmUQea63|Cj(eX6ZQ5yYaZ zV>xS|T7?uXN!N^&r*6kQ$UEdeuK7WDvmmvUi>o&{U$=ulfG@!UqOfi3l7cKEQJR5& z{bc7J-Ix4Vw`$4-j`)gui9+EJ600N3y7?M`+0X)L|Mnum%gaKvoc2tg37P5!C@Q?h z==0#|Fp*!1x=%1Bodd^HY5O`&-w!N!%jbNCq%(*vBTlT{RdKbQUeQoN!;+4~m;;RlS ziHt`lCoBxAA-LxmC_Xm8|%+KpA|69pvc`_?A8;6+jwEjONM zZ@rHiZo2P6WasIhuU29ducjbgA|6l6^iA}o9F5LD=^`&seLAf8(0Y82Lnd9zR61Mm zOV-{D#g*%C?8$=p`x(tYnL64yao}!9$0DJZgG$S!<-c*ON2`UCg(FVj zWdzm*L1C%ZlP+^mnP!9Nef~x4{96*Its`;3Dtp@SYsb+N~fzpxq zuxS>5J=#%Kp#{qT$@;BENwQ%$7C)wzRM15gZ4BIo%vY7^4sCB<^g=ad!IiT4B7IOdK(7>o=!^j5Lh*x6_tK<;e$%h>w!2Xc>xc)mf{quoi7t zgAJ^$B&M2dH(HB-A4$MXkPS<07Nm$%`@dw#F4SokOk`U9w7(%fG^c zB`RfR+}|7YMcZ3O;xS#i%H1Gx-W68wFn&nY7o@~ghtBQ-b-=vX3Idva`r@I}I4fb@ z7)N!CPqk7knsKa}Kz4ZO4AByG3OWtY$A&>!9l*TXEP+ zMyfSS#!j3J;}bJJXuGQ*J{28C-iE-QS%`n?v*MI7m-9lf26 zl<4H4H31Z6E7YDXPQ+l(wbIh`0a~XiW?~s-7cJ3CuXBGk3~d+8@-AAr~l%y?vEcGiq6Ptj}z9 zV)3^y9iet95jH=b*f76Smieu?4baEw$Lwt>E{r#JR<+8 z%0Z=le_{??se1JRTIRwz>-%jg^~Ihs?s3Jw zD4JC(k94X)hlBUP9Jy^jhK>2$_uO^y4(IxL^X!0eBJvesf*@==<1@|BgSb(8Ql z#)F(ejI~lHmG}94j2>nwL~ht?GF3!x5@^Twe6fPYUh}Yh1s$xVSd<`&eJw*v&Xn;b z(~B+;@>4fLN~n%&VEpmEqrv3>a%-j1A65|1V$VK^3H`>nF5=2=M=>qX%!En$_8De1 zRiH4f6s^Y7xTykW`=JZAz9u6yRep1%eENY|XQk=pL*bMvV`-E2k@`9#8)hG=-)a(@ zyPQe*Mlv@0X8g*dTh(^?tBQCC4CDl(2)R6Ao^>NFTS92S%`CD9AR(DJX-E<~?TQ1D& zwAhfj~M6rD)NXrb(M^>_vI1K`}Ty3SWbPsIznuC$Z) zdrlMT^klpn2$7XlCFSG(qh+X{g2G**{YiUA&IIPlFiUpVVt=To&)l(yV|+KoZa0rA zS%&PKWV$ywAgZRW6KI(j)YDT`U~LQf!ZY>R)U1#%xOS=nULu|>Ar z^8e4NFcC5Zws(pq%2{D$2io(&k%*Mg4Uis;GpDm3S3lUPz$r^8ttR0XIdD&w6oi~u zzm;&{{6%?KE(TW5N3}T>oi=N|Oa1Ci*Z2x%cxzR9_`c4D$Xt9UZSXl-oA`-aLgtz> zSr{bE0v0g6g8USgFA_*G52~>109AIr%23EO0dAFKns)S4HH`?H`EiMyhP6hvumHwK zhIZFIO)ZViTUUQ$u?nniZk|lY%`$WLrPj#PyyiSFL7{$>idUgiYI|Tr+VHeyortCJ z13BG)Z|OMQu)gtHHvIDle?u<#JvR>jh-51=I{vve2#KNZyZ_vpc=+MzNC^-B0?U*X z4=ANJ3FAmGLDjT%u-$Cu`bMeWgJUdQ7kwkMPH!Cq>Bh2cQ>ky^iZe-xAWd%PnW58N z|Id0u^s~hZdQw6FW6xH#gkGq;&&t)sRQ-jOXFlG5kk;L@20D9ybFnqmLR;c_3R7=# zM;(vO1goHR>k4b5!wF;j0&cdm#NT66Sn}}mjhCle|M1*6@|^#k;S_XL^h1q8fP|S- zW<+ghNJX>>Pd~fH==M0VhOVAYDuw7r$J3vJ51MV(2 z)5k?HeGhq)t3+5%q-+5myE`1SJZgS<{@IXO!zYC@H`noVsNciqG4CsnXD{n2FMtrm zFJ%;5KRb#Qi@I6o_t7-s)(F(>cK{E*aO9w$eOeqf#uFiIHlMq%B4P7!OGmrw$7>kQ z)aAOm&iR+tJZhrBxB$cyK+6*FePa_=>j8uegvL$-nS1M<0gtIldfc`Cc}JhNLAu2# zDDD;zclNhCUDqF7rZP7j0D1<}EN?G>EGYI4uTFMK4^Vf#ug^)A1nvdMo;*8VFJv{t zBU!mDdGMDlXl|_`=5r7lME_VmUiA8tLIy71??{o>Uh~GoGEzSXajYMB8r_ zYIIR)_$E{QCmQpAH+b5!=iG%dgs>I62YGW)gr_RIG|IIT?4DJBd z1=Icsyw zT*cjufkW=&9+RV8lFyOv%~kvHYZyJ0R#F_l)=)TSFr>pAVTXXAw^kK3yYp@lnTu(S zx$aC>rlbR7J+}aBx(kkmXJE~%M@$s~{MqMJ--*RKYy%xLc1c3q9%tL#c!Zpg3yfpK z<@!{4r>2Tk3R&K!1Ml8L|Ci4Qt-1gSX0P&(OAT5On?8Y#;Sz#W<^==k*i&|Dc07P} zfAR_JdG*j7sF54ke(OSvz4EJ zP>+aX5qw6^Al)BJVhw{Zm^NAOhPRV#$-c{!I`=#umUv9q?m!yJMt%d5$Z$@~8|i)S z{ji5K!8^Xqp*KQgu{ zuvV{tSdzI8&=F+1W*nwQPQjU9+n+2(D~)D0{P5!cdh@}W7heU%sD|nGV%aob?^8pD z%tO&0hJ@Vj)*knK#WMFGM79Be>CSD^?o+X7Lk7>^h_N<+9=h&&h^3=00r9K&e^>+kH=;_rka|T=I_n-6>)U<3|2=;wt;+<6u!$keu2Zp}2O~rwG zfB!DJix`ch5#3!4({v~7U1t*3qP7s`we65fopN|094EBD(acgbd4Oo?RY5_+U%&gP z4?+)*_F`WP|NXmO?V)A&ABxGg(W5{ZtRL0W zuE0&(jA*Cev=L@v?{Sw}(<|Tg1sicrfce)s?t-Rg#{%%|2@9t6=r$da4?$6UqCan5b!rdZy`<(YIyuo7(h+10ipx&am< z*?NGmEcqC=Z$Z~ENSQt>aC5|;P->f@jndsYelKSgR?Y8JhBTp)mG8vklFwR9A`NJy%#R`=_{vIWsJ!~9U|_1!IFl&o?za=H-erz(r$?n z$MA0EV#ei+u^^S0;opd2llFwNDSkM7BZU1dNs&2Ujyc4xE|KGHbmBO^LA0+yv58Rp z^?b3A$(N?bH_Ih%3=&PEX*N?f#2kXtBV3w*E#1{rf7Y_u=u4<{wf6dateQuqO(1=v zPO+o@b@cnkv$a^;8NuF$JPH5&<)d$Il@{ZBF!#rP{BxY^sibQ9m!B%ITNSJ>vEM<* zTK=Yrm*g-%tZcuOvH07sxziE$`!%oE5bd`)uB@pr<&uT%t!*i4vku~KcqOD*Y~Nms z6DmAhRAc-jIKD|Fo$(xVA3{gP%rJYRm|67*)T=_#M~@vlrWqQ^*}=d;kAkSsSngxj zAC_qCj4}ia;UUqs0)*QieLr5h^+o@0g%(^u2X}kmCo6;Z6ivscA^7I`srFMWjQgNg z6A!&TeRTYmsKQhrhnJNr^Z?a-cniA3Er5XjHZ-h$r=-5({#U!*4|Za|NdBow5-jI) zyv(?IMvTJwtB5Y#|LwPyQ=2j=$rw&HHCO(y%N9Y%eH~E!ZS@Ez28-Q*N)T@>3D*{YGUWqog3LSjSb-yLLoTRCbm! z7gv93|3&iCShgvTV(X5iL#i?^lW(ny|5$SC#w)Shc{{AA=&0z3Ds=>H+@CNCTeet# z{GePOcd&GJ9;cwJV{e9UaN|<@M*jk{6q}pJvnPAD3o9lQ!j4~Zt~U~-Ye18-7@cgF# zZ#Sk3b3_ylOf?S_O+|#tM{!1c{+vQN{yx~kh3&5YsH;vn--_Pj)aDethT&WPTIYcw zE)*4vV~Y~%2@b8qrj$AgDtpA+O@5=rQhIYHpYq}*x4F@yL(BS@drFFx3<>h_#60P<$ZmhM&b4&VA=8KEMMz=p$(DEQ-$nKG0@62E z_x}D6>o1|?&E{HLeq(_5Q)W{~H7|nr4-+8VRwt zrrXXcve#|Z&GHs&@un6}V~&ja zgytJ)%5E{$M}`yLW3oFQE4}N8i80rju8)#rC%#dEjb!g#ICkAUIaikFvxILL_vx_`zpb`|Pk#+AyjwbZ7kJ>252Y zV;l|NWxMVyi5@#_KoX#8$LbxiODp4Jb914Z)HPv!ZaS)RP3dOD_kceynG;qp3(WQ` z)ET1Msbn|$+YifY^4TrRuoc}?#@8M43i1+qjcFL6jt~zi4ee!rV?s7`k!VS@MqP7l zKI{eQ=|hia!OyaWOQW*IPn}~%B@(_hsr-Xz;A)9a;k+D5Nt;UCTNl9yICj}vC~ zJ+-#@!Z`OuAxKGQ7?su_Re4n4(#*N%%y(5nvy*PnG^Pq?HC4VaK}9uxZNN3is7F{N zJ?K#|DmM@pqU!mi%jzo|n$p4)C7qUh!}vyfcsazmAuOex^IQRZYvP60JZ#nye$0s7XjeF zj3PBa>AJ^4fi8}B zpp*5+dI4Ghs|_OMPLs-_iPV3G6R5f|_lr^->Js66|2hg%D_ZGzl)E5ZSz_qFpTIrj zg$kXRizM~uK?L6M1OL`)_^uB0{zkLjPynIitZ_LSQ$|)EulqARX^zNUSY` zaME%}SuF<*YXkTK{{23&FhSh$lRW4?paXY?2jpFPLTPrg^1nYz5LbhM{et_@vjT*l z2Yr7OI{XI?g?FsLfn>2BT>(s`9N>Z9En8UXu`E}BQ#{Uc=My8M5>TIb~V z^@V93V1fVNr#jzXQo(#m9pWaxH~o{VBhwqXUhnz7VNAkfo}KpZJO&SliyWl5@EQ;W!gFvgYWk6)ETJuOG)U0@GsgFS>qpT`F8 z+0(+^fVq``O%`ADe})b6=$Wno3XQnb;OqbSinto`kRyPes|B(|c1Y?!*NH>sqRYp7 z2N#f-A-IUw_X2l^Yj#}V06PHS%hUgL!7%3`3wX`|G9TPPMR^bqzQ7&CdNBCy)tUd- z2g4Zn7Y<89XY+2*Ln0Hc;qm`9w$fo*=UuJ|fP2zo_2|3yvTS+0x%vO~83-iKyR$g7 z|CuEiO@K;nSQ3DfuepX*zJu@u8~to6gSucB)u$Qz6U zSCx`;*OUMKpgoz2POAQV2LF{!z0te3+g1|`<`D12A&vk0H(dY`$umx9umgo!gQsJ> z>>J=&zS#WFtK1QOeU|;wt#~!?#SGT$?Yg<)h`+{Q-gtoI<*f@e7XRO~F2e3R96B%U z{oNe|@6Bn9A$)iQyfc5-Ko#W$UX7T4+C$_SO~QCDhd9$q$SVT>1>+(UAcNry{D0no z7VHjURqXTspYf;yuNNgRyu1pi8pr-8ljNi1>Zd+Gt^naYf5EB$K6Y_2xCoHX&r#5F zj1=D-#wXa9HAgD8+s#_fQY6b%4YKL$#@BL{S%27Dkg`4-Yc z{C`av5%YGzc6V@S-h{>(60&rdhQLvI^XOv~78?gUrPzWIwL~Sfi~2Ws#Un~K#!ynq^bGXAnxq3 z0_DVa$Jkz+O#cOGx9W)TZciyI9RJVxvi}Z5;T?bjaWF<{XaK}wDb!zPqOZw5y`wNb&)U!L#q49r zhjfQuKx~$w$ltv+Fz61Sp+Gz>uW6gjoAQgnFZ4NvPoocr548`S9sg&iN`UvZmwCml*y#vU{Ms z2sQl9duL#@Lu=1(?~YXQgxU|8V;(>~k;Y@p9=4oo7k5-2UwvML`*kCy!9KBLMMP7R zr;}79na*K3duY9!KJ@Zq}uVlV0sBU;A!Vbw9L}_s{iTOMu0m@BRY#!wO180HhS41THK2ljNpsENC+TW4D!elmtL857{bry0G-IS2Wa(; zJ?s^Eh<)RYJJTPOHtw}PMenFVl;dQ|7eSJZC;9uwZVEVDNb!pc$FVf3oU}tl+ z3KrB(4M@{ofNC~_an;hm_OrR z^r|J~8*E)yCO&ybo7qJ3cspLO_6S`MtCQ@eHCK5L8PkYcsT8q#&~YAZ==t`o6!Ej| zY22DIy93K@PXPezA7Ve;9PC4O!92+S2iv7>9@{&Cy>>uAi4Bgh?(_;GEI(ZT4`*z9_FJpss3HcCV zta2M-`F-tiu^%9XB^DOf7qK5;!HPwULPLEYT&DX<+!hi1^DO|XoOB*fV4chZz%~!y z{2@d>--GZDZXmk-11)L!d%>Uv#(gcf|7XC`>AP9N0S0s6h8AEUt)L&qBWGiAQKP-k zM{ToFSYL60-kEc`xtzUbNn6A>f7`qsczM)5ymnyJ5l-`TyysfqhP5Am%$(4AvHVz*;ltvT z`ToCW8#>*9%v1u)mp-%3B(WOcD!zB;|NlS|yve|_WJ zTpCxEMIiyYBYB+TGiM4a(!& zCw@x;4uHH{yZv4fu&uLfJFsy0X_pJ!T6-Hf=Kp#|W%^#=y6Jmy^?yr2w@B>-E^OHa zJUM=A-`dE{$8H0U4c=oIcLO*)|0EaK^2p2uRzsSP^H1H{2t1#z9CYfX?0fxx=Z;!j zyEXCj&$E}`Obw4K1a89f``W$lQu;5sS-qXT6MH4Uv&-f6SSRNupT8f&zvtl&vl#6& z>*n6I|M>MUaO&sv4dAZC=;OfsqVdN$x~&TyG*|;ur`AbJ#o|P}6#{#>biW5%9%g%M zGgF7J^oUN%g+0pLx%m<6uO5~N$iKtK9eYIEI5wyfR^BYS104Qwv^zQPvA~fS^_w#_ zo>k1=k-l1aZb8B{?WHf&Ti)t!vsnHnvyV{ zrK{X!YPojmf6|Qi_zopr0AHFMi~s-t literal 0 HcmV?d00001 From d1c30781c7602977b21eed367929fa8c0fac8f11 Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Mon, 24 Mar 2025 22:28:51 +0100 Subject: [PATCH 08/45] feat(ci): build also with Java 21 and 24 (#2738) Signed-off-by: Chris Laprun --- .github/workflows/build.yml | 6 +++--- .github/workflows/pr.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18fc5ec1ad..530921009c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: integration_tests: strategy: matrix: - java: [ 17, 21 ] + java: [ 17, 21, 24 ] kubernetes: [ 'v1.29.12','1.30.8', '1.31.4', '1.32.0' ] uses: ./.github/workflows/integration-tests.yml with: @@ -23,7 +23,7 @@ jobs: httpclient: [ 'vertx', 'jdk', 'jetty' ] uses: ./.github/workflows/integration-tests.yml with: - java-version: 21 + java-version: 24 kube-version: '1.32.0' http-client: ${{ matrix.httpclient }} experimental: true @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [ 17, 21 ] + java: [ 17, 21, 24 ] steps: - uses: actions/checkout@v4 - name: Set up Java and Maven diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index dea18217f9..27742bf7c2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,7 +22,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17 + java-version: 21 cache: 'maven' - name: Check code format run: | From 76088c1843eaa7cbde05ea8b64b6ab96f24bdee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 25 Mar 2025 15:39:40 +0100 Subject: [PATCH 09/45] docs: reconciler improvements (#2740) Co-authored-by: Chris Laprun --- .../en/docs/documentation/reconciler.md | 87 +++++++++---------- 1 file changed, 40 insertions(+), 47 deletions(-) diff --git a/docs/content/en/docs/documentation/reconciler.md b/docs/content/en/docs/documentation/reconciler.md index 330bc15ac7..26a2a20d61 100644 --- a/docs/content/en/docs/documentation/reconciler.md +++ b/docs/content/en/docs/documentation/reconciler.md @@ -5,49 +5,47 @@ weight: 45 ## Reconciliation Execution in a Nutshell -Reconciliation execution is always triggered by an event. Events typically come from a -primary resource, most of the time a custom resource, triggered by changes made to that resource -on the server (e.g. a resource is created, updated or deleted). Reconciler implementations are -associated with a given resource type and listens for such events from the Kubernetes API server +An event always triggers reconciliation execution. Events typically come from a +primary resource, usually a custom resource, triggered by changes made to that resource +on the server (e.g. a resource is created, updated, or deleted) or from secondary resources for which there is a registered event source. +Reconciler implementations are associated with a given resource type and listen for such events from the Kubernetes API server so that they can appropriately react to them. It is, however, possible for secondary sources to -trigger the reconciliation process. This usually occurs via +trigger the reconciliation process. This occurs via the [event source](#handling-related-events-with-event-sources) mechanism. -When an event is received reconciliation is executed, unless a reconciliation is already -underway for this particular resource. In other words, the framework guarantees that no -concurrent reconciliation happens for any given resource. +When we receive an event, it triggers the reconciliation unless a reconciliation is already +underway for this particular resource. In other words, the framework guarantees that no concurrent reconciliation happens for a resource. Once the reconciliation is done, the framework checks if: -- an exception was thrown during execution and if yes schedules a retry. -- new events were received during the controller execution, if yes schedule a new reconciliation. -- the reconcilier instructed the SDK to re-schedule a reconciliation at a later date, if yes - schedules a timer event with the specified delay. -- none of the above, the reconciliation is finished. +- an exception was thrown during execution, and if yes, schedules a retry. +- new events were received during the controller execution; if yes, schedule a new reconciliation. +- the reconciler results explicitly re-scheduled (`UpdateControl.rescheduleAfter(..)`) a reconciliation with a time delay, if yes, + schedules a timer event with the specific delay. +- if none of the above applies, the reconciliation is finished. -In summary, the core of the SDK is implemented as an eventing system, where events trigger +In summary, the core of the SDK is implemented as an eventing system where events trigger reconciliation requests. -## Implementing a [`Reconciler`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Reconciler.java) and/or [`Cleaner`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) +## Implementing a Reconciler and Cleaner interfaces -The lifecycle of a Kubernetes resource can be clearly separated into two phases from the -perspective of an operator depending on whether a resource is created or updated, or on the -other hand if it is marked for deletion. +To implement a reconciler, you always have to implement the [`Reconciler`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Reconciler.java) interface. -This separation-related logic is automatically handled by the framework. The framework will always -call the `reconcile` method, unless the custom resource is -[marked from deletion](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work) -. On the other, if the resource is marked from deletion and if the `Reconciler` implements the -`Cleaner` interface, only the `cleanup` method will be called. Implementing the `Cleaner` -interface allows developers to let the SDK know that they are interested in cleaning related -state (e.g. out-of-cluster resources). The SDK will therefore automatically add a finalizer -associated with your `Reconciler` so that the Kubernetes server doesn't delete your resources -before your `Reconciler` gets a chance to clean things up. -See [Finalizer support](#finalizer-support) for more details. +The lifecycle of a Kubernetes resource can be separated into two phases depending on whether the resource has already been marked for deletion or not. + +The framework out of the box supports this logic, it will always +call the `reconcile` method unless the custom resource is +[marked from deletion](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work). + +On the other hand, if the resource is marked from deletion and if the `Reconciler` implements the +[`Cleaner`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) interface, only the `cleanup` method is called. By implementing this interface +the framework will automatically handle (add/remove) the finalizers for you. + +In short, if you need to provide explicit cleanup logic, you always want to use finalizers; for a more detailed explanation, see [Finalizer support](#finalizer-support) for more details. ### Using `UpdateControl` and `DeleteControl` -These two classes are used to control the outcome or the desired behaviour after the reconciliation. +These two classes control the outcome or the desired behavior after the reconciliation. The [`UpdateControl`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/UpdateControl.java) can instruct the framework to update the status sub-resource of the resource @@ -75,13 +73,10 @@ without an update: } ``` -Note, though, that using `EventSources` should be preferred to rescheduling since the -reconciliation will then be triggered only when needed instead than on a timely basis. - -Those are the typical use cases of resource updates, however in some cases there it can happen that -the controller wants to update the resource itself (for example to add annotations) or not perform -any updates, which is also supported. +Note, though, that using `EventSources` is the preferred way of scheduling since the +reconciliation is triggered only when a resource is changed, not on a timely basis. +At the end of the reconciliation, you typically update the status sub-resources. It is also possible to update both the status and the resource with the `patchResourceAndStatus` method. In this case, the resource is updated first followed by the status, using two separate requests to the Kubernetes API. @@ -141,32 +136,30 @@ Kubernetes cluster (e.g. external resources), you might not need to use finalize use the Kubernetes [garbage collection](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#owners-dependents) mechanism as much as possible by setting owner references for your secondary resources so that -the cluster can automatically deleted them for you whenever the associated primary resource is +the cluster can automatically delete them for you whenever the associated primary resource is deleted. Note that setting owner references is the responsibility of the `Reconciler` implementation, though [dependent resources](https://javaoperatorsdk.io/docs/dependent-resources) make that process easier. -If you do need to clean such state, you need to use finalizers so that their +If you do need to clean such a state, you need to use finalizers so that their presence will prevent the Kubernetes server from deleting the resource before your operator is -ready to allow it. This allows for clean up to still occur even if your operator was down when -the resources was "deleted" by a user. +ready to allow it. This allows for clean-up even if your operator was down when the resource was marked for deletion. JOSDK makes cleaning resources in this fashion easier by taking care of managing finalizers automatically for you when needed. The only thing you need to do is let the SDK know that your -operator is interested in cleaning state associated with your primary resources by having it +operator is interested in cleaning the state associated with your primary resources by having it implement the [`Cleaner

`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Cleaner.java) interface. If your `Reconciler` doesn't implement the `Cleaner` interface, the SDK will consider -that you don't need to perform any clean-up when resources are deleted and will therefore not -activate finalizer support. In other words, finalizer support is added only if your `Reconciler` -implements the `Cleaner` interface. +that you don't need to perform any clean-up when resources are deleted and will, therefore, not activate finalizer support. +In other words, finalizer support is added only if your `Reconciler` implements the `Cleaner` interface. -Finalizers are automatically added by the framework as the first step, thus after a resource -is created, but before the first reconciliation. The finalizer is added via a separate +The framework automatically adds finalizers as the first step, thus after a resource +is created but before the first reconciliation. The finalizer is added via a separate Kubernetes API call. As a result of this update, the finalizer will then be present on the resource. The reconciliation can then proceed as normal. -The finalizer that is automatically added will be also removed after the `cleanup` is executed on +The automatically added finalizer will also be removed after the `cleanup` is executed on the reconciler. This behavior is customizable as explained [above](#using-updatecontrol-and-deletecontrol) when we addressed the use of `DeleteControl`. @@ -175,4 +168,4 @@ You can specify the name of the finalizer to use for your `Reconciler` using the [`@ControllerConfiguration`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java) annotation. If you do not specify a finalizer name, one will be automatically generated for you. -From v5 by default finalizer is added using Served Side Apply. See also UpdateControl in docs. \ No newline at end of file +From v5, by default, the finalizer is added using Server Side Apply. See also `UpdateControl` in docs. From 3fe9cc712ade54f0a2275a43e925dbd3a00903df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 28 Mar 2025 09:13:30 +0100 Subject: [PATCH 10/45] docs: social access facelift (#2741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- docs/content/en/_index.md | 2 +- docs/hugo.toml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/content/en/_index.md b/docs/content/en/_index.md index f2124a21a2..f375ebfb97 100644 --- a/docs/content/en/_index.md +++ b/docs/content/en/_index.md @@ -33,7 +33,7 @@ We do a [Pull Request](https://github.com/operator-framework/java-operator-sdk/p {{% /blocks/feature %}} -{{% blocks/feature icon="fab fa-twitter" title="Follow us on Twitter!" url="https://twitter.com/javaoperatorsdk" %}} +{{% blocks/feature icon="fa-brands fa-bluesky" title="Follow us on BlueSky!" url="https://bsky.app/profile/javaoperatorsdk.bsky.social" %}} For announcement of latest features etc. {{% /blocks/feature %}} diff --git a/docs/hugo.toml b/docs/hugo.toml index b4535c08af..435cad2451 100644 --- a/docs/hugo.toml +++ b/docs/hugo.toml @@ -160,20 +160,20 @@ enable = false [params.links] [[params.links.user]] - name ="Twitter" - url = "https://twitter.com/javaoperatorsdk" - icon = "fab fa-twitter" - desc = "Follow us on Twitter to get the latest news!" + name ="BlueSky" + url = "https://bsky.app/profile/javaoperatorsdk.bsky.social" + icon = "fa-brands fa-bluesky" + desc = "Follow us on BlueSky to get the latest news!" [[params.links.user]] name = "Slack" url = "https://kubernetes.slack.com/archives/CAW0GV7A5" icon = "fab fa-slack" -desc = "Chat with other project developers" +desc = "Chat with other project developers on Kubernetes slack" [[params.links.user]] name = "Discord" url = "https://discord.gg/DacEhAy" icon = "fab fa-discord" -desc = "Chat with others on Discord" +desc = "Chat with others on our dedicated Discord server" #[[params.links.user]] # name = "Stack Overflow" # url = "https://example.org/stack" From 3469e12b05f8cf2f06f776707b26e1486b276ec7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 07:34:36 +0200 Subject: [PATCH 11/45] chore(deps): bump org.apache.maven.plugins:maven-surefire-plugin (#2743) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdcf0f87d8..7581178488 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ 2.11 3.14.0 - 3.5.2 + 3.5.3 3.11.2 3.3.1 3.3.1 From 6f04b5a559d08755ec96c0a14b5c57a107117812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 31 Mar 2025 16:19:23 +0200 Subject: [PATCH 12/45] docs: explanation for max maxReconciliationInterval (#2745) --- .../operator/api/reconciler/ControllerConfiguration.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java index 29bf0b670f..d407ed0fc6 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java @@ -44,7 +44,9 @@ /** * Optional configuration of the maximal interval the SDK will wait for a reconciliation request - * to happen before one will be automatically triggered. + * to happen before one will be automatically triggered. The intention behind this feature is to + * have a failsafe, not to artificially force repeated reconciliations. For that use {@link + * UpdateControl#rescheduleAfter(long)}. * * @return the maximal reconciliation interval configuration */ From 7725ff4e0ffed2e5ba901c7fe255fb4bbd107663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 31 Mar 2025 16:19:50 +0200 Subject: [PATCH 13/45] docs: retry doc fixes (#2744) --- .../documentation/error-handling-retries.md | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/content/en/docs/documentation/error-handling-retries.md b/docs/content/en/docs/documentation/error-handling-retries.md index f37c10318b..a36c46f08e 100644 --- a/docs/content/en/docs/documentation/error-handling-retries.md +++ b/docs/content/en/docs/documentation/error-handling-retries.md @@ -6,7 +6,7 @@ weight: 46 ## Automatic Retries on Error JOSDK will schedule an automatic retry of the reconciliation whenever an exception is thrown by -your `Reconciler`. The retry is behavior is configurable but a default implementation is provided +your `Reconciler`. The retry behavior is configurable, but a default implementation is provided covering most of the typical use-cases, see [GenericRetry](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/retry/GenericRetry.java) . @@ -22,7 +22,7 @@ You can also configure the default retry behavior using the `@GradualRetry` anno It is possible to provide a custom implementation using the `retry` field of the `@ControllerConfiguration` annotation and specifying the class of your custom implementation. -Note that this class will need to provide an accessible no-arg constructor for automated +Note that this class must provide an accessible no-arg constructor for automated instantiation. Additionally, your implementation can be automatically configured from an annotation that you can provide by having your `Retry` implementation implement the `AnnotationConfigurable` interface, parameterized with your annotation type. See the @@ -32,25 +32,28 @@ Information about the current retry state is accessible from the [Context](https://github.com/java-operator-sdk/java-operator-sdk/blob/master/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/Context.java) object. Of note, particularly interesting is the `isLastAttempt` method, which could allow your `Reconciler` to implement a different behavior based on this status, by setting an error message -in your resource' status, for example, when attempting a last retry. +in your resource status, for example, when attempting a last retry. Note, though, that reaching the retry limit won't prevent new events to be processed. New reconciliations will happen for new events as usual. However, if an error also occurs that -would normally trigger a retry, the SDK won't schedule one at this point since the retry limit -is already reached. +would trigger a retry, the SDK won't schedule one at this point since the retry limit +has already been reached. A successful execution resets the retry state. -### Setting Error Status After Last Retry Attempt +### Reconciler Error Handler -In order to facilitate error reporting, `Reconciler` can implement the -[ErrorStatusHandler](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ErrorStatusHandler.java) -interface: +In order to facilitate error reporting you can override [`updateErrorStatus`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Reconciler.java#L52) +method in `Reconciler`: ```java -public interface ErrorStatusHandler

{ +public class MyReconciler implements Reconciler { - ErrorStatusUpdateControl

updateErrorStatus(P resource, Context

context, Exception e); + @Override + public ErrorStatusUpdateControl updateErrorStatus( + WebPage resource, Context context, Exception e) { + return handleError(resource, e); + } } ``` @@ -58,15 +61,15 @@ public interface ErrorStatusHandler

{ The `updateErrorStatus` method is called in case an exception is thrown from the `Reconciler`. It is also called even if no retry policy is configured, just after the reconciler execution. `RetryInfo.getAttemptCount()` is zero after the first reconciliation attempt, since it is not a -result of a retry (regardless of whether a retry policy is configured or not). +result of a retry (regardless of whether a retry policy is configured). -`ErrorStatusUpdateControl` is used to tell the SDK what to do and how to perform the status -update on the primary resource, always performed as a status sub-resource request. Note that -this update request will also produce an event, and will result in a reconciliation if the -controller is not generation aware. +`ErrorStatusUpdateControl` tells the SDK what to do and how to perform the status +update on the primary resource, which is always performed as a status sub-resource request. Note that +this update request will also produce an event and result in a reconciliation if the +controller is not generation-aware. This feature is only available for the `reconcile` method of the `Reconciler` interface, since -there should not be updates to resource that have been marked for deletion. +there should not be updates to resources that have been marked for deletion. Retry can be skipped in cases of unrecoverable errors: @@ -76,40 +79,37 @@ Retry can be skipped in cases of unrecoverable errors: ### Correctness and Automatic Retries -While it is possible to deactivate automatic retries, this is not desirable, unless for very -specific reasons. Errors naturally occur, whether it be transient network errors or conflicts -when a given resource is handled by a `Reconciler` but is modified at the same time by a user in -a different process. Automatic retries handle these cases nicely and will usually result in a +While it is possible to deactivate automatic retries, this is not desirable unless there is a particular reason. +Errors naturally occur, whether it be transient network errors or conflicts +when a given resource is handled by a `Reconciler` but modified simultaneously by a user in +a different process. Automatic retries handle these cases nicely and will eventually result in a successful reconciliation. -## Retry and Rescheduling and Event Handling Common Behavior +## Retry, Rescheduling and Event Handling Common Behavior -Retry, reschedule and standard event processing form a relatively complex system, each of these +Retry, reschedule, and standard event processing form a relatively complex system, each of these functionalities interacting with the others. In the following, we describe the interplay of these features: -1. A successful execution resets a retry and the rescheduled executions which were present before - the reconciliation. However, a new rescheduling can be instructed from the reconciliation - outcome (`UpdateControl` or `DeleteControl`). +1. A successful execution resets a retry and the rescheduled executions that were present before + the reconciliation. However, the reconciliation outcome can instruct a new rescheduling (`UpdateControl` or `DeleteControl`). - For example, if a reconciliation had previously been re-scheduled after some amount of time, but an event triggered - the reconciliation (or cleanup) in the mean time, the scheduled execution would be automatically cancelled, i.e. - re-scheduling a reconciliation does not guarantee that one will occur exactly at that time, it simply guarantees that - one reconciliation will occur at that time at the latest, triggering one if no event from the cluster triggered one. - Of course, it's always possible to re-schedule a new reconciliation at the end of that "automatic" reconciliation. + For example, if a reconciliation had previously been rescheduled for after some amount of time, but an event triggered + the reconciliation (or cleanup) in the meantime, the scheduled execution would be automatically cancelled, i.e. + rescheduling a reconciliation does not guarantee that one will occur precisely at that time; it simply guarantees that a reconciliation will occur at the latest. + Of course, it's always possible to reschedule a new reconciliation at the end of that "automatic" reconciliation. - Similarly, if a retry was scheduled, any event from the cluster triggering a successful execution in the mean time + Similarly, if a retry was scheduled, any event from the cluster triggering a successful execution in the meantime would cancel the scheduled retry (because there's now no point in retrying something that already succeeded) -2. In case an exception happened, a retry is initiated. However, if an event is received +2. In case an exception is thrown, a retry is initiated. However, if an event is received meanwhile, it will be reconciled instantly, and this execution won't count as a retry attempt. 3. If the retry limit is reached (so no more automatic retry would happen), but a new event received, the reconciliation will still happen, but won't reset the retry, and will still be - marked as the last attempt in the retry info. The point (1) still holds, but in case of an - error, no retry will happen. - -The thing to keep in mind when it comes to retrying or rescheduling is that JOSDK tries to avoid unnecessary work. When -you reschedule an operation, you instruct JOSDK to perform that operation at the latest by the end of the rescheduling -delay. If something occurred on the cluster that triggers that particular operation (reconciliation or cleanup), then + marked as the last attempt in the retry info. The point (1) still holds - thus successful reconciliation will reset the retry - but no retry will happen in case of an error. + +The thing to remember when it comes to retrying or rescheduling is that JOSDK tries to avoid unnecessary work. When +you reschedule an operation, you instruct JOSDK to perform that operation by the end of the rescheduling +delay at the latest. If something occurred on the cluster that triggers that particular operation (reconciliation or cleanup), then JOSDK considers that there's no point in attempting that operation again at the end of the specified delay since there -is now no point to do so anymore. The same idea also applies to retries. \ No newline at end of file +is no point in doing so anymore. The same idea also applies to retries. From 29cd363178eeaabc659df12a3028724c0a2a4bc8 Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Thu, 3 Apr 2025 12:54:32 -0400 Subject: [PATCH 14/45] fix: refine retryAwareErrorLogging (#2748) closes: #2747 Signed-off-by: Steve Hawkins --- .../processing/event/EventProcessor.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index 566996af0e..f9af175053 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -349,21 +349,29 @@ private void retryAwareErrorLogging( boolean eventPresent, Exception exception, ExecutionScope

executionScope) { - if (!eventPresent - && !retry.isLastAttempt() - && exception instanceof KubernetesClientException ex) { - if (ex.getCode() == HttpURLConnection.HTTP_CONFLICT) { - log.debug( - "Full client conflict error during event processing {}", executionScope, exception); - log.warn( - "Resource Kubernetes Resource Creator/Update Conflict during reconciliation. Message:" - + " {} Resource name: {}", - ex.getMessage(), - ex.getFullResourceName()); - return; - } + if (!retry.isLastAttempt() + && exception instanceof KubernetesClientException ex + && ex.getCode() == HttpURLConnection.HTTP_CONFLICT) { + log.debug("Full client conflict error during event processing {}", executionScope, exception); + log.info( + "Resource Kubernetes Resource Creator/Update Conflict during reconciliation. Message:" + + " {} Resource name: {}", + ex.getMessage(), + ex.getFullResourceName()); + } else if (eventPresent || !retry.isLastAttempt()) { + log.warn( + "Uncaught error during event processing {} - but another reconciliation will be attempted" + + " because a superceding event has been recieved or another retry attempt is" + + " pending.", + executionScope, + exception); + } else { + log.error( + "Uncaught error during event processing {} - no superceding event is present and this is" + + " the retry last attempt", + executionScope, + exception); } - log.error("Error during event processing {}", executionScope, exception); } private void cleanupOnSuccessfulExecution(ExecutionScope

executionScope) { From 9482c3eebede5390dcddb757d35724cf303ff22e Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Fri, 4 Apr 2025 10:37:16 -0400 Subject: [PATCH 15/45] fix: catch exceptions from updating the status (#2752) closes: #2751 Signed-off-by: Steve Hawkins --- .../event/ReconciliationDispatcher.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java index ee861982b1..8fb9a30ae9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java @@ -212,9 +212,18 @@ public boolean isLastAttempt() { P updatedResource = null; if (errorStatusUpdateControl.getResource().isPresent()) { - updatedResource = - customResourceFacade.patchStatus( - errorStatusUpdateControl.getResource().orElseThrow(), originalResource); + try { + updatedResource = + customResourceFacade.patchStatus( + errorStatusUpdateControl.getResource().orElseThrow(), originalResource); + } catch (Exception ex) { + log.error( + "updateErrorStatus failed for resource: {} with version: {} for error {}", + getUID(resource), + getVersion(resource), + e.getMessage(), + ex); + } } if (errorStatusUpdateControl.isNoRetry()) { PostExecutionControl

postExecutionControl; From 6e26778965eeeb45eafe375db067653639a39623 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 07:51:43 +0200 Subject: [PATCH 16/45] chore(deps): bump org.mockito:mockito-core from 5.16.1 to 5.17.0 (#2756) Bumps [org.mockito:mockito-core](https://github.com/mockito/mockito) from 5.16.1 to 5.17.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v5.16.1...v5.17.0) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-version: 5.17.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7581178488..0a1c44d9dd 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 7.1.0 2.0.12 2.24.3 - 5.16.1 + 5.17.0 3.17.0 0.21.0 1.13.0 From 970bb00d9b2e183b10d744ce573edf9065854c7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 09:06:09 +0200 Subject: [PATCH 17/45] chore(deps): bump com.diffplug.spotless:spotless-maven-plugin (#2758) Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 2.44.3 to 2.44.4. - [Release notes](https://github.com/diffplug/spotless/releases) - [Changelog](https://github.com/diffplug/spotless/blob/main/CHANGES.md) - [Commits](https://github.com/diffplug/spotless/compare/maven/2.44.3...maven/2.44.4) --- updated-dependencies: - dependency-name: com.diffplug.spotless:spotless-maven-plugin dependency-version: 2.44.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a1c44d9dd..23ed81b2cb 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,7 @@ 3.1.4 9.0.1 3.4.5 - 2.44.3 + 2.44.4 From 82468beaa92ecb13ccdae1aaaf2116a0c3d0d0fc Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Tue, 8 Apr 2025 09:53:45 +0200 Subject: [PATCH 18/45] fix: typos (#2755) Signed-off-by: Chris Laprun --- .../operator/processing/event/EventProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index f9af175053..d67ae9cc09 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -361,13 +361,13 @@ private void retryAwareErrorLogging( } else if (eventPresent || !retry.isLastAttempt()) { log.warn( "Uncaught error during event processing {} - but another reconciliation will be attempted" - + " because a superceding event has been recieved or another retry attempt is" + + " because a superseding event has been received or another retry attempt is" + " pending.", executionScope, exception); } else { log.error( - "Uncaught error during event processing {} - no superceding event is present and this is" + "Uncaught error during event processing {} - no superseding event is present and this is" + " the retry last attempt", executionScope, exception); From f6f0183723d5aecf816e918c382f3f02b00ba46e Mon Sep 17 00:00:00 2001 From: Mattis Bratland Date: Wed, 9 Apr 2025 16:24:29 +0200 Subject: [PATCH 19/45] feat: improve customizability of SSABasedGenericKubernetesResourceMatcher (#2757) Signed-off-by: Mattis Bratland --- ...BasedGenericKubernetesResourceMatcher.java | 24 +++++++++++++- ...dGenericKubernetesResourceMatcherTest.java | 31 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java index 1868c2872d..eed766fc95 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java @@ -110,7 +110,7 @@ public boolean matches(R actual, R desired, Context context) { removeIrrelevantValues(desiredMap); - var matches = prunedActual.equals(desiredMap); + var matches = matches(prunedActual, desiredMap, actual, desired, context); if (!matches && log.isDebugEnabled() && LoggingUtils.isNotSensitiveResource(desired)) { var diff = getDiff(prunedActual, desiredMap, objectMapper); log.debug( @@ -125,6 +125,28 @@ public boolean matches(R actual, R desired, Context context) { return matches; } + /** + * Compares the desired and actual resources for equality. + * + *

This method can be overridden to implement custom matching logic. The {@code actualMap} is a + * cleaned-up version of the actual resource with managed fields and irrelevant values removed. + * + * @param actualMap the actual resource represented as a map + * @param desiredMap the desired resource represented as a map + * @param actual the actual resource object + * @param desired the desired resource object + * @param context the current matching context + * @return {@code true} if the resources are equal, otherwise {@code false} + */ + protected boolean matches( + Map actualMap, + Map desiredMap, + R actual, + R desired, + Context context) { + return actualMap.equals(desiredMap); + } + private String getDiff( Map prunedActualMap, Map desiredMap, diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java index 69bdf59aff..176531344c 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java @@ -236,6 +236,37 @@ void testSanitizeState_daemonSetWithResources_withMismatch() { assertThat(matcher.matches(actualDaemonSet, desiredDaemonSet, mockedContext)).isFalse(); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCustomMatcher_returnsExpectedMatchBasedOnReadOnlyLabel(boolean readOnly) { + var desiredConfigMap = + loadResource("configmap.empty-owner-reference-desired.yaml", ConfigMap.class); + desiredConfigMap.getData().put("key1", "another value"); + var actualConfigMap = loadResource("configmap.empty-owner-reference.yaml", ConfigMap.class); + actualConfigMap.getMetadata().getLabels().put("readonly", Boolean.toString(readOnly)); + + var matcher = new ReadOnlyAwareMatcher(); + assertThat(matcher.matches(actualConfigMap, desiredConfigMap, mockedContext)) + .isEqualTo(readOnly); + } + + private static class ReadOnlyAwareMatcher + extends SSABasedGenericKubernetesResourceMatcher { + @Override + protected boolean matches( + Map actualMap, + Map desiredMap, + T actual, + T desired, + Context context) { + var readonly = actual.getMetadata().getLabels().get("readonly"); + if (readonly != null && readonly.equals("true")) { + return true; + } + return actualMap.equals(desiredMap); + } + } + private static R loadResource(String fileName, Class clazz) { return ReconcilerUtils.loadYaml( clazz, SSABasedGenericKubernetesResourceMatcherTest.class, fileName); From 394409e9bfb8ca08095ec6dd75cb559027076df5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 08:15:18 +0200 Subject: [PATCH 20/45] chore(deps): bump org.junit:junit-bom from 5.12.1 to 5.12.2 (#2769) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 23ed81b2cb..2e0337cf13 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ https://sonarcloud.io jdk - 5.12.1 + 5.12.2 7.1.0 2.0.12 2.24.3 From 6c738fd5af94d9bae0dc00d866ef9c10c068847b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 08:16:43 +0200 Subject: [PATCH 21/45] chore(deps): bump io.micrometer:micrometer-core from 1.14.5 to 1.14.6 (#2767) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e0337cf13..7ee6e8302f 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 3.27.3 4.3.0 2.7.3 - 1.14.5 + 1.14.6 3.2.0 0.9.14 2.18.0 From 0459fa4b508c9ff2a255603a4bc9825d677e65de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 13:42:35 +0200 Subject: [PATCH 22/45] chore(deps): bump commons-io:commons-io from 2.18.0 to 2.19.0 (#2768) Bumps commons-io:commons-io from 2.18.0 to 2.19.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-version: 2.19.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bootstrapper-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrapper-maven-plugin/pom.xml b/bootstrapper-maven-plugin/pom.xml index 1ab08b975a..44b50a3d81 100644 --- a/bootstrapper-maven-plugin/pom.xml +++ b/bootstrapper-maven-plugin/pom.xml @@ -58,7 +58,7 @@ commons-io commons-io - 2.18.0 + 2.19.0 com.github.spullara.mustache.java diff --git a/pom.xml b/pom.xml index 7ee6e8302f..f7be33133f 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ 1.14.6 3.2.0 0.9.14 - 2.18.0 + 2.19.0 4.15 2.11 From c3ed322e888bd4f63fa8fef2e3bd4bb3d7e4f485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 14 Apr 2025 14:50:00 +0200 Subject: [PATCH 23/45] fix: exclude test CRDs from operator-framework (#2771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- operator-framework/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/operator-framework/pom.xml b/operator-framework/pom.xml index cb49b0d39b..d72f91d293 100644 --- a/operator-framework/pom.xml +++ b/operator-framework/pom.xml @@ -125,6 +125,15 @@ + + org.apache.maven.plugins + maven-jar-plugin + + + META-INF/fabric8/*.yml + + + org.apache.maven.plugins maven-surefire-plugin From 42256459132e3ed60f865d8191efe8cc9e728202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 14 Apr 2025 17:47:10 +0200 Subject: [PATCH 24/45] fix: event marking bug (#2763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/processing/event/EventProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index d67ae9cc09..bdaf575814 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -196,7 +196,7 @@ private void handleEventMarking(Event event, ResourceState state) { // event as below. markEventReceived(state); } - } else if (!state.deleteEventPresent() || !state.processedMarkForDeletionPresent()) { + } else if (!state.deleteEventPresent() && !state.processedMarkForDeletionPresent()) { markEventReceived(state); } else if (log.isDebugEnabled()) { log.debug( From 2acb3f3ff594a9a1a1c5657ab1ff26585ac932c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 29 Apr 2025 17:27:19 +0200 Subject: [PATCH 25/45] feat: primary resource caching for followup reconciliation(s) (#2761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros Signed-off-by: Chris Laprun Co-authored-by: Martin Stefanko Co-authored-by: Antonio <122279781+afalhambra-hivemq@users.noreply.github.com> Co-authored-by: Chris Laprun Co-authored-by: Chris Laprun --- .../en/docs/documentation/reconciler.md | 114 +++++++++ .../PrimaryUpdateAndCacheUtils.java | 225 ++++++++++++++++++ .../support/PrimaryResourceCache.java | 65 +++++ .../processing/event/EventSourceManager.java | 1 + .../event/EventSourceRetriever.java | 3 + .../source/informer/InformerEventSource.java | 64 +++-- .../informer/TemporaryResourceCache.java | 8 +- .../support/PrimaryResourceCacheTest.java | 87 +++++++ ...=> TemporaryPrimaryResourceCacheTest.java} | 2 +- .../PeriodicTriggerEventSource.java | 52 ++++ .../StatusPatchCacheCustomResource.java | 13 + .../internal/StatusPatchCacheIT.java | 48 ++++ .../internal/StatusPatchCacheReconciler.java | 64 +++++ .../internal/StatusPatchCacheSpec.java | 14 ++ .../internal/StatusPatchCacheStatus.java | 15 ++ ...StatusPatchPrimaryCacheCustomResource.java | 14 ++ .../StatusPatchPrimaryCacheIT.java | 48 ++++ .../StatusPatchPrimaryCacheReconciler.java | 89 +++++++ .../StatusPatchPrimaryCacheSpec.java | 15 ++ .../StatusPatchPrimaryCacheStatus.java | 15 ++ 20 files changed, 917 insertions(+), 39 deletions(-) create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java rename operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/{TemporaryResourceCacheTest.java => TemporaryPrimaryResourceCacheTest.java} (99%) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/PeriodicTriggerEventSource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java diff --git a/docs/content/en/docs/documentation/reconciler.md b/docs/content/en/docs/documentation/reconciler.md index 26a2a20d61..b9ede8aa95 100644 --- a/docs/content/en/docs/documentation/reconciler.md +++ b/docs/content/en/docs/documentation/reconciler.md @@ -169,3 +169,117 @@ You can specify the name of the finalizer to use for your `Reconciler` using the annotation. If you do not specify a finalizer name, one will be automatically generated for you. From v5, by default, the finalizer is added using Server Side Apply. See also `UpdateControl` in docs. + +### Making sure the primary resource is up to date for the next reconciliation + +It is typical to want to update the status subresource with the information that is available during the reconciliation. +This is sometimes referred to as the last observed state. When the primary resource is updated, though, the framework +does not cache the resource directly, relying instead on the propagation of the update to the underlying informer's +cache. It can, therefore, happen that, if other events trigger other reconciliations before the informer cache gets +updated, your reconciler does not see the latest version of the primary resource. While this might not typically be a +problem in most cases, as caches eventually become consistent, depending on your reconciliation logic, you might still +require the latest status version possible, for example if the status subresource is used as a communication mechanism, +see [Representing Allocated Values](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#representing-allocated-values) +from the Kubernetes docs for more details. + +The framework provides utilities to help with these use cases with +[`PrimaryUpdateAndCacheUtils`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java). +These utility methods come in two flavors: + +#### Using internal cache + +In almost all cases for this purpose, you can use internal caches: + +```java + @Override +public UpdateControl reconcile( + StatusPatchCacheCustomResource resource, Context context) { + + // omitted logic + + // update with SSA requires a fresh copy + var freshCopy = createFreshCopy(primary); + freshCopy.getStatus().setValue(statusWithState()); + + var updatedResource = PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(resource, freshCopy, context); + + return UpdateControl.noUpdate(); + } +``` + +In the background `PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus` puts the result of the update into an internal +cache and will make sure that the next reconciliation will contain the most recent version of the resource. Note that it +is not necessarily the version of the resource you got as response from the update, it can be newer since other parties +can do additional updates meanwhile, but if not explicitly modified, it will contain the up-to-date status. + +See related integration test [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal). + +This approach works with the default configuration of the framework and should be good to go in most of the cases. +Without going further into the details, this won't work if `ConfigurationService.parseResourceVersionsForEventFilteringAndCaching` +is set to `false` (more precisely there are some edge cases when it won't work). For that case framework provides the following solution: + +#### Fallback approach: using `PrimaryResourceCache` cache + +As an alternative, for very rare cases when `ConfigurationService.parseResourceVersionsForEventFilteringAndCaching` +needs to be set to `false` you can use an explicit caching approach: + +```java + +// We on purpose don't use the provided predicate to show what a custom one could look like. + private final PrimaryResourceCache cache = + new PrimaryResourceCache<>( + (statusPatchCacheCustomResourcePair, statusPatchCacheCustomResource) -> + statusPatchCacheCustomResource.getStatus().getValue() + >= statusPatchCacheCustomResourcePair.afterUpdate().getStatus().getValue()); + + @Override + public UpdateControl reconcile( + StatusPatchPrimaryCacheCustomResource primary, + Context context) { + + // cache will compare the current and the cached resource and return the more recent. (And evict the old) + primary = cache.getFreshResource(primary); + + // omitted logic + + var freshCopy = createFreshCopy(primary); + + freshCopy.getStatus().setValue(statusWithState()); + + var updated = + PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(primary, freshCopy, context, cache); + + return UpdateControl.noUpdate(); + } + + @Override + public DeleteControl cleanup( + StatusPatchPrimaryCacheCustomResource resource, + Context context) + throws Exception { + // cleanup the cache on resource deletion + cache.cleanup(resource); + return DeleteControl.defaultDelete(); + } + +``` + +[`PrimaryResourceCache`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java) +is designed for this purpose. As shown in the example above, it is up to you to provide a predicate to determine if the +resource is more recent than the one available. In other words, when to evict the resource from the cache. Typically, as +shown in +the [integration test](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache) +you can have a counter in status to check on that. + +Since all of this happens explicitly, you cannot use this approach for managed dependent resources and workflows and +will need to use the unmanaged approach instead. This is due to the fact that managed dependent resources always get +their associated primary resource from the underlying informer event source cache. + +#### Additional remarks + +As shown in the integration tests, there is no optimistic locking used when updating the +[resource](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java#L41) +(in other words `metadata.resourceVersion` is set to `null`). This is desired since you don't want the patch to fail on +update. + +In addition, you can configure the [Fabric8 client retry](https://github.com/fabric8io/kubernetes-client?tab=readme-ov-file#configuring-the-client). diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java new file mode 100644 index 0000000000..174f7667f6 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java @@ -0,0 +1,225 @@ +package io.javaoperatorsdk.operator.api.reconciler; + +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.dsl.base.PatchContext; +import io.fabric8.kubernetes.client.dsl.base.PatchType; +import io.javaoperatorsdk.operator.api.reconciler.support.PrimaryResourceCache; +import io.javaoperatorsdk.operator.processing.event.ResourceID; + +/** + * Utility methods to patch the primary resource state and store it to the related cache, to make + * sure that fresh resource is present for the next reconciliation. The main use case for such + * updates is to store state is resource status. Use of optimistic locking is not desired for such + * updates, since we don't want to patch fail and lose information that we want to store. + */ +public class PrimaryUpdateAndCacheUtils { + + private PrimaryUpdateAndCacheUtils() {} + + private static final Logger log = LoggerFactory.getLogger(PrimaryUpdateAndCacheUtils.class); + + /** + * Makes sure that the up-to-date primary resource will be present during the next reconciliation. + * Using update (PUT) method. + * + * @param primary resource + * @param context of reconciliation + * @return updated resource + * @param

primary resource type + */ + public static

P updateAndCacheStatus(P primary, Context

context) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, context, () -> context.getClient().resource(primary).updateStatus()); + } + + /** + * Makes sure that the up-to-date primary resource will be present during the next reconciliation. + * Using JSON Merge patch. + * + * @param primary resource + * @param context of reconciliation + * @return updated resource + * @param

primary resource type + */ + public static

P patchAndCacheStatus(P primary, Context

context) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, context, () -> context.getClient().resource(primary).patchStatus()); + } + + /** + * Makes sure that the up-to-date primary resource will be present during the next reconciliation. + * Using JSON Patch. + * + * @param primary resource + * @param context of reconciliation + * @return updated resource + * @param

primary resource type + */ + public static

P editAndCacheStatus( + P primary, Context

context, UnaryOperator

operation) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, context, () -> context.getClient().resource(primary).editStatus(operation)); + } + + /** + * Makes sure that the up-to-date primary resource will be present during the next reconciliation. + * + * @param primary resource + * @param context of reconciliation + * @param patch free implementation of cache + * @return the updated resource. + * @param

primary resource type + */ + public static

P patchAndCacheStatus( + P primary, Context

context, Supplier

patch) { + var updatedResource = patch.get(); + context + .eventSourceRetriever() + .getControllerEventSource() + .handleRecentResourceUpdate(ResourceID.fromResource(primary), updatedResource, primary); + return updatedResource; + } + + /** + * Makes sure that the up-to-date primary resource will be present during the next reconciliation. + * Using Server Side Apply. + * + * @param primary resource + * @param freshResourceWithStatus - fresh resource with target state + * @param context of reconciliation + * @return the updated resource. + * @param

primary resource type + */ + public static

P ssaPatchAndCacheStatus( + P primary, P freshResourceWithStatus, Context

context) { + logWarnIfResourceVersionPresent(freshResourceWithStatus); + var res = + context + .getClient() + .resource(freshResourceWithStatus) + .subresource("status") + .patch( + new PatchContext.Builder() + .withForce(true) + .withFieldManager(context.getControllerConfiguration().fieldManager()) + .withPatchType(PatchType.SERVER_SIDE_APPLY) + .build()); + + context + .eventSourceRetriever() + .getControllerEventSource() + .handleRecentResourceUpdate(ResourceID.fromResource(primary), res, primary); + return res; + } + + /** + * Patches the resource and adds it to the {@link PrimaryResourceCache}. + * + * @param primary resource + * @param freshResourceWithStatus - fresh resource with target state + * @param context of reconciliation + * @param cache - resource cache managed by user + * @return the updated resource. + * @param

primary resource type + */ + public static

P ssaPatchAndCacheStatus( + P primary, P freshResourceWithStatus, Context

context, PrimaryResourceCache

cache) { + logWarnIfResourceVersionPresent(freshResourceWithStatus); + return patchAndCacheStatus( + primary, + cache, + () -> + context + .getClient() + .resource(freshResourceWithStatus) + .subresource("status") + .patch( + new PatchContext.Builder() + .withForce(true) + .withFieldManager(context.getControllerConfiguration().fieldManager()) + .withPatchType(PatchType.SERVER_SIDE_APPLY) + .build())); + } + + /** + * Patches the resource with JSON Patch and adds it to the {@link PrimaryResourceCache}. + * + * @param primary resource + * @param context of reconciliation + * @param cache - resource cache managed by user + * @return the updated resource. + * @param

primary resource type + */ + public static

P editAndCacheStatus( + P primary, Context

context, PrimaryResourceCache

cache, UnaryOperator

operation) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, cache, () -> context.getClient().resource(primary).editStatus(operation)); + } + + /** + * Patches the resource with JSON Merge patch and adds it to the {@link PrimaryResourceCache} + * provided. + * + * @param primary resource + * @param context of reconciliation + * @param cache - resource cache managed by user + * @return the updated resource. + * @param

primary resource type + */ + public static

P patchAndCacheStatus( + P primary, Context

context, PrimaryResourceCache

cache) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, cache, () -> context.getClient().resource(primary).patchStatus()); + } + + /** + * Updates the resource and adds it to the {@link PrimaryResourceCache}. + * + * @param primary resource + * @param context of reconciliation + * @param cache - resource cache managed by user + * @return the updated resource. + * @param

primary resource type + */ + public static

P updateAndCacheStatus( + P primary, Context

context, PrimaryResourceCache

cache) { + logWarnIfResourceVersionPresent(primary); + return patchAndCacheStatus( + primary, cache, () -> context.getClient().resource(primary).updateStatus()); + } + + /** + * Updates the resource using the user provided implementation anc caches the result. + * + * @param primary resource + * @param cache resource cache managed by user + * @param patch implementation of resource update* + * @return the updated resource. + * @param

primary resource type + */ + public static

P patchAndCacheStatus( + P primary, PrimaryResourceCache

cache, Supplier

patch) { + var updatedResource = patch.get(); + cache.cacheResource(primary, updatedResource); + return updatedResource; + } + + private static

void logWarnIfResourceVersionPresent(P primary) { + if (primary.getMetadata().getResourceVersion() != null) { + log.warn( + "The metadata.resourceVersion of primary resource is NOT null, " + + "using optimistic locking is discouraged for this purpose. "); + } + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java new file mode 100644 index 0000000000..4da73ab8b1 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java @@ -0,0 +1,65 @@ +package io.javaoperatorsdk.operator.api.reconciler.support; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiPredicate; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.processing.event.ResourceID; + +public class PrimaryResourceCache

{ + + private final BiPredicate, P> evictionPredicate; + private final ConcurrentHashMap> cache = new ConcurrentHashMap<>(); + + public PrimaryResourceCache(BiPredicate, P> evictionPredicate) { + this.evictionPredicate = evictionPredicate; + } + + public PrimaryResourceCache() { + this(new ResourceVersionParsingEvictionPredicate<>()); + } + + public void cacheResource(P afterUpdate) { + var resourceId = ResourceID.fromResource(afterUpdate); + cache.put(resourceId, new Pair<>(null, afterUpdate)); + } + + public void cacheResource(P beforeUpdate, P afterUpdate) { + var resourceId = ResourceID.fromResource(beforeUpdate); + cache.put(resourceId, new Pair<>(beforeUpdate, afterUpdate)); + } + + public P getFreshResource(P newVersion) { + var resourceId = ResourceID.fromResource(newVersion); + var pair = cache.get(resourceId); + if (pair == null) { + return newVersion; + } + if (!newVersion.getMetadata().getUid().equals(pair.afterUpdate().getMetadata().getUid())) { + cache.remove(resourceId); + return newVersion; + } + if (evictionPredicate.test(pair, newVersion)) { + cache.remove(resourceId); + return newVersion; + } else { + return pair.afterUpdate(); + } + } + + public void cleanup(P resource) { + cache.remove(ResourceID.fromResource(resource)); + } + + public record Pair(T beforeUpdate, T afterUpdate) {} + + /** This works in general, but it does not strictly follow the contract with k8s API */ + public static class ResourceVersionParsingEvictionPredicate + implements BiPredicate, T> { + @Override + public boolean test(Pair updatePair, T newVersion) { + return Long.parseLong(updatePair.afterUpdate().getMetadata().getResourceVersion()) + <= Long.parseLong(newVersion.getMetadata().getResourceVersion()); + } + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java index 02b91f6dd0..8b07bf110b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java @@ -208,6 +208,7 @@ public Stream> getEventSourcesStream() { return eventSources.flatMappedSources(); } + @Override public ControllerEventSource

getControllerEventSource() { return eventSources.controllerEventSource(); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceRetriever.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceRetriever.java index 066a7f5808..c5a219a026 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceRetriever.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceRetriever.java @@ -6,6 +6,7 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.source.EventSource; +import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource; public interface EventSourceRetriever

{ @@ -17,6 +18,8 @@ default EventSource getEventSourceFor(Class dependentType) { List> getEventSourcesFor(Class dependentType); + ControllerEventSource

getControllerEventSource(); + /** * Registers (and starts) the specified {@link EventSource} dynamically during the reconciliation. * If an EventSource is already registered with the specified name, the registration will be diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java index 688a88ae22..b52dc278f2 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java @@ -13,6 +13,7 @@ import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.informers.ResourceEventHandler; import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.EventHandler; @@ -20,50 +21,45 @@ import io.javaoperatorsdk.operator.processing.event.source.PrimaryToSecondaryMapper; /** - * Wraps informer(s) so it is connected to the eventing system of the framework. Note that since - * it's it is built on top of Informers, it also support caching resources using caching from - * fabric8 client Informer caches and additional caches described below. + * Wraps informer(s) so they are connected to the eventing system of the framework. Note that since + * this is built on top of Fabric8 client Informers, it also supports caching resources using + * caching from informer caches as well as additional caches described below. * *

InformerEventSource also supports two features to better handle events and caching of - * resources on top of Informers from fabric8 Kubernetes client. These two features implementation - * wise are related to each other:
+ * resources on top of Informers from the Fabric8 Kubernetes client. These two features are related + * to each other as follows: * - *

1. API that allows to make sure the cache contains the fresh resource after an update. This is - * important for {@link io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource} and - * mainly for {@link - * io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource} so after - * reconcile if getResource() called always return the fresh resource. To achieve this - * handleRecentResourceUpdate() and handleRecentResourceCreate() needs to be called explicitly after - * resource created/updated using the kubernetes client. (These calls are done automatically by - * KubernetesDependentResource implementation.). In the background this will store the new resource - * in a temporary cache {@link TemporaryResourceCache} which do additional checks. After a new event - * is received the cachec object is removed from this cache, since in general then it is already in - * the cache of informer.
+ *

    + *
  1. Ensuring the cache contains the fresh resource after an update. This is important for + * {@link io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource} and mainly + * for {@link + * io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource} so + * that {@link + * io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource#getSecondaryResource(HasMetadata, + * Context)} always returns the latest version of the resource after a reconciliation. To + * achieve this {@link #handleRecentResourceUpdate(ResourceID, HasMetadata, HasMetadata)} and + * {@link #handleRecentResourceCreate(ResourceID, HasMetadata)} need to be called explicitly + * after a resource is created or updated using the kubernetes client. These calls are done + * automatically by the KubernetesDependentResource implementation. In the background this + * will store the new resource in a temporary cache {@link TemporaryResourceCache} which does + * additional checks. After a new event is received the cached object is removed from this + * cache, since it is then usually already in the informer cache. + *
  2. Avoiding unneeded reconciliations after resources are created or updated. This filters out + * events that are the results of updates and creates made by the controller itself because we + * typically don't want the associated informer to trigger an event causing a useless + * reconciliation (as the change originates from the reconciler itself). For the details see + * {@link #canSkipEvent(HasMetadata, HasMetadata, ResourceID)} and related usage. + *
* - *

2. Additional API is provided that is meant to be used with the combination of the previous - * one, and the goal is to filter out events that are the results of updates and creates made by the - * controller itself. For example if in reconciler a ConfigMaps is created, there should be an - * Informer in place to handle change events of that ConfigMap, but since it has bean created (or - * updated) by the reconciler this should not trigger an additional reconciliation by default. In - * order to achieve this prepareForCreateOrUpdateEventFiltering(..) method needs to be called before - * the operation of the k8s client. And the operation from point 1. after the k8s client call. See - * it's usage in CreateUpdateEventFilterTestReconciler integration test for the usage. (Again this - * is managed for the developer if using dependent resources.)
- * Roughly it works in a way that before the K8S API call is made, we set mark the resource ID, and - * from that point informer won't propagate events further just will start record them. After the - * client operation is done, it's checked and analysed what events were received and based on that - * it will propagate event or not and/or put the new resource into the temporal cache - so if the - * event not arrived yet about the update will be able to filter it in the future. - * - * @param resource type watching - * @param

type of the primary resource + * @param resource type being watched + * @param

type of the associated primary resource */ public class InformerEventSource extends ManagedInformerEventSource> implements ResourceEventHandler { - private static final Logger log = LoggerFactory.getLogger(InformerEventSource.class); public static final String PREVIOUS_ANNOTATION_KEY = "javaoperatorsdk.io/previous"; + private static final Logger log = LoggerFactory.getLogger(InformerEventSource.class); // we need direct control for the indexer to propagate the just update resource also to the index private final PrimaryToSecondaryIndex primaryToSecondaryIndex; private final PrimaryToSecondaryMapper

primaryToSecondaryMapper; diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCache.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCache.java index 247cdb9aa5..9ec5b3694c 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCache.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCache.java @@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -167,9 +167,9 @@ public synchronized boolean isKnownResourceVersion(T resource) { } /** - * @return true if {@link InformerEventSourceConfiguration#parseResourceVersions()} is enabled and - * the resourceVersion of newResource is numerically greater than cachedResource, otherwise - * false + * @return true if {@link ConfigurationService#parseResourceVersionsForEventFilteringAndCaching()} + * is enabled and the resourceVersion of newResource is numerically greater than + * cachedResource, otherwise false */ private boolean isLaterResourceVersion(ResourceID resourceId, T newResource, T cachedResource) { try { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java new file mode 100644 index 0000000000..58e3ce8a0a --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java @@ -0,0 +1,87 @@ +package io.javaoperatorsdk.operator.api.reconciler.support; + +import org.junit.jupiter.api.Test; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.sample.simple.TestCustomResource; +import io.javaoperatorsdk.operator.sample.simple.TestCustomResourceSpec; + +import static org.assertj.core.api.Assertions.assertThat; + +class PrimaryResourceCacheTest { + + PrimaryResourceCache versionParsingCache = + new PrimaryResourceCache<>( + new PrimaryResourceCache.ResourceVersionParsingEvictionPredicate<>()); + + @Test + void returnsThePassedValueIfCacheIsEmpty() { + var cr = customResource("1"); + + var res = versionParsingCache.getFreshResource(cr); + + assertThat(cr).isSameAs(res); + } + + @Test + void returnsTheCachedIfNotEvictedAccordingToPredicate() { + var cr = customResource("2"); + + versionParsingCache.cacheResource(cr); + + var res = versionParsingCache.getFreshResource(customResource("1")); + assertThat(cr).isSameAs(res); + } + + @Test + void ifMoreFreshPassedCachedIsEvicted() { + var cr = customResource("2"); + versionParsingCache.cacheResource(cr); + var newCR = customResource("3"); + + var res = versionParsingCache.getFreshResource(newCR); + var resOnOlder = versionParsingCache.getFreshResource(cr); + + assertThat(newCR).isSameAs(res); + assertThat(resOnOlder).isSameAs(cr); + assertThat(newCR).isNotSameAs(cr); + } + + @Test + void cleanupRemovesCachedResources() { + var cr = customResource("2"); + versionParsingCache.cacheResource(cr); + + versionParsingCache.cleanup(customResource("3")); + + var olderCR = customResource("1"); + var res = versionParsingCache.getFreshResource(olderCR); + assertThat(olderCR).isSameAs(res); + } + + @Test + void removesIfNewResourceWithDifferentUid() { + var cr = customResource("2"); + versionParsingCache.cacheResource(cr); + var crWithDifferentUid = customResource("1"); + cr.getMetadata().setUid("otheruid"); + + var res = versionParsingCache.getFreshResource(crWithDifferentUid); + + assertThat(res).isSameAs(crWithDifferentUid); + } + + private TestCustomResource customResource(String resourceVersion) { + var cr = new TestCustomResource(); + cr.setMetadata( + new ObjectMetaBuilder() + .withName("test1") + .withNamespace("default") + .withUid("uid") + .withResourceVersion(resourceVersion) + .build()); + cr.setSpec(new TestCustomResourceSpec()); + cr.getSpec().setKey("key"); + return cr; + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCacheTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryPrimaryResourceCacheTest.java similarity index 99% rename from operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCacheTest.java rename to operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryPrimaryResourceCacheTest.java index d31408beb6..e62888832f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryResourceCacheTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/TemporaryPrimaryResourceCacheTest.java @@ -19,7 +19,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -class TemporaryResourceCacheTest { +class TemporaryPrimaryResourceCacheTest { public static final String RESOURCE_VERSION = "2"; diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/PeriodicTriggerEventSource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/PeriodicTriggerEventSource.java new file mode 100644 index 0000000000..366777409a --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/PeriodicTriggerEventSource.java @@ -0,0 +1,52 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache; + +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.OperatorException; +import io.javaoperatorsdk.operator.processing.event.Event; +import io.javaoperatorsdk.operator.processing.event.ResourceID; +import io.javaoperatorsdk.operator.processing.event.source.AbstractEventSource; +import io.javaoperatorsdk.operator.processing.event.source.IndexerResourceCache; + +public class PeriodicTriggerEventSource

+ extends AbstractEventSource { + + public static final int DEFAULT_PERIOD = 30; + private final Timer timer = new Timer(); + private final IndexerResourceCache

primaryCache; + private final int period; + + public PeriodicTriggerEventSource(IndexerResourceCache

primaryCache) { + this(primaryCache, DEFAULT_PERIOD); + } + + public PeriodicTriggerEventSource(IndexerResourceCache

primaryCache, int period) { + super(Void.class); + this.primaryCache = primaryCache; + this.period = period; + } + + @Override + public Set getSecondaryResources(P primary) { + return Set.of(); + } + + @Override + public void start() throws OperatorException { + super.start(); + timer.schedule( + new TimerTask() { + @Override + public void run() { + primaryCache + .list() + .forEach(r -> getEventHandler().handleEvent(new Event(ResourceID.fromResource(r)))); + } + }, + 0, + period); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java new file mode 100644 index 0000000000..2a2d8b83fd --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java @@ -0,0 +1,13 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.internal; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("spcl") +public class StatusPatchCacheCustomResource + extends CustomResource implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java new file mode 100644 index 0000000000..f78511f250 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java @@ -0,0 +1,48 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.internal; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +public class StatusPatchCacheIT { + + public static final String TEST_1 = "test1"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + .withReconciler(StatusPatchCacheReconciler.class) + .build(); + + @Test + void testStatusAlwaysUpToDate() { + var reconciler = extension.getReconcilerOfType(StatusPatchCacheReconciler.class); + + extension.create(testResource()); + + // the reconciliation is periodically triggered, the status values should be increasing + // monotonically + await() + .pollDelay(Duration.ofSeconds(1)) + .pollInterval(Duration.ofMillis(30)) + .untilAsserted( + () -> { + assertThat(reconciler.errorPresent).isFalse(); + assertThat(reconciler.latestValue).isGreaterThan(10); + }); + } + + StatusPatchCacheCustomResource testResource() { + var res = new StatusPatchCacheCustomResource(); + res.setMetadata(new ObjectMetaBuilder().withName(TEST_1).build()); + res.setSpec(new StatusPatchCacheSpec()); + return res; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java new file mode 100644 index 0000000000..8a3a72a901 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java @@ -0,0 +1,64 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.internal; + +import java.util.List; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; +import io.javaoperatorsdk.operator.api.reconciler.PrimaryUpdateAndCacheUtils; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.baseapi.statuscache.PeriodicTriggerEventSource; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; + +@ControllerConfiguration +public class StatusPatchCacheReconciler implements Reconciler { + + public volatile int latestValue = 0; + public volatile boolean errorPresent = false; + + @Override + public UpdateControl reconcile( + StatusPatchCacheCustomResource resource, Context context) { + + if (resource.getStatus() != null && resource.getStatus().getValue() != latestValue) { + errorPresent = true; + throw new IllegalStateException( + "status is not up to date. Latest value: " + + latestValue + + " status values: " + + resource.getStatus().getValue()); + } + + var freshCopy = createFreshCopy(resource); + + freshCopy + .getStatus() + .setValue(resource.getStatus() == null ? 1 : resource.getStatus().getValue() + 1); + + var updated = PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(resource, freshCopy, context); + latestValue = updated.getStatus().getValue(); + + return UpdateControl.noUpdate(); + } + + @Override + public List> prepareEventSources( + EventSourceContext context) { + // periodic event triggering for testing purposes + return List.of(new PeriodicTriggerEventSource<>(context.getPrimaryCache())); + } + + private StatusPatchCacheCustomResource createFreshCopy(StatusPatchCacheCustomResource resource) { + var res = new StatusPatchCacheCustomResource(); + res.setMetadata( + new ObjectMetaBuilder() + .withName(resource.getMetadata().getName()) + .withNamespace(resource.getMetadata().getNamespace()) + .build()); + res.setStatus(new StatusPatchCacheStatus()); + + return res; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java new file mode 100644 index 0000000000..d1426fd943 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java @@ -0,0 +1,14 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.internal; + +public class StatusPatchCacheSpec { + + private int counter = 0; + + public int getCounter() { + return counter; + } + + public void setCounter(int counter) { + this.counter = counter; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java new file mode 100644 index 0000000000..00bc4b6f04 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.internal; + +public class StatusPatchCacheStatus { + + private Integer value = 0; + + public Integer getValue() { + return value; + } + + public StatusPatchCacheStatus setValue(Integer value) { + this.value = value; + return this; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java new file mode 100644 index 0000000000..84b145cac3 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java @@ -0,0 +1,14 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("spc") +public class StatusPatchPrimaryCacheCustomResource + extends CustomResource + implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java new file mode 100644 index 0000000000..a884ec0758 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java @@ -0,0 +1,48 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class StatusPatchPrimaryCacheIT { + + public static final String TEST_1 = "test1"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + .withReconciler(StatusPatchPrimaryCacheReconciler.class) + .build(); + + @Test + void testStatusAlwaysUpToDate() { + var reconciler = extension.getReconcilerOfType(StatusPatchPrimaryCacheReconciler.class); + + extension.create(testResource()); + + // the reconciliation is periodically triggered, the status values should be increasing + // monotonically + await() + .pollDelay(Duration.ofSeconds(1)) + .pollInterval(Duration.ofMillis(30)) + .untilAsserted( + () -> { + assertThat(reconciler.errorPresent).isFalse(); + assertThat(reconciler.latestValue).isGreaterThan(10); + }); + } + + StatusPatchPrimaryCacheCustomResource testResource() { + var res = new StatusPatchPrimaryCacheCustomResource(); + res.setMetadata(new ObjectMetaBuilder().withName(TEST_1).build()); + res.setSpec(new StatusPatchPrimaryCacheSpec()); + return res; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java new file mode 100644 index 0000000000..c25fcddfec --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java @@ -0,0 +1,89 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; + +import java.util.List; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; +import io.javaoperatorsdk.operator.api.reconciler.PrimaryUpdateAndCacheUtils; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.api.reconciler.support.PrimaryResourceCache; +import io.javaoperatorsdk.operator.baseapi.statuscache.PeriodicTriggerEventSource; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; + +@ControllerConfiguration +public class StatusPatchPrimaryCacheReconciler + implements Reconciler, + Cleaner { + + public volatile int latestValue = 0; + public volatile boolean errorPresent = false; + + // We on purpose don't use the provided predicate to show what a custom one could look like. + private final PrimaryResourceCache cache = + new PrimaryResourceCache<>( + (statusPatchCacheCustomResourcePair, statusPatchCacheCustomResource) -> + statusPatchCacheCustomResource.getStatus().getValue() + >= statusPatchCacheCustomResourcePair.afterUpdate().getStatus().getValue()); + + @Override + public UpdateControl reconcile( + StatusPatchPrimaryCacheCustomResource primary, + Context context) { + + primary = cache.getFreshResource(primary); + + if (primary.getStatus() != null && primary.getStatus().getValue() != latestValue) { + errorPresent = true; + throw new IllegalStateException( + "status is not up to date. Latest value: " + + latestValue + + " status values: " + + primary.getStatus().getValue()); + } + + var freshCopy = createFreshCopy(primary); + freshCopy + .getStatus() + .setValue(primary.getStatus() == null ? 1 : primary.getStatus().getValue() + 1); + + var updated = + PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(primary, freshCopy, context, cache); + latestValue = updated.getStatus().getValue(); + + return UpdateControl.noUpdate(); + } + + @Override + public List> prepareEventSources( + EventSourceContext context) { + // periodic event triggering for testing purposes + return List.of(new PeriodicTriggerEventSource<>(context.getPrimaryCache())); + } + + private StatusPatchPrimaryCacheCustomResource createFreshCopy( + StatusPatchPrimaryCacheCustomResource resource) { + var res = new StatusPatchPrimaryCacheCustomResource(); + res.setMetadata( + new ObjectMetaBuilder() + .withName(resource.getMetadata().getName()) + .withNamespace(resource.getMetadata().getNamespace()) + .build()); + res.setStatus(new StatusPatchPrimaryCacheStatus()); + + return res; + } + + @Override + public DeleteControl cleanup( + StatusPatchPrimaryCacheCustomResource resource, + Context context) + throws Exception { + cache.cleanup(resource); + return DeleteControl.defaultDelete(); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java new file mode 100644 index 0000000000..90630c1ae8 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; + +public class StatusPatchPrimaryCacheSpec { + + private boolean messageInStatus = true; + + public boolean isMessageInStatus() { + return messageInStatus; + } + + public StatusPatchPrimaryCacheSpec setMessageInStatus(boolean messageInStatus) { + this.messageInStatus = messageInStatus; + return this; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java new file mode 100644 index 0000000000..0687d5576a --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; + +public class StatusPatchPrimaryCacheStatus { + + private Integer value = 0; + + public Integer getValue() { + return value; + } + + public StatusPatchPrimaryCacheStatus setValue(Integer value) { + this.value = value; + return this; + } +} From a92bf98ed4f9983432a486c33c55e130f231f36d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 30 Apr 2025 17:14:37 +0200 Subject: [PATCH 26/45] fix: retry finalizer removal on http 422 (#2776) --- .../event/ReconciliationDispatcher.java | 14 +++- ...currentFinalizerRemovalCustomResource.java | 13 ++++ .../ConcurrentFinalizerRemovalIT.java | 68 +++++++++++++++++++ ...ConcurrentFinalizerRemovalReconciler1.java | 29 ++++++++ ...ConcurrentFinalizerRemovalReconciler2.java | 29 ++++++++ .../ConcurrentFinalizerRemovalSpec.java | 15 ++++ 6 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalCustomResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler1.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler2.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalSpec.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java index 8fb9a30ae9..9b34794066 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java @@ -381,8 +381,13 @@ public P conflictRetryingPatch( } catch (KubernetesClientException e) { log.trace("Exception during patch for resource: {}", resource); retryIndex++; - // only retry on conflict (HTTP 409), otherwise fail - if (e.getCode() != 409) { + // only retry on conflict (409) and unprocessable content (422) which + // can happen if JSON Patch is not a valid request since there was + // a concurrent request which already removed another finalizer: + // List element removal from a list is by index in JSON Patch + // so if addressing a second finalizer but first is meanwhile removed + // it is a wrong request. + if (e.getCode() != 409 && e.getCode() != 422) { throw e; } if (retryIndex >= MAX_UPDATE_RETRY) { @@ -392,6 +397,11 @@ public P conflictRetryingPatch( + ") retry attempts to patch resource: " + ResourceID.fromResource(resource)); } + log.debug( + "Retrying patch for resource name: {}, namespace: {}; HTTP code: {}", + resource.getMetadata().getName(), + resource.getMetadata().getNamespace(), + e.getCode()); resource = customResourceFacade.getResource( resource.getMetadata().getNamespace(), resource.getMetadata().getName()); diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalCustomResource.java new file mode 100644 index 0000000000..3f9b61d1a8 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalCustomResource.java @@ -0,0 +1,13 @@ +package io.javaoperatorsdk.operator.baseapi.concurrentfinalizerremoval; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("cfr") +public class ConcurrentFinalizerRemovalCustomResource + extends CustomResource implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalIT.java new file mode 100644 index 0000000000..da166ce2c1 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalIT.java @@ -0,0 +1,68 @@ +package io.javaoperatorsdk.operator.baseapi.concurrentfinalizerremoval; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; +import io.javaoperatorsdk.operator.processing.retry.GenericRetry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class ConcurrentFinalizerRemovalIT { + + private static final Logger log = LoggerFactory.getLogger(ConcurrentFinalizerRemovalIT.class); + public static final String TEST_RESOURCE_NAME = "test"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + // should work without a retry, thus not retry the whole reconciliation but to retry + // finalizer removal only. + .withReconciler( + new ConcurrentFinalizerRemovalReconciler1(), + o -> + o.withRetry(GenericRetry.noRetry()).withFinalizer("reconciler1.sample/finalizer")) + .withReconciler( + new ConcurrentFinalizerRemovalReconciler2(), + o -> + o.withRetry(GenericRetry.noRetry()).withFinalizer("reconciler2.sample/finalizer")) + .build(); + + @Test + void concurrentFinalizerRemoval() { + for (int i = 0; i < 10; i++) { + var resource = extension.create(createResource()); + await() + .untilAsserted( + () -> { + var res = + extension.get( + ConcurrentFinalizerRemovalCustomResource.class, TEST_RESOURCE_NAME); + assertThat(res.getMetadata().getFinalizers()).hasSize(2); + }); + resource.getMetadata().setResourceVersion(null); + extension.delete(resource); + + await() + .untilAsserted( + () -> { + var res = + extension.get( + ConcurrentFinalizerRemovalCustomResource.class, TEST_RESOURCE_NAME); + assertThat(res).isNull(); + }); + } + } + + public ConcurrentFinalizerRemovalCustomResource createResource() { + ConcurrentFinalizerRemovalCustomResource res = new ConcurrentFinalizerRemovalCustomResource(); + res.setMetadata(new ObjectMetaBuilder().withName(TEST_RESOURCE_NAME).build()); + res.setSpec(new ConcurrentFinalizerRemovalSpec()); + res.getSpec().setNumber(0); + return res; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler1.java new file mode 100644 index 0000000000..b789255cb5 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler1.java @@ -0,0 +1,29 @@ +package io.javaoperatorsdk.operator.baseapi.concurrentfinalizerremoval; + +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; + +@ControllerConfiguration +public class ConcurrentFinalizerRemovalReconciler1 + implements Reconciler, + Cleaner { + + @Override + public UpdateControl reconcile( + ConcurrentFinalizerRemovalCustomResource resource, + Context context) { + return UpdateControl.noUpdate(); + } + + @Override + public DeleteControl cleanup( + ConcurrentFinalizerRemovalCustomResource resource, + Context context) + throws Exception { + return DeleteControl.defaultDelete(); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler2.java new file mode 100644 index 0000000000..0b8993a8f5 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalReconciler2.java @@ -0,0 +1,29 @@ +package io.javaoperatorsdk.operator.baseapi.concurrentfinalizerremoval; + +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; + +@ControllerConfiguration +public class ConcurrentFinalizerRemovalReconciler2 + implements Reconciler, + Cleaner { + + @Override + public UpdateControl reconcile( + ConcurrentFinalizerRemovalCustomResource resource, + Context context) { + return UpdateControl.noUpdate(); + } + + @Override + public DeleteControl cleanup( + ConcurrentFinalizerRemovalCustomResource resource, + Context context) + throws Exception { + return DeleteControl.defaultDelete(); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalSpec.java new file mode 100644 index 0000000000..d740721f30 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/concurrentfinalizerremoval/ConcurrentFinalizerRemovalSpec.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.baseapi.concurrentfinalizerremoval; + +public class ConcurrentFinalizerRemovalSpec { + + private int number; + + public int getNumber() { + return number; + } + + public ConcurrentFinalizerRemovalSpec setNumber(int number) { + this.number = number; + return this; + } +} From c5441fb6f7b66428aa6987c33f51f4ed74c22ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 May 2025 08:03:19 +0200 Subject: [PATCH 27/45] chore(deps): bump fabric8-client.version from 7.1.0 to 7.2.0 (#2780) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f7be33133f..9f41d27848 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ jdk 5.12.2 - 7.1.0 + 7.2.0 2.0.12 2.24.3 5.17.0 From 3b29d2fdfea154f7cbf5d3274cfa878c8fa501b9 Mon Sep 17 00:00:00 2001 From: Donnerbart Date: Thu, 8 May 2025 09:52:53 +0200 Subject: [PATCH 28/45] feat: Make primary resource accessible from Context (#2782) Signed-off-by: David Sondermann --- .../operator/api/reconciler/Context.java | 7 ++++ .../api/reconciler/DefaultContext.java | 33 ++++++++++--------- .../event/ReconciliationDispatcher.java | 2 -- .../api/reconciler/DefaultContextTest.java | 17 +++++++--- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Context.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Context.java index e5fbaad68e..f47deb9734 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Context.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Context.java @@ -46,6 +46,13 @@ default Stream getSecondaryResourcesAsStream(Class expectedType) { /** ExecutorService initialized by framework for workflows. Used for workflow standalone mode. */ ExecutorService getWorkflowExecutorService(); + /** + * Retrieves the primary resource. + * + * @return the primary resource associated with the current reconciliation + */ + P getPrimaryResource(); + /** * Retrieves the primary resource cache. * diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java index b5ea66f8bc..2acf8d13ca 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java @@ -44,18 +44,6 @@ public Set getSecondaryResources(Class expectedType) { return getSecondaryResourcesAsStream(expectedType).collect(Collectors.toSet()); } - @Override - public IndexedResourceCache

getPrimaryCache() { - return controller.getEventSourceManager().getControllerEventSource(); - } - - @Override - public boolean isNextReconciliationImminent() { - return controller - .getEventProcessor() - .isNextReconciliationImminent(ResourceID.fromResource(primaryResource)); - } - @Override public Stream getSecondaryResourcesAsStream(Class expectedType) { return controller.getEventSourceManager().getEventSourcesFor(expectedType).stream() @@ -114,12 +102,25 @@ public ExecutorService getWorkflowExecutorService() { return controller.getExecutorServiceManager().workflowExecutorService(); } + @Override + public P getPrimaryResource() { + return primaryResource; + } + + @Override + public IndexedResourceCache

getPrimaryCache() { + return controller.getEventSourceManager().getControllerEventSource(); + } + + @Override + public boolean isNextReconciliationImminent() { + return controller + .getEventProcessor() + .isNextReconciliationImminent(ResourceID.fromResource(primaryResource)); + } + public DefaultContext

setRetryInfo(RetryInfo retryInfo) { this.retryInfo = retryInfo; return this; } - - public P getPrimaryResource() { - return primaryResource; - } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java index 9b34794066..c4b161ef27 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java @@ -180,10 +180,8 @@ private PostExecutionControl

reconcileExecution( return createPostExecutionControl(updatedCustomResource, updateControl); } - @SuppressWarnings("unchecked") private PostExecutionControl

handleErrorStatusHandler( P resource, P originalResource, Context

context, Exception e) throws Exception { - RetryInfo retryInfo = context .getRetryInfo() diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContextTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContextTest.java index 296974c4cd..b289d68b22 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContextTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContextTest.java @@ -9,19 +9,19 @@ import io.javaoperatorsdk.operator.processing.event.NoEventSourceForClassException; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; class DefaultContextTest { - Secret primary = new Secret(); - Controller mockController = mock(Controller.class); + private final Secret primary = new Secret(); + private final Controller mockController = mock(); - DefaultContext context = new DefaultContext<>(null, mockController, primary); + private final DefaultContext context = new DefaultContext<>(null, mockController, primary); @Test + @SuppressWarnings("unchecked") void getSecondaryResourceReturnsEmptyOptionalOnNonActivatedDRType() { var mockManager = mock(EventSourceManager.class); when(mockController.getEventSourceManager()).thenReturn(mockManager); @@ -30,7 +30,14 @@ void getSecondaryResourceReturnsEmptyOptionalOnNonActivatedDRType() { .thenThrow(new NoEventSourceForClassException(ConfigMap.class)); var res = context.getSecondaryResource(ConfigMap.class); - assertThat(res).isEmpty(); } + + @Test + void setRetryInfo() { + RetryInfo retryInfo = mock(); + var newContext = context.setRetryInfo(retryInfo); + assertThat(newContext).isSameAs(context); + assertThat(newContext.getRetryInfo()).hasValue(retryInfo); + } } From e775349f9e23f46c1148245fe73b5e4b8d108117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 8 May 2025 10:01:29 +0200 Subject: [PATCH 29/45] Bump minikube and kubernetes versions (#2777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/build.yml | 2 +- .github/workflows/e2e-test.yml | 4 ++-- .github/workflows/integration-tests.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 530921009c..b81fa41b6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: java: [ 17, 21, 24 ] - kubernetes: [ 'v1.29.12','1.30.8', '1.31.4', '1.32.0' ] + kubernetes: [ '1.30.12', '1.31.8', '1.32.4','1.33.0' ] uses: ./.github/workflows/integration-tests.yml with: java-version: ${{ matrix.java }} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 4ac58ab062..e06b427960 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -30,10 +30,10 @@ jobs: uses: actions/checkout@v4 - name: Setup Minikube-Kubernetes - uses: manusa/actions-setup-minikube@v2.13.1 + uses: manusa/actions-setup-minikube@v2.14.0 with: minikube version: v1.34.0 - kubernetes version: v1.32.0 + kubernetes version: v1.33.0 github token: ${{ secrets.GITHUB_TOKEN }} driver: docker diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d5aca2ad54..75a6093371 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -39,7 +39,7 @@ jobs: java-version: ${{ inputs.java-version }} cache: 'maven' - name: Set up Minikube - uses: manusa/actions-setup-minikube@v2.13.1 + uses: manusa/actions-setup-minikube@v2.14.0 with: minikube version: 'v1.34.0' kubernetes version: '${{ inputs.kube-version }}' From e91e39ab11d58f40379bf171d0684df6bb853fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 9 May 2025 09:42:25 +0200 Subject: [PATCH 30/45] improve: remove fabric8 related daily build (#2790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../fabric8-next-version-schedule.yml | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/fabric8-next-version-schedule.yml diff --git a/.github/workflows/fabric8-next-version-schedule.yml b/.github/workflows/fabric8-next-version-schedule.yml deleted file mode 100644 index 64d2042135..0000000000 --- a/.github/workflows/fabric8-next-version-schedule.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Fabric8 Client Snapshot Build - -env: - MAVEN_ARGS: -V -ntp -e - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true -on: - schedule: - # Run on end of the day - - cron: '0 0 * * *' - workflow_dispatch: -jobs: - check_format_and_unit_tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: 'fabric8-next-version' - - name: Set up Java and Maven - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 17 - - name: Run unit tests - run: ./mvnw ${MAVEN_ARGS} clean install --file pom.xml - - build: - uses: ./.github/workflows/build.yml \ No newline at end of file From 820210ce52bf9b8e8cd95c0e67fef051faa6fa1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 9 May 2025 14:20:43 +0200 Subject: [PATCH 31/45] improve: increase bounded cache IT GC timeout (#2785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the test fails some times because of this Signed-off-by: Attila Mészáros --- .../processing/event/source/cache/BoundedCacheTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/BoundedCacheTestBase.java b/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/BoundedCacheTestBase.java index 05d31a7479..532e5237f8 100644 --- a/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/BoundedCacheTestBase.java +++ b/caffeine-bounded-cache-support/src/test/java/io/javaoperatorsdk/operator/processing/event/source/cache/BoundedCacheTestBase.java @@ -44,7 +44,7 @@ void reconciliationWorksWithLimitedCache() { private void assertConfigMapsDeleted() { await() - .atMost(Duration.ofSeconds(30)) + .atMost(Duration.ofSeconds(120)) .untilAsserted( () -> IntStream.range(0, NUMBER_OF_RESOURCE_TO_TEST) From 5b673a40e5d082b4d76c29de7fb8da7da8b48588 Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Mon, 12 May 2025 09:57:29 +0200 Subject: [PATCH 32/45] feat: allow easier configuration of matcher (#2760) * feat: allow easier configuration of matcher Signed-off-by: Chris Laprun * refactor: avoid recording default matcher Signed-off-by: Chris Laprun * refactor: make matcher configurable instead of settable Signed-off-by: Chris Laprun --------- Signed-off-by: Chris Laprun --- .../kubernetes/KubernetesDependent.java | 19 +++++++++++++++ .../KubernetesDependentConverter.java | 14 ++++++++++- .../KubernetesDependentResource.java | 5 +++- .../KubernetesDependentResourceConfig.java | 10 +++++++- ...ernetesDependentResourceConfigBuilder.java | 9 +++++++- ...dGenericKubernetesResourceMatcherTest.java | 23 +++++++++++++++++-- 6 files changed, 74 insertions(+), 6 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependent.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependent.java index 4e32246a38..484ffb64c8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependent.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependent.java @@ -27,5 +27,24 @@ boolean createResourceOnlyIfNotExistingWithSSA() default */ BooleanWithUndefined useSSA() default BooleanWithUndefined.UNDEFINED; + /** + * The underlying Informer event source configuration + * + * @return the {@link Informer} configuration + */ Informer informer() default @Informer; + + /** + * The specific matcher implementation to use when Server-Side Apply (SSA) is used, when case the + * default one isn't working appropriately. Typically, this could be needed to cover border cases + * with some Kubernetes resources that are modified by their controllers to normalize or add + * default values, which could result in infinite loops with the default matcher. Using a specific + * matcher could also be an optimization decision if determination of whether two resources match + * can be done faster than what can be done with the default exhaustive algorithm. + * + * @return the class of the specific matcher to use for the associated dependent resources + * @since 5.1 + */ + Class matcher() default + SSABasedGenericKubernetesResourceMatcher.class; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentConverter.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentConverter.java index 6f1d7e3a64..7d68b0e106 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentConverter.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentConverter.java @@ -22,10 +22,22 @@ public KubernetesDependentResourceConfig configFrom( DEFAULT_CREATE_RESOURCE_ONLY_IF_NOT_EXISTING_WITH_SSA; Boolean useSSA = null; + SSABasedGenericKubernetesResourceMatcher matcher = + SSABasedGenericKubernetesResourceMatcher.getInstance(); if (configAnnotation != null) { createResourceOnlyIfNotExistingWithSSA = configAnnotation.createResourceOnlyIfNotExistingWithSSA(); useSSA = configAnnotation.useSSA().asBoolean(); + + // check if we have a specific matcher + Class> dependentResourceClass = + (Class>) spec.getDependentResourceClass(); + final var context = + Utils.contextFor( + controllerConfig, dependentResourceClass, configAnnotation.annotationType()); + matcher = + Utils.instantiate( + configAnnotation.matcher(), SSABasedGenericKubernetesResourceMatcher.class, context); } var informerConfiguration = @@ -35,7 +47,7 @@ public KubernetesDependentResourceConfig configFrom( controllerConfig); return new KubernetesDependentResourceConfig<>( - useSSA, createResourceOnlyIfNotExistingWithSSA, informerConfiguration); + useSSA, createResourceOnlyIfNotExistingWithSSA, informerConfiguration, matcher); } @SuppressWarnings({"unchecked"}) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java index 382ac7525c..ea7edbc1a0 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java @@ -37,6 +37,7 @@ public abstract class KubernetesDependentResource> { private static final Logger log = LoggerFactory.getLogger(KubernetesDependentResource.class); + private final boolean garbageCollected = this instanceof GarbageCollected; private KubernetesDependentResourceConfig kubernetesDependentResourceConfig; private volatile Boolean useSSA; @@ -112,7 +113,9 @@ public Result match(R actualResource, R desired, P primary, Context

contex addMetadata(true, actualResource, desired, primary, context); if (useSSA(context)) { matches = - SSABasedGenericKubernetesResourceMatcher.getInstance() + configuration() + .map(KubernetesDependentResourceConfig::matcher) + .orElse(SSABasedGenericKubernetesResourceMatcher.getInstance()) .matches(actualResource, desired, context); } else { matches = diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java index c3424750d2..bcfe2f9fe6 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java @@ -10,14 +10,18 @@ public class KubernetesDependentResourceConfig { private final Boolean useSSA; private final boolean createResourceOnlyIfNotExistingWithSSA; private final InformerConfiguration informerConfig; + private final SSABasedGenericKubernetesResourceMatcher matcher; public KubernetesDependentResourceConfig( Boolean useSSA, boolean createResourceOnlyIfNotExistingWithSSA, - InformerConfiguration informerConfig) { + InformerConfiguration informerConfig, + SSABasedGenericKubernetesResourceMatcher matcher) { this.useSSA = useSSA; this.createResourceOnlyIfNotExistingWithSSA = createResourceOnlyIfNotExistingWithSSA; this.informerConfig = informerConfig; + this.matcher = + matcher != null ? matcher : SSABasedGenericKubernetesResourceMatcher.getInstance(); } public boolean createResourceOnlyIfNotExistingWithSSA() { @@ -31,4 +35,8 @@ public Boolean useSSA() { public InformerConfiguration informerConfig() { return informerConfig; } + + public SSABasedGenericKubernetesResourceMatcher matcher() { + return matcher; + } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfigBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfigBuilder.java index 7694fe1d46..371fb700c3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfigBuilder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfigBuilder.java @@ -8,6 +8,7 @@ public final class KubernetesDependentResourceConfigBuilder informerConfiguration; + private SSABasedGenericKubernetesResourceMatcher matcher; public KubernetesDependentResourceConfigBuilder() {} @@ -29,8 +30,14 @@ public KubernetesDependentResourceConfigBuilder withKubernetesDependentInform return this; } + public KubernetesDependentResourceConfigBuilder withSSAMatcher( + SSABasedGenericKubernetesResourceMatcher matcher) { + this.matcher = matcher; + return this; + } + public KubernetesDependentResourceConfig build() { return new KubernetesDependentResourceConfig<>( - useSSA, createResourceOnlyIfNotExistingWithSSA, informerConfiguration); + useSSA, createResourceOnlyIfNotExistingWithSSA, informerConfiguration, matcher); } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java index 176531344c..e87842c103 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java @@ -22,6 +22,7 @@ import io.javaoperatorsdk.operator.api.reconciler.Context; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -39,6 +40,7 @@ void setup() { when(mockedContext.getClient()).thenReturn(client); final var configurationService = mock(ConfigurationService.class); + when(configurationService.shouldUseSSA(any(), any(), any())).thenReturn(true); final var controllerConfiguration = mock(ControllerConfiguration.class); when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); when(controllerConfiguration.fieldManager()).thenReturn("controller"); @@ -239,17 +241,34 @@ void testSanitizeState_daemonSetWithResources_withMismatch() { @ParameterizedTest @ValueSource(booleans = {true, false}) void testCustomMatcher_returnsExpectedMatchBasedOnReadOnlyLabel(boolean readOnly) { + var dr = new ConfigMapDR(); + dr.configureWith( + new KubernetesDependentResourceConfigBuilder() + .withSSAMatcher(new ReadOnlyAwareMatcher()) + .build()); var desiredConfigMap = loadResource("configmap.empty-owner-reference-desired.yaml", ConfigMap.class); desiredConfigMap.getData().put("key1", "another value"); var actualConfigMap = loadResource("configmap.empty-owner-reference.yaml", ConfigMap.class); actualConfigMap.getMetadata().getLabels().put("readonly", Boolean.toString(readOnly)); - var matcher = new ReadOnlyAwareMatcher(); - assertThat(matcher.matches(actualConfigMap, desiredConfigMap, mockedContext)) + ConfigMap ignoredPrimary = null; + assertThat( + dr.match( + actualConfigMap, + desiredConfigMap, + ignoredPrimary, + (Context) mockedContext) + .matched()) .isEqualTo(readOnly); } + private static class ConfigMapDR extends KubernetesDependentResource { + public ConfigMapDR() { + super(ConfigMap.class); + } + } + private static class ReadOnlyAwareMatcher extends SSABasedGenericKubernetesResourceMatcher { @Override From 681dd59be1f65b211f91ce618005235848cdf3c0 Mon Sep 17 00:00:00 2001 From: Donnerbart Date: Mon, 12 May 2025 16:52:57 +0200 Subject: [PATCH 33/45] Increase code coverage of SSABasedGenericKubernetesResourceMatcher (#2781) * test: Add missing tests for SSABasedGenericKubernetesResourceMatcher Signed-off-by: David Sondermann * test: Add missing tests for SSABasedGenericKubernetesResourceMatcher Signed-off-by: David Sondermann --------- Signed-off-by: David Sondermann --- ...BasedGenericKubernetesResourceMatcher.java | 326 +++++++++--------- ...dGenericKubernetesResourceMatcherTest.java | 213 +++++++++--- ...nfigmap.empty-owner-reference-desired.yaml | 2 - ...-managed-fields-additional-controller.yaml | 1 + .../multi-container-pod-desired.yaml | 2 +- .../kubernetes/multi-container-pod.yaml | 2 +- .../dependent/kubernetes/secret-desired.yaml | 7 + .../secret-with-finalizer-desired.yaml | 9 + .../kubernetes/secret-with-finalizer.yaml | 25 ++ .../dependent/kubernetes/secret.yaml | 19 + 10 files changed, 393 insertions(+), 213 deletions(-) create mode 100644 operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-desired.yaml create mode 100644 operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer-desired.yaml create mode 100644 operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer.yaml create mode 100644 operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret.yaml diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java index eed766fc95..3c051acfb4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java @@ -55,18 +55,6 @@ public class SSABasedGenericKubernetesResourceMatcher { public static final String APPLY_OPERATION = "Apply"; public static final String DOT_KEY = "."; - @SuppressWarnings("rawtypes") - private static final SSABasedGenericKubernetesResourceMatcher INSTANCE = - new SSABasedGenericKubernetesResourceMatcher<>(); - - private static final List IGNORED_METADATA = - List.of("creationTimestamp", "deletionTimestamp", "generation", "selfLink", "uid"); - - @SuppressWarnings("unchecked") - public static SSABasedGenericKubernetesResourceMatcher getInstance() { - return INSTANCE; - } - private static final String F_PREFIX = "f:"; private static final String K_PREFIX = "k:"; private static final String V_PREFIX = "v:"; @@ -76,9 +64,21 @@ public static SSABasedGenericKubernetesResourceMatcher(); + + private static final List IGNORED_METADATA = + List.of("creationTimestamp", "deletionTimestamp", "generation", "selfLink", "uid"); + private static final Logger log = LoggerFactory.getLogger(SSABasedGenericKubernetesResourceMatcher.class); + @SuppressWarnings("unchecked") + public static SSABasedGenericKubernetesResourceMatcher getInstance() { + return INSTANCE; + } + @SuppressWarnings("unchecked") public boolean matches(R actual, R desired, Context context) { var optionalManagedFieldsEntry = @@ -147,54 +147,35 @@ protected boolean matches( return actualMap.equals(desiredMap); } - private String getDiff( - Map prunedActualMap, - Map desiredMap, - KubernetesSerialization serialization) { - var actualYaml = serialization.asYaml(sortMap(prunedActualMap)); - var desiredYaml = serialization.asYaml(sortMap(desiredMap)); - if (log.isTraceEnabled()) { - log.trace("Pruned actual resource:\n {} \ndesired resource:\n {} ", actualYaml, desiredYaml); - } - - var patch = DiffUtils.diff(actualYaml.lines().toList(), desiredYaml.lines().toList()); - var unifiedDiff = - UnifiedDiffUtils.generateUnifiedDiff("", "", actualYaml.lines().toList(), patch, 1); - return String.join("\n", unifiedDiff); - } - - @SuppressWarnings("unchecked") - Map sortMap(Map map) { - var sortedKeys = new ArrayList<>(map.keySet()); - Collections.sort(sortedKeys); - - var sortedMap = new LinkedHashMap(); - for (var key : sortedKeys) { - var value = map.get(key); - if (value instanceof Map) { - sortedMap.put(key, sortMap((Map) value)); - } else if (value instanceof List) { - sortedMap.put(key, sortListItems((List) value)); - } else { - sortedMap.put(key, value); - } + private Optional checkIfFieldManagerExists(R actual, String fieldManager) { + var targetManagedFields = + actual.getMetadata().getManagedFields().stream() + // Only the apply operations are interesting for us since those were created properly be + // SSA patch. An update can be present with same fieldManager when migrating and having + // the same field manager name + .filter( + f -> + f.getManager().equals(fieldManager) && f.getOperation().equals(APPLY_OPERATION)) + .toList(); + if (targetManagedFields.isEmpty()) { + log.debug( + "No field manager exists for resource: {} with name: {} and operation {}", + actual.getKind(), + actual.getMetadata().getName(), + APPLY_OPERATION); + return Optional.empty(); } - return sortedMap; - } - - @SuppressWarnings("unchecked") - List sortListItems(List list) { - var sortedList = new ArrayList<>(); - for (var item : list) { - if (item instanceof Map) { - sortedList.add(sortMap((Map) item)); - } else if (item instanceof List) { - sortedList.add(sortListItems((List) item)); - } else { - sortedList.add(item); - } + // this should not happen in theory + if (targetManagedFields.size() > 1) { + throw new OperatorException( + "More than one field manager exists with name: " + + fieldManager + + " in resource: " + + actual.getKind() + + " with name: " + + actual.getMetadata().getName()); } - return sortedList; + return Optional.of(targetManagedFields.get(0)); } /** Correct for known issue with SSA */ @@ -245,20 +226,7 @@ private void sanitizeState(R actual, R desired, Map actualMap) { } @SuppressWarnings("unchecked") - private static void removeIrrelevantValues(Map desiredMap) { - var metadata = (Map) desiredMap.get(METADATA_KEY); - metadata.remove(NAME_KEY); - metadata.remove(NAMESPACE_KEY); - IGNORED_METADATA.forEach(metadata::remove); - if (metadata.isEmpty()) { - desiredMap.remove(METADATA_KEY); - } - desiredMap.remove(KIND_KEY); - desiredMap.remove(API_VERSION_KEY); - } - - @SuppressWarnings("unchecked") - private static void keepOnlyManagedFields( + static void keepOnlyManagedFields( Map result, Map actualMap, Map managedFields, @@ -292,7 +260,7 @@ private static void keepOnlyManagedFields( } } else { // this should handle the case when the value is complex in the actual map (not just a - // simple value). + // simple value) result.put(keyInActual, actualMap.get(keyInActual)); } } else { @@ -304,30 +272,33 @@ private static void keepOnlyManagedFields( } } - @SuppressWarnings("unchecked") - private static void fillResultsAndTraverseFurther( - Map result, - Map actualMap, - Map managedFields, - KubernetesSerialization objectMapper, - String key, - String keyInActual, - Object managedFieldValue) { - var emptyMapValue = new HashMap(); - result.put(keyInActual, emptyMapValue); - var actualMapValue = actualMap.getOrDefault(keyInActual, Collections.emptyMap()); - log.debug("key: {} actual map value: managedFieldValue: {}", keyInActual, managedFieldValue); - keepOnlyManagedFields( - emptyMapValue, - (Map) actualMapValue, - (Map) managedFields.get(key), - objectMapper); - } - private static boolean isNestedValue(Map managedFieldValue) { return !managedFieldValue.isEmpty(); } + private static boolean isListKeyEntrySet(Set> managedEntrySet) { + return isKeyPrefixedSkippingDotKey(managedEntrySet, K_PREFIX); + } + + private static boolean isSetValueField(Set> managedEntrySet) { + return isKeyPrefixedSkippingDotKey(managedEntrySet, V_PREFIX); + } + + /** + * Sometimes (not always) the first subfield of a managed field ("f:") is ".:{}", it looks that + * those are added when there are more subfields of a referenced field. See test samples. Does not + * seem to provide additional functionality, so can be just skipped for now. + */ + private static boolean isKeyPrefixedSkippingDotKey( + Set> managedEntrySet, String prefix) { + var iterator = managedEntrySet.iterator(); + var managedFieldEntry = iterator.next(); + if (managedFieldEntry.getKey().equals(DOT_KEY)) { + managedFieldEntry = iterator.next(); + } + return managedFieldEntry.getKey().startsWith(prefix); + } + /** * List entries referenced by key, or when "k:" prefix is used. It works in a way that it selects * the target element based on the field(s) in "k:" for example when there is a list of element of @@ -372,6 +343,36 @@ private static void handleListKeyEntrySet( }); } + @SuppressWarnings("unchecked") + private static Map.Entry> selectListEntryBasedOnKey( + String key, List> values, KubernetesSerialization objectMapper) { + Map ids = objectMapper.unmarshal(key, Map.class); + var possibleTargets = new ArrayList>(1); + int lastIndex = -1; + for (int i = 0; i < values.size(); i++) { + var value = values.get(i); + if (value.entrySet().containsAll(ids.entrySet())) { + possibleTargets.add(value); + lastIndex = i; + } + } + if (possibleTargets.isEmpty()) { + throw new IllegalStateException( + "Cannot find list element for key: " + + key + + " in map: " + + values.stream().map(Map::keySet).toList()); + } + if (possibleTargets.size() > 1) { + throw new IllegalStateException( + "More targets found in list element for key: " + + key + + " in map: " + + values.stream().map(Map::keySet).toList()); + } + return new AbstractMap.SimpleEntry<>(lastIndex, possibleTargets.get(0)); + } + /** * Set values, the {@code "v:"} prefix. Form in managed fields: {@code * "f:some-set":{"v:1":{}},"v:2":{},"v:3":{}}. @@ -407,90 +408,87 @@ public static Object parseKeyValue( return objectMapper.unmarshal(stringValue.trim(), type); } - private static boolean isSetValueField(Set> managedEntrySet) { - return isKeyPrefixedSkippingDotKey(managedEntrySet, V_PREFIX); + @SuppressWarnings("unchecked") + private static void fillResultsAndTraverseFurther( + Map result, + Map actualMap, + Map managedFields, + KubernetesSerialization objectMapper, + String key, + String keyInActual, + Object managedFieldValue) { + var emptyMapValue = new HashMap(); + result.put(keyInActual, emptyMapValue); + var actualMapValue = actualMap.getOrDefault(keyInActual, Collections.emptyMap()); + log.debug("key: {} actual map value: managedFieldValue: {}", keyInActual, managedFieldValue); + keepOnlyManagedFields( + emptyMapValue, + (Map) actualMapValue, + (Map) managedFields.get(key), + objectMapper); } - private static boolean isListKeyEntrySet(Set> managedEntrySet) { - return isKeyPrefixedSkippingDotKey(managedEntrySet, K_PREFIX); + @SuppressWarnings("unchecked") + private static void removeIrrelevantValues(Map desiredMap) { + var metadata = (Map) desiredMap.get(METADATA_KEY); + metadata.remove(NAME_KEY); + metadata.remove(NAMESPACE_KEY); + IGNORED_METADATA.forEach(metadata::remove); + if (metadata.isEmpty()) { + desiredMap.remove(METADATA_KEY); + } + desiredMap.remove(KIND_KEY); + desiredMap.remove(API_VERSION_KEY); } - /** - * Sometimes (not always) the first subfield of a managed field ("f:") is ".:{}", it looks that - * those are added when there are more subfields of a referenced field. See test samples. Does not - * seem to provide additional functionality, so can be just skipped for now. - */ - private static boolean isKeyPrefixedSkippingDotKey( - Set> managedEntrySet, String prefix) { - var iterator = managedEntrySet.iterator(); - var managedFieldEntry = iterator.next(); - if (managedFieldEntry.getKey().equals(DOT_KEY)) { - managedFieldEntry = iterator.next(); + private static String getDiff( + Map prunedActualMap, + Map desiredMap, + KubernetesSerialization serialization) { + var actualYaml = serialization.asYaml(sortMap(prunedActualMap)); + var desiredYaml = serialization.asYaml(sortMap(desiredMap)); + if (log.isTraceEnabled()) { + log.trace("Pruned actual resource:\n {} \ndesired resource:\n {} ", actualYaml, desiredYaml); } - return managedFieldEntry.getKey().startsWith(prefix); + + var patch = DiffUtils.diff(actualYaml.lines().toList(), desiredYaml.lines().toList()); + var unifiedDiff = + UnifiedDiffUtils.generateUnifiedDiff("", "", actualYaml.lines().toList(), patch, 1); + return String.join("\n", unifiedDiff); } @SuppressWarnings("unchecked") - private static Map.Entry> selectListEntryBasedOnKey( - String key, List> values, KubernetesSerialization objectMapper) { - Map ids = objectMapper.unmarshal(key, Map.class); - var possibleTargets = new ArrayList>(1); - int lastIndex = -1; - for (int i = 0; i < values.size(); i++) { - var value = values.get(i); - if (value.entrySet().containsAll(ids.entrySet())) { - possibleTargets.add(value); - lastIndex = i; + static Map sortMap(Map map) { + var sortedKeys = new ArrayList<>(map.keySet()); + Collections.sort(sortedKeys); + + var sortedMap = new LinkedHashMap(); + for (var key : sortedKeys) { + var value = map.get(key); + if (value instanceof Map) { + sortedMap.put(key, sortMap((Map) value)); + } else if (value instanceof List) { + sortedMap.put(key, sortListItems((List) value)); + } else { + sortedMap.put(key, value); } } - if (possibleTargets.isEmpty()) { - throw new IllegalStateException( - "Cannot find list element for key: " - + key - + " in map: " - + values.stream().map(Map::keySet).toList()); - } - if (possibleTargets.size() > 1) { - throw new IllegalStateException( - "More targets found in list element for key: " - + key - + " in map: " - + values.stream().map(Map::keySet).toList()); - } - return new AbstractMap.SimpleEntry<>(lastIndex, possibleTargets.get(0)); + return sortedMap; } - private Optional checkIfFieldManagerExists(R actual, String fieldManager) { - var targetManagedFields = - actual.getMetadata().getManagedFields().stream() - // Only the apply operations are interesting for us since those were created properly be - // SSA - // Patch. An update can be present with same fieldManager when migrating and having the - // same - // field manager name. - .filter( - f -> - f.getManager().equals(fieldManager) && f.getOperation().equals(APPLY_OPERATION)) - .toList(); - if (targetManagedFields.isEmpty()) { - log.debug( - "No field manager exists for resource: {} with name: {} and operation {}", - actual.getKind(), - actual.getMetadata().getName(), - APPLY_OPERATION); - return Optional.empty(); - } - // this should not happen in theory - if (targetManagedFields.size() > 1) { - throw new OperatorException( - "More than one field manager exists with name: " - + fieldManager - + " in resource: " - + actual.getKind() - + " with name: " - + actual.getMetadata().getName()); + @SuppressWarnings("unchecked") + static List sortListItems(List list) { + var sortedList = new ArrayList<>(); + for (var item : list) { + if (item instanceof Map) { + sortedList.add(sortMap((Map) item)); + } else if (item instanceof List) { + sortedList.add(sortListItems((List) item)); + } else { + sortedList.add(item); + } } - return Optional.of(targetManagedFields.get(0)); + return sortedList; } private static String keyWithoutPrefix(String key) { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java index e87842c103..c339e5ebf6 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcherTest.java @@ -1,6 +1,5 @@ package io.javaoperatorsdk.operator.processing.dependent.kubernetes; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,17 +10,20 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.apps.DaemonSet; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.apps.ReplicaSet; import io.fabric8.kubernetes.api.model.apps.StatefulSet; import io.javaoperatorsdk.operator.MockKubernetesClient; +import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.ReconcilerUtils; import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.api.reconciler.Context; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,6 +49,54 @@ void setup() { when(mockedContext.getControllerConfiguration()).thenReturn(controllerConfiguration); } + @Test + void noMatchWhenNoMatchingController() { + var desired = loadResource("nginx-deployment.yaml", Deployment.class); + var actual = + loadResource("deployment-with-managed-fields-additional-controller.yaml", Deployment.class); + actual + .getMetadata() + .getManagedFields() + .removeIf(managedFieldsEntry -> managedFieldsEntry.getManager().equals("controller")); + + assertThat(matcher.matches(actual, desired, mockedContext)).isFalse(); + } + + @Test + void exceptionWhenDuplicateController() { + var desired = loadResource("nginx-deployment.yaml", Deployment.class); + var actual = + loadResource("deployment-with-managed-fields-additional-controller.yaml", Deployment.class); + actual.getMetadata().getManagedFields().stream() + .filter(managedFieldsEntry -> managedFieldsEntry.getManager().equals("controller")) + .findFirst() + .ifPresent( + managedFieldsEntry -> actual.getMetadata().getManagedFields().add(managedFieldsEntry)); + + assertThatThrownBy(() -> matcher.matches(actual, desired, mockedContext)) + .isInstanceOf(OperatorException.class) + .hasMessage( + "More than one field manager exists with name: controller in resource: Deployment with" + + " name: test"); + } + + @Test + void matchWithSensitiveResource() { + var desired = loadResource("secret-desired.yaml", Secret.class); + var actual = loadResource("secret.yaml", Secret.class); + + assertThat(matcher.matches(actual, desired, mockedContext)).isTrue(); + } + + @Test + void noMatchWithSensitiveResource() { + var desired = loadResource("secret-desired.yaml", Secret.class); + var actual = loadResource("secret.yaml", Secret.class); + actual.getData().put("key1", "dmFsMg=="); + + assertThat(matcher.matches(actual, desired, mockedContext)).isFalse(); + } + @Test void checksIfAddsNotAddedByController() { var desired = loadResource("nginx-deployment.yaml", Deployment.class); @@ -56,7 +106,40 @@ void checksIfAddsNotAddedByController() { assertThat(matcher.matches(actual, desired, mockedContext)).isTrue(); } - // In the example the owner reference in a list is referenced by "k:", while all the fields are + @Test + void throwExceptionWhenManagedListEntryNotFound() { + var desired = loadResource("nginx-deployment.yaml", Deployment.class); + var actual = + loadResource("deployment-with-managed-fields-additional-controller.yaml", Deployment.class); + final var container = actual.getSpec().getTemplate().getSpec().getContainers().get(0); + container.setName("foobar"); + + assertThatThrownBy(() -> matcher.matches(actual, desired, mockedContext)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot find list element for key: {\"name\":\"nginx\"} in map: [[image," + + " imagePullPolicy, name, ports, resources, terminationMessagePath," + + " terminationMessagePolicy]]"); + } + + @Test + void throwExceptionWhenDuplicateManagedListEntryFound() { + var desired = loadResource("nginx-deployment.yaml", Deployment.class); + var actual = + loadResource("deployment-with-managed-fields-additional-controller.yaml", Deployment.class); + final var container = actual.getSpec().getTemplate().getSpec().getContainers().get(0); + actual.getSpec().getTemplate().getSpec().getContainers().add(container); + + assertThatThrownBy(() -> matcher.matches(actual, desired, mockedContext)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "More targets found in list element for key: {\"name\":\"nginx\"} in map: [[image," + + " imagePullPolicy, name, ports, resources, terminationMessagePath," + + " terminationMessagePolicy], [image, imagePullPolicy, name, ports, resources," + + " terminationMessagePath, terminationMessagePolicy]]"); + } + + // in the example the owner reference in a list is referenced by "k:", while all the fields are // managed but not listed @Test void emptyListElementMatchesAllFields() { @@ -118,45 +201,11 @@ void addedLabelInDesiredMakesMatchFail() { } @Test - @SuppressWarnings("unchecked") - void sortListItemsTest() { - var nestedMap1 = new HashMap(); - nestedMap1.put("z", 26); - nestedMap1.put("y", 25); - - var nestedMap2 = new HashMap(); - nestedMap2.put("b", 26); - nestedMap2.put("c", 25); - nestedMap2.put("a", 24); - - var unsortedListItems = List.of(1, nestedMap1, nestedMap2); - var sortedListItems = matcher.sortListItems(unsortedListItems); - assertThat(sortedListItems).element(0).isEqualTo(1); - - var sortedNestedMap1 = (Map) sortedListItems.get(1); - assertThat(sortedNestedMap1.keySet()).containsExactly("y", "z"); + void withFinalizer() { + var desired = loadResource("secret-with-finalizer-desired.yaml", Secret.class); + var actual = loadResource("secret-with-finalizer.yaml", Secret.class); - var sortedNestedMap2 = (Map) sortedListItems.get(2); - assertThat(sortedNestedMap2.keySet()).containsExactly("a", "b", "c"); - } - - @Test - @SuppressWarnings("unchecked") - void testSortMapWithNestedMap() { - var nestedMap = new HashMap(); - nestedMap.put("z", 26); - nestedMap.put("y", 25); - - var unsortedMap = new HashMap(); - unsortedMap.put("b", nestedMap); - unsortedMap.put("a", 1); - unsortedMap.put("c", 2); - - var sortedMap = matcher.sortMap(unsortedMap); - assertThat(sortedMap.keySet()).containsExactly("a", "b", "c"); - - var sortedNestedMap = (Map) sortedMap.get("b"); - assertThat(sortedNestedMap.keySet()).containsExactly("y", "z"); + assertThat(matcher.matches(actual, desired, mockedContext)).isTrue(); } @ParameterizedTest @@ -205,6 +254,23 @@ void testSanitizeState_statefulSetWithResources_withMismatch() { assertThat(matcher.matches(actualStatefulSet, desiredStatefulSet, mockedContext)).isFalse(); } + @Test + void testSanitizeState_statefulSet_withResourceTypeMismatch() { + var desiredReplicaSet = loadResource("sample-rs-resources-desired.yaml", ReplicaSet.class); + var actualStatefulSet = loadResource("sample-sts-resources.yaml", StatefulSet.class); + + assertThat(matcher.matches(actualStatefulSet, desiredReplicaSet, mockedContext)).isFalse(); + } + + @Test + void testSanitizeState_deployment_withResourceTypeMismatch() { + var desiredReplicaSet = loadResource("sample-rs-resources-desired.yaml", ReplicaSet.class); + var actualDeployment = + loadResource("deployment-with-managed-fields-additional-controller.yaml", Deployment.class); + + assertThat(matcher.matches(actualDeployment, desiredReplicaSet, mockedContext)).isFalse(); + } + @Test void testSanitizeState_replicaSetWithResources() { var desiredReplicaSet = loadResource("sample-rs-resources-desired.yaml", ReplicaSet.class); @@ -222,6 +288,14 @@ void testSanitizeState_replicaSetWithResources_withMismatch() { assertThat(matcher.matches(actualReplicaSet, desiredReplicaSet, mockedContext)).isFalse(); } + @Test + void testSanitizeState_replicaSet_withResourceTypeMismatch() { + var desiredDaemonSet = loadResource("sample-ds-resources-desired.yaml", DaemonSet.class); + var actualReplicaSet = loadResource("sample-rs-resources.yaml", ReplicaSet.class); + + assertThat(matcher.matches(actualReplicaSet, desiredDaemonSet, mockedContext)).isFalse(); + } + @Test void testSanitizeState_daemonSetWithResources() { var desiredDaemonSet = loadResource("sample-ds-resources-desired.yaml", DaemonSet.class); @@ -238,6 +312,14 @@ void testSanitizeState_daemonSetWithResources_withMismatch() { assertThat(matcher.matches(actualDaemonSet, desiredDaemonSet, mockedContext)).isFalse(); } + @Test + void testSanitizeState_daemonSet_withResourceTypeMismatch() { + var desiredReplicaSet = loadResource("sample-rs-resources-desired.yaml", ReplicaSet.class); + var actualDaemonSet = loadResource("sample-ds-resources.yaml", DaemonSet.class); + + assertThat(matcher.matches(actualDaemonSet, desiredReplicaSet, mockedContext)).isFalse(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void testCustomMatcher_returnsExpectedMatchBasedOnReadOnlyLabel(boolean readOnly) { @@ -263,6 +345,52 @@ void testCustomMatcher_returnsExpectedMatchBasedOnReadOnlyLabel(boolean readOnly .isEqualTo(readOnly); } + @Test + void keepOnlyManagedFields_withInvalidManagedFieldsKey() { + assertThatThrownBy( + () -> + SSABasedGenericKubernetesResourceMatcher.keepOnlyManagedFields( + Map.of(), + Map.of(), + Map.of("invalid", 1), + mockedContext.getClient().getKubernetesSerialization())) // + .isInstanceOf(IllegalStateException.class) // + .hasMessage("Key: invalid has no prefix: f:"); + } + + @Test + @SuppressWarnings("unchecked") + void testSortMap() { + final var unsortedMap = Map.of("b", Map.of("z", 26, "y", 25), "a", List.of("w", "v"), "c", 2); + + var sortedMap = SSABasedGenericKubernetesResourceMatcher.sortMap(unsortedMap); + assertThat(sortedMap.keySet()).containsExactly("a", "b", "c"); + + var sortedNestedMap = (Map) sortedMap.get("b"); + assertThat(sortedNestedMap.keySet()).containsExactly("y", "z"); + } + + @Test + @SuppressWarnings("unchecked") + void testSortListItems() { + final var unsortedList = + List.of(1, Map.of("z", 26, "y", 25), Map.of("b", 26, "c", 25, "a", 24), List.of("w", "v")); + + var sortedListItems = SSABasedGenericKubernetesResourceMatcher.sortListItems(unsortedList); + assertThat(sortedListItems).element(0).isEqualTo(1); + + var sortedNestedMap1 = (Map) sortedListItems.get(1); + assertThat(sortedNestedMap1.keySet()).containsExactly("y", "z"); + + var sortedNestedMap2 = (Map) sortedListItems.get(2); + assertThat(sortedNestedMap2.keySet()).containsExactly("a", "b", "c"); + } + + private static R loadResource(String fileName, Class clazz) { + return ReconcilerUtils.loadYaml( + clazz, SSABasedGenericKubernetesResourceMatcherTest.class, fileName); + } + private static class ConfigMapDR extends KubernetesDependentResource { public ConfigMapDR() { super(ConfigMap.class); @@ -285,9 +413,4 @@ protected boolean matches( return actualMap.equals(desiredMap); } } - - private static R loadResource(String fileName, Class clazz) { - return ReconcilerUtils.loadYaml( - clazz, SSABasedGenericKubernetesResourceMatcherTest.class, fileName); - } } diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/configmap.empty-owner-reference-desired.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/configmap.empty-owner-reference-desired.yaml index 3a9d018266..01d27e39b3 100644 --- a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/configmap.empty-owner-reference-desired.yaml +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/configmap.empty-owner-reference-desired.yaml @@ -10,5 +10,3 @@ metadata: uid: 1ef74cb4-dbbd-45ef-9caf-aa76186594ea data: key1: "val1" - - diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/deployment-with-managed-fields-additional-controller.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/deployment-with-managed-fields-additional-controller.yaml index d82b5c8933..38358a16c0 100644 --- a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/deployment-with-managed-fields-additional-controller.yaml +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/deployment-with-managed-fields-additional-controller.yaml @@ -25,6 +25,7 @@ metadata: f:image: {} f:name: {} f:ports: + .: {} k:{"containerPort":80,"protocol":"TCP"}: .: {} f:containerPort: {} diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod-desired.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod-desired.yaml index 92ece6df00..e400532fad 100644 --- a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod-desired.yaml +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod-desired.yaml @@ -18,4 +18,4 @@ spec: - name: shared-data mountPath: /data command: ["/bin/sh"] - args: ["-c", "echo Level Up Blue Team! > /data/index.html"] \ No newline at end of file + args: ["-c", "echo Level Up Blue Team! > /data/index.html"] diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod.yaml index e1334117b6..6a5f2d82b4 100644 --- a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod.yaml +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/multi-container-pod.yaml @@ -211,4 +211,4 @@ status: podIPs: - ip: 10.244.0.3 qosClass: BestEffort - startTime: "2023-06-08T11:50:59Z" \ No newline at end of file + startTime: "2023-06-08T11:50:59Z" diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-desired.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-desired.yaml new file mode 100644 index 0000000000..29f9866592 --- /dev/null +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-desired.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: test1 + namespace: default +data: + key1: "dmFsMQ==" diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer-desired.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer-desired.yaml new file mode 100644 index 0000000000..f0e64b1a60 --- /dev/null +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer-desired.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + finalizers: + - test-finalizer + name: test1 + namespace: default +data: + key1: "dmFsMQ==" diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer.yaml new file mode 100644 index 0000000000..fa9ffc13a0 --- /dev/null +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret-with-finalizer.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +data: + key1: "dmFsMQ==" +kind: Secret +metadata: + creationTimestamp: "2023-06-07T11:08:34Z" + finalizers: + - test-finalizer + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key1: {} + f:metadata: + f:finalizers: + .: {} + v:"test-finalizer": {} + manager: controller + operation: Apply + time: "2023-06-07T11:08:34Z" + name: test1 + namespace: default + resourceVersion: "400" + uid: 1d47f98f-ff1e-46d8-bbb5-6658ec488ae2 diff --git a/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret.yaml b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret.yaml new file mode 100644 index 0000000000..a6dc3b3c3e --- /dev/null +++ b/operator-framework-core/src/test/resources/io/javaoperatorsdk/operator/processing/dependent/kubernetes/secret.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +data: + key1: "dmFsMQ==" +kind: Secret +metadata: + creationTimestamp: "2023-06-07T11:08:34Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key1: {} + manager: controller + operation: Apply + time: "2023-06-07T11:08:34Z" + name: test1 + namespace: default + resourceVersion: "400" + uid: 1d47f98f-ff1e-46d8-bbb5-6658ec488ae2 From 63bbec523154a0eae0da7f883ae5a5c545234473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 08:16:30 +0200 Subject: [PATCH 34/45] chore(deps): bump io.micrometer:micrometer-core from 1.14.6 to 1.15.0 (#2793) Bumps [io.micrometer:micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.14.6 to 1.15.0. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.14.6...v1.15.0) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-version: 1.15.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f41d27848..3daa583203 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 3.27.3 4.3.0 2.7.3 - 1.14.6 + 1.15.0 3.2.0 0.9.14 2.19.0 From b66edb1b458231ba26dc3d1016eed5167745ca56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 15 May 2025 17:45:19 +0200 Subject: [PATCH 35/45] chore: fabric8 client to v7.3 (#2801) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3daa583203..132f51e312 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ jdk 5.12.2 - 7.2.0 + 7.3.0 2.0.12 2.24.3 5.17.0 From 937a9a983890d7f956d8c9c2698e5f376af05fda Mon Sep 17 00:00:00 2001 From: Martin Stefanko Date: Thu, 15 May 2025 18:43:11 +0200 Subject: [PATCH 36/45] feat: automatically derive the the dependent resource type if not specified (#2772) --- .../templates/ConfigMapDependentResource.java | 4 - .../dependent-resources.md | 4 - .../operator/api/config/Utils.java | 33 ++++++++ ...actEventSourceHolderDependentResource.java | 12 ++- .../AbstractExternalDependentResource.java | 2 + .../AbstractPollingDependentResource.java | 2 + .../PerResourcePollingDependentResource.java | 2 + .../CRUDKubernetesDependentResource.java | 2 + .../CRUDNoGCKubernetesDependentResource.java | 2 + .../KubernetesDependentResource.java | 2 + .../ControllerConfigurationOverriderTest.java | 25 +----- .../operator/api/config/UtilsTest.java | 7 +- ...dentResourceConfigurationResolverTest.java | 14 +--- .../GenericKubernetesResourceMatcherTest.java | 4 - .../KubernetesDependentResourceTest.java | 76 +++++++++++++++++++ .../UnmodifiablePartConfigMapDependent.java | 4 - .../config/BaseConfigurationServiceTest.java | 19 +---- ...ConfigMapDeleterBulkDependentResource.java | 4 - .../ReadOnlyBulkDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - ...ntAnnotationSecondaryMapperReconciler.java | 4 - ...stomMappingConfigMapDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../FilteredDependentConfigMap.java | 4 - .../ConfigMapDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../DependentResourceCrossRefReconciler.java | 8 -- .../dependentssa/SSAConfigMapDependent.java | 4 - .../ConfigMapDependentResource.java | 4 - ...endentGarbageCollectionTestReconciler.java | 4 - ...endentResourceMultiInformerConfigMap1.java | 4 - ...endentResourceMultiInformerConfigMap2.java | 4 - ...gedDependentNoDiscriminatorConfigMap1.java | 4 - ...gedDependentNoDiscriminatorConfigMap2.java | 4 - ...pleManagedDependentResourceConfigMap1.java | 4 - ...pleManagedDependentResourceConfigMap2.java | 4 - .../MultipleOwnerDependentConfigMap.java | 4 - ...DependentPrimaryIndexerTestReconciler.java | 4 - .../ConfigMapDependent.java | 4 - .../SecretDependent.java | 4 - .../dependent/readonly/ReadOnlyDependent.java | 7 +- .../restart/ConfigMapDependentResource.java | 4 - .../ServiceDependentResource.java | 4 - .../ServiceAccountDependentResource.java | 4 - .../ServiceDependentResource.java | 4 - .../StandaloneDependentTestReconciler.java | 4 - ...lSetDesiredSanitizerDependentResource.java | 4 - .../CRDPresentActivationDependent.java | 4 - .../ConfigMapDependentResource.java | 4 - .../RouteDependentResource.java | 4 - .../ConfigMapDependent.java | 4 - .../SecretDependent.java | 4 - .../ConfigMapDependentResource1.java | 4 - .../ConfigMapDependentResource2.java | 4 - .../SecretDependentResource.java | 4 - .../ConfigMapDependentResource1.java | 4 - .../ConfigMapDependentResource2.java | 4 - .../ConfigMapDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../RouteDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../DeploymentDependentResource.java | 4 - .../ConfigMapDependent.java | 4 - .../ConfigMapDependent.java | 4 - .../ConfigMapDependentResource.java | 4 - .../SecretDependentResource.java | 4 - .../ConfigMapDependent.java | 4 - .../dependent/SchemaDependentResource.java | 4 - .../dependent/SecretDependentResource.java | 4 - .../sample/DeploymentDependentResource.java | 4 - .../sample/ServiceDependentResource.java | 4 - .../ConfigMapDependentResource.java | 4 - .../DeploymentDependentResource.java | 4 - .../IngressDependentResource.java | 4 - .../ServiceDependentResource.java | 4 - 76 files changed, 142 insertions(+), 315 deletions(-) create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceTest.java diff --git a/bootstrapper-maven-plugin/src/main/resources/templates/ConfigMapDependentResource.java b/bootstrapper-maven-plugin/src/main/resources/templates/ConfigMapDependentResource.java index a8d43c60db..59eae8b01c 100644 --- a/bootstrapper-maven-plugin/src/main/resources/templates/ConfigMapDependentResource.java +++ b/bootstrapper-maven-plugin/src/main/resources/templates/ConfigMapDependentResource.java @@ -17,10 +17,6 @@ public class ConfigMapDependentResource public static final String KEY = "key"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired({{artifactClassId}}CustomResource primary, Context<{{artifactClassId}}CustomResource> context) { diff --git a/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md b/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md index b9fcb7acf5..304e20bafe 100644 --- a/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md +++ b/docs/content/en/docs/documentation/dependent-resource-and-workflows/dependent-resources.md @@ -136,10 +136,6 @@ Deleted (or set to be garbage collected). The following example shows how to cre @KubernetesDependent(labelSelector = WebPageManagedDependentsReconciler.SELECTOR) class DeploymentDependentResource extends CRUDKubernetesDependentResource { - public DeploymentDependentResource() { - super(Deployment.class); - } - @Override protected Deployment desired(WebPage webPage, Context context) { var deploymentName = deploymentName(webPage); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java index f11fc47eef..3b6f94a025 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java @@ -134,6 +134,39 @@ public static Class getTypeArgumentFromExtendedClassByIndex(Class clazz, i } } + public static Class getTypeArgumentFromHierarchyByIndex(Class clazz, int index) { + return getTypeArgumentFromHierarchyByIndex(clazz, null, index); + } + + public static Class getTypeArgumentFromHierarchyByIndex( + Class clazz, Class expectedImplementedInterface, int index) { + Class c = clazz; + while (!(c.getGenericSuperclass() instanceof ParameterizedType)) { + c = c.getSuperclass(); + } + Class actualTypeArgument = + (Class) ((ParameterizedType) c.getGenericSuperclass()).getActualTypeArguments()[index]; + if (expectedImplementedInterface != null + && !expectedImplementedInterface.isAssignableFrom(actualTypeArgument)) { + throw new IllegalArgumentException( + GENERIC_PARAMETER_TYPE_ERROR_PREFIX + + clazz.getName() + + "because it doesn't extend a class that is parametrized with the type that" + + " implements " + + expectedImplementedInterface.getSimpleName() + + ". Please provide the resource type in the constructor (e.g.," + + " super(Deployment.class)."); + } else if (expectedImplementedInterface == null && actualTypeArgument.equals(Object.class)) { + throw new IllegalArgumentException( + GENERIC_PARAMETER_TYPE_ERROR_PREFIX + + clazz.getName() + + " because it doesn't extend a class that is parametrized with the type we want to" + + " retrieve or because it's Object.class. Please provide the resource type in the " + + "constructor (e.g., super(Deployment.class)."); + } + return actualTypeArgument; + } + public static Class getFirstTypeArgumentFromInterface( Class clazz, Class expectedImplementedInterface) { return getTypeArgumentFromInterfaceByIndex(clazz, expectedImplementedInterface, 0); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractEventSourceHolderDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractEventSourceHolderDependentResource.java index 5cee9467f1..7f2674892f 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractEventSourceHolderDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractEventSourceHolderDependentResource.java @@ -3,6 +3,7 @@ import java.util.Optional; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.api.reconciler.Ignore; @@ -23,13 +24,22 @@ public abstract class AbstractEventSourceHolderDependentResource< private boolean isCacheFillerEventSource; protected String eventSourceNameToUse; + @SuppressWarnings("unchecked") + protected AbstractEventSourceHolderDependentResource() { + this(null, null); + } + protected AbstractEventSourceHolderDependentResource(Class resourceType) { this(resourceType, null); } protected AbstractEventSourceHolderDependentResource(Class resourceType, String name) { super(name); - this.resourceType = resourceType; + if (resourceType == null) { + this.resourceType = (Class) Utils.getTypeArgumentFromHierarchyByIndex(getClass(), 0); + } else { + this.resourceType = resourceType; + } } /** diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractExternalDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractExternalDependentResource.java index 1148895709..4c828b7eb9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractExternalDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/AbstractExternalDependentResource.java @@ -21,6 +21,8 @@ public abstract class AbstractExternalDependentResource< private InformerEventSource externalStateEventSource; + protected AbstractExternalDependentResource() {} + @SuppressWarnings("unchecked") protected AbstractExternalDependentResource(Class resourceType) { super(resourceType); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/AbstractPollingDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/AbstractPollingDependentResource.java index 659b8b4720..3cf93cba53 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/AbstractPollingDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/AbstractPollingDependentResource.java @@ -16,6 +16,8 @@ public abstract class AbstractPollingDependentResource public static final Duration DEFAULT_POLLING_PERIOD = Duration.ofMillis(5000); private Duration pollingPeriod; + protected AbstractPollingDependentResource() {} + protected AbstractPollingDependentResource(Class resourceType) { this(resourceType, DEFAULT_POLLING_PERIOD); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PerResourcePollingDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PerResourcePollingDependentResource.java index 8cbe9f48d5..c0181207d8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PerResourcePollingDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PerResourcePollingDependentResource.java @@ -14,6 +14,8 @@ public abstract class PerResourcePollingDependentResource implements PerResourcePollingEventSource.ResourceFetcher { + public PerResourcePollingDependentResource() {} + public PerResourcePollingDependentResource(Class resourceType) { super(resourceType); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDKubernetesDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDKubernetesDependentResource.java index afe4302fc3..392ac6d894 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDKubernetesDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDKubernetesDependentResource.java @@ -18,6 +18,8 @@ public abstract class CRUDKubernetesDependentResource implements Creator, Updater, GarbageCollected

{ + public CRUDKubernetesDependentResource() {} + public CRUDKubernetesDependentResource(Class resourceType) { super(resourceType); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDNoGCKubernetesDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDNoGCKubernetesDependentResource.java index 549f26437a..3b3c11b006 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDNoGCKubernetesDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/CRUDNoGCKubernetesDependentResource.java @@ -20,6 +20,8 @@ public class CRUDNoGCKubernetesDependentResource extends KubernetesDependentResource implements Creator, Updater, Deleter

{ + public CRUDNoGCKubernetesDependentResource() {} + public CRUDNoGCKubernetesDependentResource(Class resourceType) { super(resourceType); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java index ea7edbc1a0..ab6e4eaca4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java @@ -42,6 +42,8 @@ public abstract class KubernetesDependentResource kubernetesDependentResourceConfig; private volatile Boolean useSSA; + public KubernetesDependentResource() {} + public KubernetesDependentResource(Class resourceType) { this(resourceType, null); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java index 33191a8141..49d0b76017 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java @@ -359,21 +359,11 @@ public UpdateControl reconcile(ConfigMap resource, Context } public static class ReadOnlyDependent extends KubernetesDependentResource - implements GarbageCollected { - - public ReadOnlyDependent() { - super(ConfigMap.class); - } - } + implements GarbageCollected {} @KubernetesDependent(informer = @Informer(namespaces = Constants.WATCH_ALL_NAMESPACES)) public static class WatchAllNSDependent extends KubernetesDependentResource - implements GarbageCollected { - - public WatchAllNSDependent() { - super(ConfigMap.class); - } - } + implements GarbageCollected {} @Workflow(dependents = @Dependent(type = OverriddenNSDependent.class)) @ControllerConfiguration( @@ -394,10 +384,6 @@ public static class OverriddenNSDependent implements GarbageCollected { private static final String DEP_NS = "dependentNS"; - - public OverriddenNSDependent() { - super(ConfigMap.class); - } } @Workflow( @@ -415,12 +401,7 @@ public UpdateControl reconcile(ConfigMap resource, Context private static class NamedDependentResource extends KubernetesDependentResource - implements GarbageCollected { - - public NamedDependentResource() { - super(ConfigMap.class); - } - } + implements GarbageCollected {} private static class ExternalDependentResource implements DependentResource, diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/UtilsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/UtilsTest.java index 2b75b399c2..a2246f018a 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/UtilsTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/UtilsTest.java @@ -118,10 +118,5 @@ public UpdateControl reconcile(ConfigMap resource, Context } public static class TestKubernetesDependentResource - extends KubernetesDependentResource { - - public TestKubernetesDependentResource() { - super(Deployment.class); - } - } + extends KubernetesDependentResource {} } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationResolverTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationResolverTest.java index dd3caf0bd0..27bd2b9dae 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationResolverTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationResolverTest.java @@ -144,20 +144,10 @@ public UpdateControl reconcile(ConfigMap resource, Context } public static class ConfigMapDep extends KubernetesDependentResource - implements GarbageCollected { - - public ConfigMapDep() { - super(ConfigMap.class); - } - } + implements GarbageCollected {} public static class ServiceDep extends KubernetesDependentResource - implements GarbageCollected { - - public ServiceDep() { - super(Service.class); - } - } + implements GarbageCollected {} @CustomAnnotation(value = CustomAnnotatedDep.PROVIDED_VALUE) @Configured( diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/GenericKubernetesResourceMatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/GenericKubernetesResourceMatcherTest.java index 0d85ee7225..3062e360e2 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/GenericKubernetesResourceMatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/GenericKubernetesResourceMatcherTest.java @@ -186,10 +186,6 @@ HasMetadata createPrimary(String caseName) { private static class ServiceAccountDR extends KubernetesDependentResource { - public ServiceAccountDR() { - super(ServiceAccount.class); - } - @Override protected ServiceAccount desired(HasMetadata primary, Context context) { return new ServiceAccountBuilder() diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceTest.java new file mode 100644 index 0000000000..6c48311f4b --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceTest.java @@ -0,0 +1,76 @@ +package io.javaoperatorsdk.operator.processing.dependent.kubernetes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.javaoperatorsdk.operator.sample.simple.TestCustomResource; + +import static io.javaoperatorsdk.operator.api.config.Utils.GENERIC_PARAMETER_TYPE_ERROR_PREFIX; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KubernetesDependentResourceTest { + + @ParameterizedTest + @ValueSource( + classes = { + TestDeploymentDependentResource.class, + ChildTestDeploymentDependentResource.class, + GrandChildTestDeploymentDependentResource.class, + ChildTypeWithValidKubernetesDependentResource.class, + ConstructorOverridedCorrectDeployementDependentResource.class + }) + void checkResourceTypeDerivationWithInheritance(Class clazz) throws Exception { + KubernetesDependentResource dependentResource = + (KubernetesDependentResource) clazz.getDeclaredConstructor().newInstance(); + assertThat(dependentResource).isInstanceOf(KubernetesDependentResource.class); + assertThat(dependentResource.resourceType()).isEqualTo(Deployment.class); + } + + private static class TestDeploymentDependentResource + extends KubernetesDependentResource {} + + private static class ChildTestDeploymentDependentResource + extends TestDeploymentDependentResource {} + + private static class GrandChildTestDeploymentDependentResource + extends ChildTestDeploymentDependentResource {} + + private static class ChildTypeWithValidKubernetesDependentResource + extends KubernetesDependentResource {} + + private static class ConstructorOverridedCorrectDeployementDependentResource + extends KubernetesDependentResource { + public ConstructorOverridedCorrectDeployementDependentResource() { + super(Deployment.class); + } + } + + @Test + void validateInvalidTypeDerivationTypesThrowException() { + assertThatThrownBy(() -> new InvalidChildTestDeploymentDependentResource()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + GENERIC_PARAMETER_TYPE_ERROR_PREFIX + + InvalidChildTestDeploymentDependentResource.class.getName() + + " because it doesn't extend a class that is parametrized with the type we want to" + + " retrieve or because it's Object.class. Please provide the resource type in the " + + "constructor (e.g., super(Deployment.class)."); + assertThatThrownBy(() -> new InvalidGrandChildTestDeploymentDependentResource()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + GENERIC_PARAMETER_TYPE_ERROR_PREFIX + + InvalidGrandChildTestDeploymentDependentResource.class.getName() + + " because it doesn't extend a class that is parametrized with the type we want to" + + " retrieve or because it's Object.class. Please provide the resource type in the " + + "constructor (e.g., super(Deployment.class)."); + } + + private static class InvalidChildTestDeploymentDependentResource + extends ChildTypeWithValidKubernetesDependentResource {} + + private static class InvalidGrandChildTestDeploymentDependentResource + extends InvalidChildTestDeploymentDependentResource {} +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/unmodifiabledependentpart/UnmodifiablePartConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/unmodifiabledependentpart/UnmodifiablePartConfigMapDependent.java index a4559055a4..c6f0759410 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/unmodifiabledependentpart/UnmodifiablePartConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/unmodifiabledependentpart/UnmodifiablePartConfigMapDependent.java @@ -14,10 +14,6 @@ public class UnmodifiablePartConfigMapDependent public static final String UNMODIFIABLE_INITIAL_DATA_KEY = "initialDataKey"; public static final String ACTUAL_DATA_KEY = "actualDataKey"; - public UnmodifiablePartConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( UnmodifiableDependentPartCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/BaseConfigurationServiceTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/BaseConfigurationServiceTest.java index 2f202436ff..25926e6405 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/BaseConfigurationServiceTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/BaseConfigurationServiceTest.java @@ -414,12 +414,7 @@ public UpdateControl reconcile( @KubernetesDependent(useSSA = BooleanWithUndefined.TRUE) public static class WithAnnotation - extends CRUDKubernetesDependentResource { - - public WithAnnotation() { - super(ConfigMap.class); - } - } + extends CRUDKubernetesDependentResource {} } public static class MissingAnnotationReconciler implements Reconciler { @@ -443,18 +438,10 @@ public UpdateControl reconcile(ConfigMap resource, Context } private static class DefaultDependent - extends KubernetesDependentResource { - public DefaultDependent() { - super(ConfigMapReader.class); - } - } + extends KubernetesDependentResource {} @KubernetesDependent(useSSA = BooleanWithUndefined.FALSE) - private static class NonSSADependent extends KubernetesDependentResource { - public NonSSADependent() { - super(Service.class); - } - } + private static class NonSSADependent extends KubernetesDependentResource {} } public static class TestRetry implements Retry, AnnotationConfigurable { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/ConfigMapDeleterBulkDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/ConfigMapDeleterBulkDependentResource.java index a9163a4ec3..cf3c96b82a 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/ConfigMapDeleterBulkDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/ConfigMapDeleterBulkDependentResource.java @@ -21,10 +21,6 @@ public class ConfigMapDeleterBulkDependentResource public static final String ADDITIONAL_DATA_KEY = "additionalData"; public static final String INDEX_DELIMITER = "-"; - public ConfigMapDeleterBulkDependentResource() { - super(ConfigMap.class); - } - @Override public Map desiredResources( BulkDependentTestCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/readonly/ReadOnlyBulkDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/readonly/ReadOnlyBulkDependentResource.java index b61ad7c230..1eab400888 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/readonly/ReadOnlyBulkDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/bulkdependent/readonly/ReadOnlyBulkDependentResource.java @@ -23,10 +23,6 @@ public class ReadOnlyBulkDependentResource public static final String INDEX_DELIMITER = "-"; - public ReadOnlyBulkDependentResource() { - super(ConfigMap.class); - } - @Override public Map getSecondaryResources( BulkDependentTestCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/cleanermanageddependent/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/cleanermanageddependent/ConfigMapDependentResource.java index dfb3cf2ee2..3c94775045 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/cleanermanageddependent/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/cleanermanageddependent/ConfigMapDependentResource.java @@ -19,10 +19,6 @@ public class ConfigMapDependentResource private static final AtomicInteger numberOfCleanupExecutions = new AtomicInteger(0); - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( CleanerForManagedDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/createonlyifnotexistsdependentwithssa/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/createonlyifnotexistsdependentwithssa/ConfigMapDependentResource.java index 14ba61513a..67532ee159 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/createonlyifnotexistsdependentwithssa/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/createonlyifnotexistsdependentwithssa/ConfigMapDependentResource.java @@ -11,10 +11,6 @@ public class ConfigMapDependentResource extends CRUDKubernetesDependentResource< ConfigMap, CreateOnlyIfNotExistingDependentWithSSACustomResource> { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( CreateOnlyIfNotExistingDependentWithSSACustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentannotationsecondarymapper/DependentAnnotationSecondaryMapperReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentannotationsecondarymapper/DependentAnnotationSecondaryMapperReconciler.java index 2ba4ee5ef0..71146df638 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentannotationsecondarymapper/DependentAnnotationSecondaryMapperReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentannotationsecondarymapper/DependentAnnotationSecondaryMapperReconciler.java @@ -41,10 +41,6 @@ public static class ConfigMapDependentResource Updater, Deleter { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentAnnotationSecondaryMapperResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentcustommappingannotation/CustomMappingConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentcustommappingannotation/CustomMappingConfigMapDependentResource.java index 0a3aeba0e1..081cf31dbd 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentcustommappingannotation/CustomMappingConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentcustommappingannotation/CustomMappingConfigMapDependentResource.java @@ -30,10 +30,6 @@ public class CustomMappingConfigMapDependentResource CUSTOM_TYPE_KEY, DependentCustomMappingCustomResource.class); - public CustomMappingConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentCustomMappingCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentdifferentnamespace/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentdifferentnamespace/ConfigMapDependentResource.java index f4dc408825..30e0de5b7d 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentdifferentnamespace/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentdifferentnamespace/ConfigMapDependentResource.java @@ -15,10 +15,6 @@ public class ConfigMapDependentResource public static final String NAMESPACE = "default"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentDifferentNamespaceCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentfilter/FilteredDependentConfigMap.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentfilter/FilteredDependentConfigMap.java index 3ba4df63f4..3b12673b4c 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentfilter/FilteredDependentConfigMap.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentfilter/FilteredDependentConfigMap.java @@ -15,10 +15,6 @@ public class FilteredDependentConfigMap extends CRUDKubernetesDependentResource { - public FilteredDependentConfigMap() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentFilterTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentoperationeventfiltering/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentoperationeventfiltering/ConfigMapDependentResource.java index 13879227e1..87b827c527 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentoperationeventfiltering/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentoperationeventfiltering/ConfigMapDependentResource.java @@ -13,10 +13,6 @@ public class ConfigMapDependentResource public static final String KEY = "key1"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentOperationEventFilterCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentreinitialization/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentreinitialization/ConfigMapDependentResource.java index 704febbf67..2a245a3721 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentreinitialization/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentreinitialization/ConfigMapDependentResource.java @@ -11,10 +11,6 @@ public class ConfigMapDependentResource extends CRUDKubernetesDependentResource { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentReInitializationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentresourcecrossref/DependentResourceCrossRefReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentresourcecrossref/DependentResourceCrossRefReconciler.java index 51a285aa4b..5d54ecdabe 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentresourcecrossref/DependentResourceCrossRefReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentresourcecrossref/DependentResourceCrossRefReconciler.java @@ -59,10 +59,6 @@ public boolean isErrorHappened() { public static class SecretDependentResource extends CRUDKubernetesDependentResource { - public SecretDependentResource() { - super(Secret.class); - } - @Override protected Secret desired( DependentResourceCrossRefResource primary, @@ -81,10 +77,6 @@ protected Secret desired( public static class ConfigMapDependentResource extends CRUDKubernetesDependentResource { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentResourceCrossRefResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentssa/SSAConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentssa/SSAConfigMapDependent.java index dc47f1f8df..49d2c1de44 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentssa/SSAConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/dependentssa/SSAConfigMapDependent.java @@ -16,10 +16,6 @@ public class SSAConfigMapDependent public static final String DATA_KEY = "key1"; - public SSAConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentSSACustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/informerrelatedbehavior/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/informerrelatedbehavior/ConfigMapDependentResource.java index 57427a3537..348921cd93 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/informerrelatedbehavior/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/informerrelatedbehavior/ConfigMapDependentResource.java @@ -16,10 +16,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "key"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( InformerRelatedBehaviorTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/kubernetesdependentgarbagecollection/DependentGarbageCollectionTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/kubernetesdependentgarbagecollection/DependentGarbageCollectionTestReconciler.java index 36af4fadb4..880aaa6884 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/kubernetesdependentgarbagecollection/DependentGarbageCollectionTestReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/kubernetesdependentgarbagecollection/DependentGarbageCollectionTestReconciler.java @@ -70,10 +70,6 @@ private static class ConfigMapDependentResource Updater, GarbageCollected { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( DependentGarbageCollectionTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap1.java index 2ea2b1daba..855944ef98 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap1.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap1.java @@ -17,10 +17,6 @@ public class MultipleManagedDependentResourceMultiInformerConfigMap1 public static final String NAME_SUFFIX = "-1"; - public MultipleManagedDependentResourceMultiInformerConfigMap1() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleManagedDependentResourceMultiInformerCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap2.java index dbc0934ada..7b28b322b7 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap2.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledependentsametypemultiinformer/MultipleManagedDependentResourceMultiInformerConfigMap2.java @@ -18,10 +18,6 @@ public class MultipleManagedDependentResourceMultiInformerConfigMap2 public static final String NAME_SUFFIX = "-2"; - public MultipleManagedDependentResourceMultiInformerConfigMap2() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleManagedDependentResourceMultiInformerCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap1.java index 94584e8172..2fb65c6dde 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap1.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap1.java @@ -17,10 +17,6 @@ public class MultipleManagedDependentNoDiscriminatorConfigMap1 public static final String NAME_SUFFIX = "-1"; - public MultipleManagedDependentNoDiscriminatorConfigMap1() { - super(ConfigMap.class); - } - /* * Showcases optimization to avoid computing the whole desired state by providing the ResourceID * of the target resource. In this particular case this would not be necessary, since desired diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap2.java index 8836badb1f..4455ef5d9b 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap2.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipledrsametypenodiscriminator/MultipleManagedDependentNoDiscriminatorConfigMap2.java @@ -18,10 +18,6 @@ public class MultipleManagedDependentNoDiscriminatorConfigMap2 public static final String NAME_SUFFIX = "-2"; - public MultipleManagedDependentNoDiscriminatorConfigMap2() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleManagedDependentNoDiscriminatorCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap1.java index 38e2acc050..687bfcb5ac 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap1.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap1.java @@ -16,10 +16,6 @@ public class MultipleManagedDependentResourceConfigMap1 public static final String NAME_SUFFIX = "-1"; - public MultipleManagedDependentResourceConfigMap1() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleManagedDependentResourceCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap2.java index 95dcc66490..1a1d6b51c0 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap2.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanageddependentsametype/MultipleManagedDependentResourceConfigMap2.java @@ -16,10 +16,6 @@ public class MultipleManagedDependentResourceConfigMap2 public static final String NAME_SUFFIX = "-2"; - public MultipleManagedDependentResourceConfigMap2() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleManagedDependentResourceCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipleupdateondependent/MultipleOwnerDependentConfigMap.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipleupdateondependent/MultipleOwnerDependentConfigMap.java index 781451bba0..28ddfcc907 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipleupdateondependent/MultipleOwnerDependentConfigMap.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multipleupdateondependent/MultipleOwnerDependentConfigMap.java @@ -19,10 +19,6 @@ public class MultipleOwnerDependentConfigMap public static final String RESOURCE_NAME = "test1"; - public MultipleOwnerDependentConfigMap() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleOwnerDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primaryindexer/DependentPrimaryIndexerTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primaryindexer/DependentPrimaryIndexerTestReconciler.java index a7e72e592b..52094972da 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primaryindexer/DependentPrimaryIndexerTestReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primaryindexer/DependentPrimaryIndexerTestReconciler.java @@ -61,10 +61,6 @@ public List> prepareEventSource public static class ReadOnlyConfigMapDependent extends KubernetesDependentResource { - public ReadOnlyConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( PrimaryIndexerTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/ConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/ConfigMapDependent.java index 806b87dfa1..2b63bbf6b1 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/ConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/ConfigMapDependent.java @@ -11,10 +11,6 @@ public class ConfigMapDependent public static final String TEST_CONFIG_MAP_NAME = "testconfigmap"; - public ConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( PrimaryToSecondaryDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/SecretDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/SecretDependent.java index c3a4f5b77b..6371f453d7 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/SecretDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/primarytosecondaydependent/SecretDependent.java @@ -13,10 +13,6 @@ public class SecretDependent extends CRUDKubernetesDependentResource { - public SecretDependent() { - super(Secret.class); - } - @Override protected Secret desired( PrimaryToSecondaryDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/readonly/ReadOnlyDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/readonly/ReadOnlyDependent.java index 63e43fef95..a6f0662948 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/readonly/ReadOnlyDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/readonly/ReadOnlyDependent.java @@ -5,9 +5,4 @@ import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource; @KubernetesDependent -public class ReadOnlyDependent extends KubernetesDependentResource { - - public ReadOnlyDependent() { - super(ConfigMap.class); - } -} +public class ReadOnlyDependent extends KubernetesDependentResource {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/restart/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/restart/ConfigMapDependentResource.java index a7df64f117..358718b107 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/restart/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/restart/ConfigMapDependentResource.java @@ -16,10 +16,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "key"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( RestartTestCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/servicestrictmatcher/ServiceDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/servicestrictmatcher/ServiceDependentResource.java index df7a275122..56e34330e1 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/servicestrictmatcher/ServiceDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/servicestrictmatcher/ServiceDependentResource.java @@ -19,10 +19,6 @@ public class ServiceDependentResource public static AtomicInteger updated = new AtomicInteger(0); - public ServiceDependentResource() { - super(Service.class); - } - @Override protected Service desired( ServiceStrictMatcherTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/specialresourcesdependent/ServiceAccountDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/specialresourcesdependent/ServiceAccountDependentResource.java index 9808b364a6..1a598992b5 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/specialresourcesdependent/ServiceAccountDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/specialresourcesdependent/ServiceAccountDependentResource.java @@ -12,10 +12,6 @@ public class ServiceAccountDependentResource extends CRUDKubernetesDependentResource { - public ServiceAccountDependentResource() { - super(ServiceAccount.class); - } - @Override protected ServiceAccount desired( SpecialResourceCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/ssalegacymatcher/ServiceDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/ssalegacymatcher/ServiceDependentResource.java index 510aefddcf..f4007b6151 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/ssalegacymatcher/ServiceDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/ssalegacymatcher/ServiceDependentResource.java @@ -18,10 +18,6 @@ public class ServiceDependentResource public static AtomicInteger createUpdateCount = new AtomicInteger(0); - public ServiceDependentResource() { - super(Service.class); - } - @Override protected Service desired( SSALegacyMatcherCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/standalonedependent/StandaloneDependentTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/standalonedependent/StandaloneDependentTestReconciler.java index 43125eef90..f5d9571711 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/standalonedependent/StandaloneDependentTestReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/standalonedependent/StandaloneDependentTestReconciler.java @@ -70,10 +70,6 @@ public boolean isErrorOccurred() { private static class DeploymentDependentResource extends CRUDKubernetesDependentResource { - public DeploymentDependentResource() { - super(Deployment.class); - } - @Override protected Deployment desired( StandaloneDependentTestCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/statefulsetdesiredsanitizer/StatefulSetDesiredSanitizerDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/statefulsetdesiredsanitizer/StatefulSetDesiredSanitizerDependentResource.java index 72bcc8d8a3..fb8e4a6880 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/statefulsetdesiredsanitizer/StatefulSetDesiredSanitizerDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/statefulsetdesiredsanitizer/StatefulSetDesiredSanitizerDependentResource.java @@ -12,10 +12,6 @@ public class StatefulSetDesiredSanitizerDependentResource public static volatile Boolean nonMatchedAtLeastOnce; - public StatefulSetDesiredSanitizerDependentResource() { - super(StatefulSet.class); - } - @Override protected StatefulSet desired( StatefulSetDesiredSanitizerCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/crdpresentactivation/CRDPresentActivationDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/crdpresentactivation/CRDPresentActivationDependent.java index 715caf5314..11923e274b 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/crdpresentactivation/CRDPresentActivationDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/crdpresentactivation/CRDPresentActivationDependent.java @@ -8,10 +8,6 @@ public class CRDPresentActivationDependent extends CRUDNoGCKubernetesDependentResource< CRDPresentActivationDependentCustomResource, CRDPresentActivationCustomResource> { - public CRDPresentActivationDependent() { - super(CRDPresentActivationDependentCustomResource.class); - } - @Override protected CRDPresentActivationDependentCustomResource desired( CRDPresentActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/ConfigMapDependentResource.java index feacf06cb5..c9078848b4 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/ConfigMapDependentResource.java @@ -10,10 +10,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "data"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( GetNonActiveSecondaryCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/RouteDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/RouteDependentResource.java index 99e34df514..77ebef373a 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/RouteDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/getnonactivesecondary/RouteDependentResource.java @@ -8,10 +8,6 @@ public class RouteDependentResource extends CRUDKubernetesDependentResource { - public RouteDependentResource() { - super(Route.class); - } - @Override protected Route desired( GetNonActiveSecondaryCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/ConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/ConfigMapDependent.java index b4eebf36f0..adc633b877 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/ConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/ConfigMapDependent.java @@ -11,10 +11,6 @@ public class ConfigMapDependent extends CRUDNoGCKubernetesDependentResource< ConfigMap, ManagedDependentDefaultDeleteConditionCustomResource> { - public ConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( ManagedDependentDefaultDeleteConditionCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/SecretDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/SecretDependent.java index 2ae036e7ee..a7d52511ea 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/SecretDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/manageddependentdeletecondition/SecretDependent.java @@ -13,10 +13,6 @@ public class SecretDependent extends CRUDNoGCKubernetesDependentResource< Secret, ManagedDependentDefaultDeleteConditionCustomResource> { - public SecretDependent() { - super(Secret.class); - } - @Override protected Secret desired( ManagedDependentDefaultDeleteConditionCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource1.java index cce3eb8ad4..ed83b870ab 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource1.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource1.java @@ -17,10 +17,6 @@ public class ConfigMapDependentResource1 public static final String DATA_KEY = "data"; public static final String SUFFIX = "1"; - public ConfigMapDependentResource1() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleDependentActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource2.java index 8b0a4d89bb..73ccb55cdb 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource2.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/ConfigMapDependentResource2.java @@ -17,10 +17,6 @@ public class ConfigMapDependentResource2 public static final String DATA_KEY = "data"; public static final String SUFFIX = "2"; - public ConfigMapDependentResource2() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( MultipleDependentActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/SecretDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/SecretDependentResource.java index 9b629c5af4..330f0e3c0f 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/SecretDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/multipledependentwithactivation/SecretDependentResource.java @@ -11,10 +11,6 @@ public class SecretDependentResource extends CRUDKubernetesDependentResource { - public SecretDependentResource() { - super(Secret.class); - } - @Override protected Secret desired( MultipleDependentActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource1.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource1.java index f3cc3144c6..eec904d2c7 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource1.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource1.java @@ -15,10 +15,6 @@ public class ConfigMapDependentResource1 extends CRUDKubernetesDependentResource { - public ConfigMapDependentResource1() { - super(ConfigMap.class); - } - @Override public ReconcileResult reconcile( OrderedManagedDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource2.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource2.java index aae0fe79b3..8e4a1467ec 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource2.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/orderedmanageddependent/ConfigMapDependentResource2.java @@ -15,10 +15,6 @@ public class ConfigMapDependentResource2 extends CRUDKubernetesDependentResource { - public ConfigMapDependentResource2() { - super(ConfigMap.class); - } - @Override public ReconcileResult reconcile( OrderedManagedDependentCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcleanup/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcleanup/ConfigMapDependentResource.java index 4edba945a8..cb2357bf8b 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcleanup/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcleanup/ConfigMapDependentResource.java @@ -13,10 +13,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "data"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowActivationCleanupCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/ConfigMapDependentResource.java index 181e35eb2d..5f2e92ed55 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/ConfigMapDependentResource.java @@ -12,10 +12,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "data"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowActivationConditionCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/RouteDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/RouteDependentResource.java index 64a0c9e299..7d2d091c94 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/RouteDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowactivationcondition/RouteDependentResource.java @@ -8,10 +8,6 @@ public class RouteDependentResource extends CRUDKubernetesDependentResource { - public RouteDependentResource() { - super(Route.class); - } - @Override protected Route desired( WorkflowActivationConditionCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/ConfigMapDependentResource.java index 29866202fc..fac6ecae88 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/ConfigMapDependentResource.java @@ -25,10 +25,6 @@ public class ConfigMapDependentResource private static final Logger log = LoggerFactory.getLogger(ConfigMapDependentResource.class); - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowAllFeatureCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/DeploymentDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/DeploymentDependentResource.java index 36e3a95b92..92956e05f6 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/DeploymentDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowallfeature/DeploymentDependentResource.java @@ -8,10 +8,6 @@ public class DeploymentDependentResource extends CRUDNoGCKubernetesDependentResource { - public DeploymentDependentResource() { - super(Deployment.class); - } - @Override protected Deployment desired( WorkflowAllFeatureCustomResource primary, Context context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitcleanup/ConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitcleanup/ConfigMapDependent.java index b42f14cace..17190c5a92 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitcleanup/ConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitcleanup/ConfigMapDependent.java @@ -11,10 +11,6 @@ public class ConfigMapDependent extends CRUDNoGCKubernetesDependentResource { - public ConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowExplicitCleanupCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitinvocation/ConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitinvocation/ConfigMapDependent.java index cc404328a9..a2638b48b5 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitinvocation/ConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowexplicitinvocation/ConfigMapDependent.java @@ -12,10 +12,6 @@ public class ConfigMapDependent extends CRUDNoGCKubernetesDependentResource< ConfigMap, WorkflowExplicitInvocationCustomResource> { - public ConfigMapDependent() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowExplicitInvocationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/ConfigMapDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/ConfigMapDependentResource.java index 2bd666b64c..3366a61a1f 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/ConfigMapDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/ConfigMapDependentResource.java @@ -13,10 +13,6 @@ public class ConfigMapDependentResource public static final String DATA_KEY = "data"; - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired( WorkflowMultipleActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/SecretDependentResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/SecretDependentResource.java index b392259ca9..cd1fbfcedc 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/SecretDependentResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowmultipleactivation/SecretDependentResource.java @@ -11,10 +11,6 @@ public class SecretDependentResource extends CRUDKubernetesDependentResource { - public SecretDependentResource() { - super(Secret.class); - } - @Override protected Secret desired( WorkflowMultipleActivationCustomResource primary, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowsilentexceptionhandling/ConfigMapDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowsilentexceptionhandling/ConfigMapDependent.java index bd8d4099ff..6dbac41f8c 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowsilentexceptionhandling/ConfigMapDependent.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/workflow/workflowsilentexceptionhandling/ConfigMapDependent.java @@ -9,10 +9,6 @@ public class ConfigMapDependent extends CRUDNoGCKubernetesDependentResource< ConfigMap, HandleWorkflowExceptionsInReconcilerCustomResource> { - public ConfigMapDependent() { - super(ConfigMap.class); - } - @Override public ReconcileResult reconcile( HandleWorkflowExceptionsInReconcilerCustomResource primary, diff --git a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SchemaDependentResource.java b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SchemaDependentResource.java index 5bc210f7d4..ec2b03325c 100644 --- a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SchemaDependentResource.java +++ b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SchemaDependentResource.java @@ -54,10 +54,6 @@ public class SchemaDependentResource private MySQLDbConfig dbConfig; - public SchemaDependentResource() { - super(Schema.class); - } - @Override public Optional configuration() { return Optional.of(new ResourcePollerConfig(getPollingPeriod(), dbConfig)); diff --git a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SecretDependentResource.java b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SecretDependentResource.java index 092ac22e24..cff28feadd 100644 --- a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SecretDependentResource.java +++ b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/dependent/SecretDependentResource.java @@ -27,10 +27,6 @@ public class SecretDependentResource extends KubernetesDependentResource { - public DeploymentDependentResource() { - super(Deployment.class); - } - private static String tomcatImage(Tomcat tomcat) { return "tomcat:" + tomcat.getSpec().getVersion(); } diff --git a/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/ServiceDependentResource.java b/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/ServiceDependentResource.java index b42a42257d..bb0359458e 100644 --- a/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/ServiceDependentResource.java +++ b/sample-operators/tomcat-operator/src/main/java/io/javaoperatorsdk/operator/sample/ServiceDependentResource.java @@ -13,10 +13,6 @@ informer = @Informer(labelSelector = "app.kubernetes.io/managed-by=tomcat-operator")) public class ServiceDependentResource extends CRUDKubernetesDependentResource { - public ServiceDependentResource() { - super(Service.class); - } - @Override protected Service desired(Tomcat tomcat, Context context) { final ObjectMeta tomcatMetadata = tomcat.getMetadata(); diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ConfigMapDependentResource.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ConfigMapDependentResource.java index 816db3688e..0cf8faad7c 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ConfigMapDependentResource.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ConfigMapDependentResource.java @@ -20,10 +20,6 @@ public class ConfigMapDependentResource extends CRUDKubernetesDependentResource { - public ConfigMapDependentResource() { - super(ConfigMap.class); - } - @Override protected ConfigMap desired(WebPage webPage, Context context) { Map data = new HashMap<>(); diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/DeploymentDependentResource.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/DeploymentDependentResource.java index 3476464f1f..4deef0f1c0 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/DeploymentDependentResource.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/DeploymentDependentResource.java @@ -22,10 +22,6 @@ public class DeploymentDependentResource extends CRUDKubernetesDependentResource { - public DeploymentDependentResource() { - super(Deployment.class); - } - @Override protected Deployment desired(WebPage webPage, Context context) { Map labels = new HashMap<>(); diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/IngressDependentResource.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/IngressDependentResource.java index 994a35c98c..3f3e64e8ed 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/IngressDependentResource.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/IngressDependentResource.java @@ -15,10 +15,6 @@ informer = @Informer(labelSelector = WebPageManagedDependentsReconciler.SELECTOR)) public class IngressDependentResource extends CRUDKubernetesDependentResource { - public IngressDependentResource() { - super(Ingress.class); - } - @Override protected Ingress desired(WebPage webPage, Context context) { return makeDesiredIngress(webPage); diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ServiceDependentResource.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ServiceDependentResource.java index b1ab856f62..01e8953fa9 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ServiceDependentResource.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/dependentresource/ServiceDependentResource.java @@ -22,10 +22,6 @@ public class ServiceDependentResource .CRUDKubernetesDependentResource< Service, WebPage> { - public ServiceDependentResource() { - super(Service.class); - } - @Override protected Service desired(WebPage webPage, Context context) { Map serviceLabels = new HashMap<>(); From 5520c14b3fcb27b6c69d352209f90f78626ca994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 16 May 2025 11:21:09 +0200 Subject: [PATCH 37/45] improve: status cache for next reconciliation - only the lock version (#2800) --- .../en/docs/documentation/reconciler.md | 102 +----- .../PrimaryUpdateAndCacheUtils.java | 296 ++++++++---------- .../support/PrimaryResourceCache.java | 65 ---- .../PrimaryUpdateAndCacheUtilsTest.java | 111 +++++++ .../support/PrimaryResourceCacheTest.java | 87 ----- ...atusPatchCacheWithLockCustomResource.java} | 8 +- ...T.java => StatusPatchCacheWithLockIT.java} | 14 +- ...> StatusPatchCacheWithLockReconciler.java} | 29 +- ...java => StatusPatchCacheWithLockSpec.java} | 4 +- .../StatusPatchCacheWithLockStatus.java | 15 + .../StatusPatchCacheCustomResource.java | 13 - .../internal/StatusPatchCacheStatus.java | 15 - .../StatusPatchPrimaryCacheIT.java | 48 --- .../StatusPatchPrimaryCacheReconciler.java | 89 ------ .../StatusPatchPrimaryCacheSpec.java | 15 - .../StatusPatchPrimaryCacheStatus.java | 15 - 16 files changed, 306 insertions(+), 620 deletions(-) delete mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtilsTest.java delete mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java rename operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/{primarycache/StatusPatchPrimaryCacheCustomResource.java => StatusPatchCacheWithLockCustomResource.java} (60%) rename operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/{internal/StatusPatchCacheIT.java => StatusPatchCacheWithLockIT.java} (76%) rename operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/{internal/StatusPatchCacheReconciler.java => StatusPatchCacheWithLockReconciler.java} (60%) rename operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/{internal/StatusPatchCacheSpec.java => StatusPatchCacheWithLockSpec.java} (60%) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockStatus.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java diff --git a/docs/content/en/docs/documentation/reconciler.md b/docs/content/en/docs/documentation/reconciler.md index b9ede8aa95..fa51399de7 100644 --- a/docs/content/en/docs/documentation/reconciler.md +++ b/docs/content/en/docs/documentation/reconciler.md @@ -175,23 +175,23 @@ From v5, by default, the finalizer is added using Server Side Apply. See also `U It is typical to want to update the status subresource with the information that is available during the reconciliation. This is sometimes referred to as the last observed state. When the primary resource is updated, though, the framework does not cache the resource directly, relying instead on the propagation of the update to the underlying informer's -cache. It can, therefore, happen that, if other events trigger other reconciliations before the informer cache gets +cache. It can, therefore, happen that, if other events trigger other reconciliations, before the informer cache gets updated, your reconciler does not see the latest version of the primary resource. While this might not typically be a problem in most cases, as caches eventually become consistent, depending on your reconciliation logic, you might still -require the latest status version possible, for example if the status subresource is used as a communication mechanism, -see [Representing Allocated Values](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#representing-allocated-values) +require the latest status version possible, for example, if the status subresource is used to store allocated values. +See [Representing Allocated Values](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#representing-allocated-values) from the Kubernetes docs for more details. -The framework provides utilities to help with these use cases with -[`PrimaryUpdateAndCacheUtils`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java). -These utility methods come in two flavors: +The framework provides the +[`PrimaryUpdateAndCacheUtils`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java) utility class +to help with these use cases. -#### Using internal cache - -In almost all cases for this purpose, you can use internal caches: +This class' methods use internal caches in combination with update methods that leveraging +optimistic locking. If the update method fails on optimistic locking, it will retry +using a fresh resource from the server as base for modification. ```java - @Override +@Override public UpdateControl reconcile( StatusPatchCacheCustomResource resource, Context context) { @@ -201,85 +201,17 @@ public UpdateControl reconcile( var freshCopy = createFreshCopy(primary); freshCopy.getStatus().setValue(statusWithState()); - var updatedResource = PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(resource, freshCopy, context); - - return UpdateControl.noUpdate(); - } -``` - -In the background `PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus` puts the result of the update into an internal -cache and will make sure that the next reconciliation will contain the most recent version of the resource. Note that it -is not necessarily the version of the resource you got as response from the update, it can be newer since other parties -can do additional updates meanwhile, but if not explicitly modified, it will contain the up-to-date status. - -See related integration test [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal). - -This approach works with the default configuration of the framework and should be good to go in most of the cases. -Without going further into the details, this won't work if `ConfigurationService.parseResourceVersionsForEventFilteringAndCaching` -is set to `false` (more precisely there are some edge cases when it won't work). For that case framework provides the following solution: - -#### Fallback approach: using `PrimaryResourceCache` cache - -As an alternative, for very rare cases when `ConfigurationService.parseResourceVersionsForEventFilteringAndCaching` -needs to be set to `false` you can use an explicit caching approach: - -```java - -// We on purpose don't use the provided predicate to show what a custom one could look like. - private final PrimaryResourceCache cache = - new PrimaryResourceCache<>( - (statusPatchCacheCustomResourcePair, statusPatchCacheCustomResource) -> - statusPatchCacheCustomResource.getStatus().getValue() - >= statusPatchCacheCustomResourcePair.afterUpdate().getStatus().getValue()); - - @Override - public UpdateControl reconcile( - StatusPatchPrimaryCacheCustomResource primary, - Context context) { - - // cache will compare the current and the cached resource and return the more recent. (And evict the old) - primary = cache.getFreshResource(primary); - - // omitted logic - - var freshCopy = createFreshCopy(primary); + var updatedResource = PrimaryUpdateAndCacheUtils.ssaPatchStatusAndCacheResource(resource, freshCopy, context); - freshCopy.getStatus().setValue(statusWithState()); - - var updated = - PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(primary, freshCopy, context, cache); - return UpdateControl.noUpdate(); } - - @Override - public DeleteControl cleanup( - StatusPatchPrimaryCacheCustomResource resource, - Context context) - throws Exception { - // cleanup the cache on resource deletion - cache.cleanup(resource); - return DeleteControl.defaultDelete(); - } - ``` -[`PrimaryResourceCache`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java) -is designed for this purpose. As shown in the example above, it is up to you to provide a predicate to determine if the -resource is more recent than the one available. In other words, when to evict the resource from the cache. Typically, as -shown in -the [integration test](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache) -you can have a counter in status to check on that. - -Since all of this happens explicitly, you cannot use this approach for managed dependent resources and workflows and -will need to use the unmanaged approach instead. This is due to the fact that managed dependent resources always get -their associated primary resource from the underlying informer event source cache. - -#### Additional remarks +After the update `PrimaryUpdateAndCacheUtils.ssaPatchStatusAndCacheResource` puts the result of the update into an internal +cache and the framework will make sure that the next reconciliation contains the most recent version of the resource. +Note that it is not necessarily the same version returned as response from the update, it can be a newer version since other parties +can do additional updates meanwhile. However, unless it has been explicitly modified, that +resource will contain the up-to-date status. -As shown in the integration tests, there is no optimistic locking used when updating the -[resource](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java#L41) -(in other words `metadata.resourceVersion` is set to `null`). This is desired since you don't want the patch to fail on -update. -In addition, you can configure the [Fabric8 client retry](https://github.com/fabric8io/kubernetes-client?tab=readme-ov-file#configuring-the-client). +See related integration test [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache). diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java index 174f7667f6..ac0fe9675c 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java @@ -1,146 +1,85 @@ package io.javaoperatorsdk.operator.api.reconciler; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.dsl.base.PatchContext; import io.fabric8.kubernetes.client.dsl.base.PatchType; -import io.javaoperatorsdk.operator.api.reconciler.support.PrimaryResourceCache; +import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.processing.event.ResourceID; /** * Utility methods to patch the primary resource state and store it to the related cache, to make - * sure that fresh resource is present for the next reconciliation. The main use case for such - * updates is to store state is resource status. Use of optimistic locking is not desired for such - * updates, since we don't want to patch fail and lose information that we want to store. + * sure that the latest version of the resource is present for the next reconciliation. The main use + * case for such updates is to store state is resource status. + * + *

The way the framework handles this is with retryable updates with optimistic locking, and + * caches the updated resource from the response in an overlay cache on top of the Informer cache. + * If the update fails, it reads the primary resource from the cluster, applies the modifications + * again and retries the update. */ public class PrimaryUpdateAndCacheUtils { + public static final int DEFAULT_MAX_RETRY = 10; + private PrimaryUpdateAndCacheUtils() {} private static final Logger log = LoggerFactory.getLogger(PrimaryUpdateAndCacheUtils.class); /** - * Makes sure that the up-to-date primary resource will be present during the next reconciliation. - * Using update (PUT) method. - * - * @param primary resource - * @param context of reconciliation - * @return updated resource - * @param

primary resource type - */ - public static

P updateAndCacheStatus(P primary, Context

context) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, context, () -> context.getClient().resource(primary).updateStatus()); - } - - /** - * Makes sure that the up-to-date primary resource will be present during the next reconciliation. - * Using JSON Merge patch. - * - * @param primary resource - * @param context of reconciliation - * @return updated resource - * @param

primary resource type + * Updates the status with optimistic locking and caches the result for next reconciliation. For + * details see {@link #updateAndCacheResource}. */ - public static

P patchAndCacheStatus(P primary, Context

context) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, context, () -> context.getClient().resource(primary).patchStatus()); + public static

P updateStatusAndCacheResource( + P primary, Context

context, UnaryOperator

modificationFunction) { + return updateAndCacheResource( + primary, + context, + modificationFunction, + r -> context.getClient().resource(r).updateStatus()); } /** - * Makes sure that the up-to-date primary resource will be present during the next reconciliation. - * Using JSON Patch. - * - * @param primary resource - * @param context of reconciliation - * @return updated resource - * @param

primary resource type + * Patches the status using JSON Merge Patch with optimistic locking and caches the result for + * next reconciliation. For details see {@link #updateAndCacheResource}. */ - public static

P editAndCacheStatus( - P primary, Context

context, UnaryOperator

operation) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, context, () -> context.getClient().resource(primary).editStatus(operation)); + public static

P mergePatchStatusAndCacheResource( + P primary, Context

context, UnaryOperator

modificationFunction) { + return updateAndCacheResource( + primary, context, modificationFunction, r -> context.getClient().resource(r).patchStatus()); } /** - * Makes sure that the up-to-date primary resource will be present during the next reconciliation. - * - * @param primary resource - * @param context of reconciliation - * @param patch free implementation of cache - * @return the updated resource. - * @param

primary resource type + * Patches the status using JSON Patch with optimistic locking and caches the result for next + * reconciliation. For details see {@link #updateAndCacheResource}. */ - public static

P patchAndCacheStatus( - P primary, Context

context, Supplier

patch) { - var updatedResource = patch.get(); - context - .eventSourceRetriever() - .getControllerEventSource() - .handleRecentResourceUpdate(ResourceID.fromResource(primary), updatedResource, primary); - return updatedResource; + public static

P patchStatusAndCacheResource( + P primary, Context

context, UnaryOperator

modificationFunction) { + return updateAndCacheResource( + primary, + context, + UnaryOperator.identity(), + r -> context.getClient().resource(r).editStatus(modificationFunction)); } /** - * Makes sure that the up-to-date primary resource will be present during the next reconciliation. - * Using Server Side Apply. - * - * @param primary resource - * @param freshResourceWithStatus - fresh resource with target state - * @param context of reconciliation - * @return the updated resource. - * @param

primary resource type + * Patches the status using Server Side Apply with optimistic locking and caches the result for + * next reconciliation. For details see {@link #updateAndCacheResource}. */ - public static

P ssaPatchAndCacheStatus( + public static

P ssaPatchStatusAndCacheResource( P primary, P freshResourceWithStatus, Context

context) { - logWarnIfResourceVersionPresent(freshResourceWithStatus); - var res = - context - .getClient() - .resource(freshResourceWithStatus) - .subresource("status") - .patch( - new PatchContext.Builder() - .withForce(true) - .withFieldManager(context.getControllerConfiguration().fieldManager()) - .withPatchType(PatchType.SERVER_SIDE_APPLY) - .build()); - - context - .eventSourceRetriever() - .getControllerEventSource() - .handleRecentResourceUpdate(ResourceID.fromResource(primary), res, primary); - return res; - } - - /** - * Patches the resource and adds it to the {@link PrimaryResourceCache}. - * - * @param primary resource - * @param freshResourceWithStatus - fresh resource with target state - * @param context of reconciliation - * @param cache - resource cache managed by user - * @return the updated resource. - * @param

primary resource type - */ - public static

P ssaPatchAndCacheStatus( - P primary, P freshResourceWithStatus, Context

context, PrimaryResourceCache

cache) { - logWarnIfResourceVersionPresent(freshResourceWithStatus); - return patchAndCacheStatus( + return updateAndCacheResource( primary, - cache, - () -> + context, + r -> freshResourceWithStatus, + r -> context .getClient() - .resource(freshResourceWithStatus) + .resource(r) .subresource("status") .patch( new PatchContext.Builder() @@ -151,75 +90,104 @@ public static

P ssaPatchAndCacheStatus( } /** - * Patches the resource with JSON Patch and adds it to the {@link PrimaryResourceCache}. - * - * @param primary resource - * @param context of reconciliation - * @param cache - resource cache managed by user - * @return the updated resource. - * @param

primary resource type - */ - public static

P editAndCacheStatus( - P primary, Context

context, PrimaryResourceCache

cache, UnaryOperator

operation) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, cache, () -> context.getClient().resource(primary).editStatus(operation)); - } - - /** - * Patches the resource with JSON Merge patch and adds it to the {@link PrimaryResourceCache} - * provided. + * Same as {@link #updateAndCacheResource(HasMetadata, Context, UnaryOperator, UnaryOperator, + * int)} using the default maximum retry number as defined by {@link #DEFAULT_MAX_RETRY}. * - * @param primary resource + * @param resourceToUpdate original resource to update * @param context of reconciliation - * @param cache - resource cache managed by user - * @return the updated resource. - * @param

primary resource type + * @param modificationFunction modifications to make on primary + * @param updateMethod the update method implementation + * @param

primary type + * @return the updated resource */ - public static

P patchAndCacheStatus( - P primary, Context

context, PrimaryResourceCache

cache) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, cache, () -> context.getClient().resource(primary).patchStatus()); + public static

P updateAndCacheResource( + P resourceToUpdate, + Context

context, + UnaryOperator

modificationFunction, + UnaryOperator

updateMethod) { + return updateAndCacheResource( + resourceToUpdate, context, modificationFunction, updateMethod, DEFAULT_MAX_RETRY); } /** - * Updates the resource and adds it to the {@link PrimaryResourceCache}. + * Modifies the primary using the specified modification function, then uses the modified resource + * for the request to update with provided update method. As the {@code resourceVersion} field of + * the modified resource is set to the value found in the specified resource to update, the update + * operation will therefore use optimistic locking on the server. If the request fails on + * optimistic update, we read the resource again from the K8S API server and retry the whole + * process. In short, we make sure we always update the resource with optimistic locking, then we + * cache the resource in an internal cache. Without further going into details, the optimistic + * locking is needed so we can reliably handle the caching. * - * @param primary resource + * @param resourceToUpdate original resource to update * @param context of reconciliation - * @param cache - resource cache managed by user - * @return the updated resource. - * @param

primary resource type - */ - public static

P updateAndCacheStatus( - P primary, Context

context, PrimaryResourceCache

cache) { - logWarnIfResourceVersionPresent(primary); - return patchAndCacheStatus( - primary, cache, () -> context.getClient().resource(primary).updateStatus()); - } - - /** - * Updates the resource using the user provided implementation anc caches the result. - * - * @param primary resource - * @param cache resource cache managed by user - * @param patch implementation of resource update* - * @return the updated resource. - * @param

primary resource type + * @param modificationFunction modifications to make on primary + * @param updateMethod the update method implementation + * @param maxRetry maximum number of retries before giving up + * @param

primary type + * @return the updated resource */ - public static

P patchAndCacheStatus( - P primary, PrimaryResourceCache

cache, Supplier

patch) { - var updatedResource = patch.get(); - cache.cacheResource(primary, updatedResource); - return updatedResource; - } - - private static

void logWarnIfResourceVersionPresent(P primary) { - if (primary.getMetadata().getResourceVersion() != null) { - log.warn( - "The metadata.resourceVersion of primary resource is NOT null, " - + "using optimistic locking is discouraged for this purpose. "); + @SuppressWarnings("unchecked") + public static

P updateAndCacheResource( + P resourceToUpdate, + Context

context, + UnaryOperator

modificationFunction, + UnaryOperator

updateMethod, + int maxRetry) { + + if (log.isDebugEnabled()) { + log.debug("Conflict retrying update for: {}", ResourceID.fromResource(resourceToUpdate)); + } + P modified = null; + int retryIndex = 0; + while (true) { + try { + modified = modificationFunction.apply(resourceToUpdate); + modified + .getMetadata() + .setResourceVersion(resourceToUpdate.getMetadata().getResourceVersion()); + var updated = updateMethod.apply(modified); + context + .eventSourceRetriever() + .getControllerEventSource() + .handleRecentResourceUpdate( + ResourceID.fromResource(resourceToUpdate), updated, resourceToUpdate); + return updated; + } catch (KubernetesClientException e) { + log.trace("Exception during patch for resource: {}", resourceToUpdate); + retryIndex++; + // only retry on conflict (409) and unprocessable content (422) which + // can happen if JSON Patch is not a valid request since there was + // a concurrent request which already removed another finalizer: + // List element removal from a list is by index in JSON Patch + // so if addressing a second finalizer but first is meanwhile removed + // it is a wrong request. + if (e.getCode() != 409 && e.getCode() != 422) { + throw e; + } + if (retryIndex > maxRetry) { + log.warn("Retry exhausted, last desired resource: {}", modified); + throw new OperatorException( + "Exceeded maximum (" + + maxRetry + + ") retry attempts to patch resource: " + + ResourceID.fromResource(resourceToUpdate), + e); + } + log.debug( + "Retrying patch for resource name: {}, namespace: {}; HTTP code: {}", + resourceToUpdate.getMetadata().getName(), + resourceToUpdate.getMetadata().getNamespace(), + e.getCode()); + resourceToUpdate = + (P) + context + .getClient() + .resources(resourceToUpdate.getClass()) + .inNamespace(resourceToUpdate.getMetadata().getNamespace()) + .withName(resourceToUpdate.getMetadata().getName()) + .get(); + } } } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java deleted file mode 100644 index 4da73ab8b1..0000000000 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCache.java +++ /dev/null @@ -1,65 +0,0 @@ -package io.javaoperatorsdk.operator.api.reconciler.support; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.BiPredicate; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.processing.event.ResourceID; - -public class PrimaryResourceCache

{ - - private final BiPredicate, P> evictionPredicate; - private final ConcurrentHashMap> cache = new ConcurrentHashMap<>(); - - public PrimaryResourceCache(BiPredicate, P> evictionPredicate) { - this.evictionPredicate = evictionPredicate; - } - - public PrimaryResourceCache() { - this(new ResourceVersionParsingEvictionPredicate<>()); - } - - public void cacheResource(P afterUpdate) { - var resourceId = ResourceID.fromResource(afterUpdate); - cache.put(resourceId, new Pair<>(null, afterUpdate)); - } - - public void cacheResource(P beforeUpdate, P afterUpdate) { - var resourceId = ResourceID.fromResource(beforeUpdate); - cache.put(resourceId, new Pair<>(beforeUpdate, afterUpdate)); - } - - public P getFreshResource(P newVersion) { - var resourceId = ResourceID.fromResource(newVersion); - var pair = cache.get(resourceId); - if (pair == null) { - return newVersion; - } - if (!newVersion.getMetadata().getUid().equals(pair.afterUpdate().getMetadata().getUid())) { - cache.remove(resourceId); - return newVersion; - } - if (evictionPredicate.test(pair, newVersion)) { - cache.remove(resourceId); - return newVersion; - } else { - return pair.afterUpdate(); - } - } - - public void cleanup(P resource) { - cache.remove(ResourceID.fromResource(resource)); - } - - public record Pair(T beforeUpdate, T afterUpdate) {} - - /** This works in general, but it does not strictly follow the contract with k8s API */ - public static class ResourceVersionParsingEvictionPredicate - implements BiPredicate, T> { - @Override - public boolean test(Pair updatePair, T newVersion) { - return Long.parseLong(updatePair.afterUpdate().getMetadata().getResourceVersion()) - <= Long.parseLong(newVersion.getMetadata().getResourceVersion()); - } - } -} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtilsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtilsTest.java new file mode 100644 index 0000000000..438941db9c --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtilsTest.java @@ -0,0 +1,111 @@ +package io.javaoperatorsdk.operator.api.reconciler; + +import java.util.function.UnaryOperator; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.javaoperatorsdk.operator.OperatorException; +import io.javaoperatorsdk.operator.TestUtils; +import io.javaoperatorsdk.operator.processing.event.EventSourceRetriever; +import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource; +import io.javaoperatorsdk.operator.sample.simple.TestCustomResource; + +import static io.javaoperatorsdk.operator.api.reconciler.PrimaryUpdateAndCacheUtils.DEFAULT_MAX_RETRY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class PrimaryUpdateAndCacheUtilsTest { + + Context context = mock(Context.class); + KubernetesClient client = mock(KubernetesClient.class); + Resource resource = mock(Resource.class); + + @BeforeEach + void setup() { + when(context.getClient()).thenReturn(client); + var esr = mock(EventSourceRetriever.class); + when(context.eventSourceRetriever()).thenReturn(esr); + when(esr.getControllerEventSource()).thenReturn(mock(ControllerEventSource.class)); + var mixedOp = mock(MixedOperation.class); + when(client.resources(any())).thenReturn(mixedOp); + when(mixedOp.inNamespace(any())).thenReturn(mixedOp); + when(mixedOp.withName(any())).thenReturn(resource); + when(resource.get()).thenReturn(TestUtils.testCustomResource1()); + } + + @Test + void handlesUpdate() { + var updated = + PrimaryUpdateAndCacheUtils.updateAndCacheResource( + TestUtils.testCustomResource1(), + context, + r -> { + var res = TestUtils.testCustomResource1(); + // setting this to null to test if value set in the implementation + res.getMetadata().setResourceVersion(null); + res.getSpec().setValue("updatedValue"); + return res; + }, + r -> { + // checks if the resource version is set from the original resource + assertThat(r.getMetadata().getResourceVersion()).isEqualTo("1"); + var res = TestUtils.testCustomResource1(); + res.setSpec(r.getSpec()); + res.getMetadata().setResourceVersion("2"); + return res; + }); + + assertThat(updated.getMetadata().getResourceVersion()).isEqualTo("2"); + assertThat(updated.getSpec().getValue()).isEqualTo("updatedValue"); + } + + @Test + void retriesConflicts() { + var updateOperation = mock(UnaryOperator.class); + + when(updateOperation.apply(any())) + .thenThrow(new KubernetesClientException("", 409, null)) + .thenReturn(TestUtils.testCustomResource1()); + + var updated = + PrimaryUpdateAndCacheUtils.updateAndCacheResource( + TestUtils.testCustomResource1(), + context, + r -> { + var res = TestUtils.testCustomResource1(); + res.getSpec().setValue("updatedValue"); + return res; + }, + updateOperation); + + assertThat(updated).isNotNull(); + verify(resource, times(1)).get(); + } + + @Test + void throwsIfRetryExhausted() { + var updateOperation = mock(UnaryOperator.class); + + when(updateOperation.apply(any())).thenThrow(new KubernetesClientException("", 409, null)); + + assertThrows( + OperatorException.class, + () -> + PrimaryUpdateAndCacheUtils.updateAndCacheResource( + TestUtils.testCustomResource1(), + context, + UnaryOperator.identity(), + updateOperation)); + verify(resource, times(DEFAULT_MAX_RETRY)).get(); + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java deleted file mode 100644 index 58e3ce8a0a..0000000000 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/reconciler/support/PrimaryResourceCacheTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package io.javaoperatorsdk.operator.api.reconciler.support; - -import org.junit.jupiter.api.Test; - -import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; -import io.javaoperatorsdk.operator.sample.simple.TestCustomResource; -import io.javaoperatorsdk.operator.sample.simple.TestCustomResourceSpec; - -import static org.assertj.core.api.Assertions.assertThat; - -class PrimaryResourceCacheTest { - - PrimaryResourceCache versionParsingCache = - new PrimaryResourceCache<>( - new PrimaryResourceCache.ResourceVersionParsingEvictionPredicate<>()); - - @Test - void returnsThePassedValueIfCacheIsEmpty() { - var cr = customResource("1"); - - var res = versionParsingCache.getFreshResource(cr); - - assertThat(cr).isSameAs(res); - } - - @Test - void returnsTheCachedIfNotEvictedAccordingToPredicate() { - var cr = customResource("2"); - - versionParsingCache.cacheResource(cr); - - var res = versionParsingCache.getFreshResource(customResource("1")); - assertThat(cr).isSameAs(res); - } - - @Test - void ifMoreFreshPassedCachedIsEvicted() { - var cr = customResource("2"); - versionParsingCache.cacheResource(cr); - var newCR = customResource("3"); - - var res = versionParsingCache.getFreshResource(newCR); - var resOnOlder = versionParsingCache.getFreshResource(cr); - - assertThat(newCR).isSameAs(res); - assertThat(resOnOlder).isSameAs(cr); - assertThat(newCR).isNotSameAs(cr); - } - - @Test - void cleanupRemovesCachedResources() { - var cr = customResource("2"); - versionParsingCache.cacheResource(cr); - - versionParsingCache.cleanup(customResource("3")); - - var olderCR = customResource("1"); - var res = versionParsingCache.getFreshResource(olderCR); - assertThat(olderCR).isSameAs(res); - } - - @Test - void removesIfNewResourceWithDifferentUid() { - var cr = customResource("2"); - versionParsingCache.cacheResource(cr); - var crWithDifferentUid = customResource("1"); - cr.getMetadata().setUid("otheruid"); - - var res = versionParsingCache.getFreshResource(crWithDifferentUid); - - assertThat(res).isSameAs(crWithDifferentUid); - } - - private TestCustomResource customResource(String resourceVersion) { - var cr = new TestCustomResource(); - cr.setMetadata( - new ObjectMetaBuilder() - .withName("test1") - .withNamespace("default") - .withUid("uid") - .withResourceVersion(resourceVersion) - .build()); - cr.setSpec(new TestCustomResourceSpec()); - cr.getSpec().setKey("key"); - return cr; - } -} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockCustomResource.java similarity index 60% rename from operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java rename to operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockCustomResource.java index 84b145cac3..8ab742a975 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheCustomResource.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockCustomResource.java @@ -1,4 +1,4 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; +package io.javaoperatorsdk.operator.baseapi.statuscache; import io.fabric8.kubernetes.api.model.Namespaced; import io.fabric8.kubernetes.client.CustomResource; @@ -8,7 +8,7 @@ @Group("sample.javaoperatorsdk") @Version("v1") -@ShortNames("spc") -public class StatusPatchPrimaryCacheCustomResource - extends CustomResource +@ShortNames("spwl") +public class StatusPatchCacheWithLockCustomResource + extends CustomResource implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockIT.java similarity index 76% rename from operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java rename to operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockIT.java index f78511f250..c5752f4aae 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockIT.java @@ -1,4 +1,4 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.internal; +package io.javaoperatorsdk.operator.baseapi.statuscache; import java.time.Duration; @@ -11,19 +11,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; -public class StatusPatchCacheIT { +public class StatusPatchCacheWithLockIT { public static final String TEST_1 = "test1"; @RegisterExtension LocallyRunOperatorExtension extension = LocallyRunOperatorExtension.builder() - .withReconciler(StatusPatchCacheReconciler.class) + .withReconciler(StatusPatchCacheWithLockReconciler.class) .build(); @Test void testStatusAlwaysUpToDate() { - var reconciler = extension.getReconcilerOfType(StatusPatchCacheReconciler.class); + var reconciler = extension.getReconcilerOfType(StatusPatchCacheWithLockReconciler.class); extension.create(testResource()); @@ -39,10 +39,10 @@ void testStatusAlwaysUpToDate() { }); } - StatusPatchCacheCustomResource testResource() { - var res = new StatusPatchCacheCustomResource(); + StatusPatchCacheWithLockCustomResource testResource() { + var res = new StatusPatchCacheWithLockCustomResource(); res.setMetadata(new ObjectMetaBuilder().withName(TEST_1).build()); - res.setSpec(new StatusPatchCacheSpec()); + res.setSpec(new StatusPatchCacheWithLockSpec()); return res; } } diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockReconciler.java similarity index 60% rename from operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java rename to operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockReconciler.java index 8a3a72a901..364f8e9ff5 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockReconciler.java @@ -1,4 +1,4 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.internal; +package io.javaoperatorsdk.operator.baseapi.statuscache; import java.util.List; @@ -9,18 +9,19 @@ import io.javaoperatorsdk.operator.api.reconciler.PrimaryUpdateAndCacheUtils; import io.javaoperatorsdk.operator.api.reconciler.Reconciler; import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; -import io.javaoperatorsdk.operator.baseapi.statuscache.PeriodicTriggerEventSource; import io.javaoperatorsdk.operator.processing.event.source.EventSource; @ControllerConfiguration -public class StatusPatchCacheReconciler implements Reconciler { +public class StatusPatchCacheWithLockReconciler + implements Reconciler { public volatile int latestValue = 0; public volatile boolean errorPresent = false; @Override - public UpdateControl reconcile( - StatusPatchCacheCustomResource resource, Context context) { + public UpdateControl reconcile( + StatusPatchCacheWithLockCustomResource resource, + Context context) { if (resource.getStatus() != null && resource.getStatus().getValue() != latestValue) { errorPresent = true; @@ -31,33 +32,39 @@ public UpdateControl reconcile( + resource.getStatus().getValue()); } + // test also resource update happening meanwhile reconciliation + resource.getSpec().setCounter(resource.getSpec().getCounter() + 1); + context.getClient().resource(resource).update(); + var freshCopy = createFreshCopy(resource); freshCopy .getStatus() .setValue(resource.getStatus() == null ? 1 : resource.getStatus().getValue() + 1); - var updated = PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(resource, freshCopy, context); + var updated = + PrimaryUpdateAndCacheUtils.ssaPatchStatusAndCacheResource(resource, freshCopy, context); latestValue = updated.getStatus().getValue(); return UpdateControl.noUpdate(); } @Override - public List> prepareEventSources( - EventSourceContext context) { + public List> prepareEventSources( + EventSourceContext context) { // periodic event triggering for testing purposes return List.of(new PeriodicTriggerEventSource<>(context.getPrimaryCache())); } - private StatusPatchCacheCustomResource createFreshCopy(StatusPatchCacheCustomResource resource) { - var res = new StatusPatchCacheCustomResource(); + private StatusPatchCacheWithLockCustomResource createFreshCopy( + StatusPatchCacheWithLockCustomResource resource) { + var res = new StatusPatchCacheWithLockCustomResource(); res.setMetadata( new ObjectMetaBuilder() .withName(resource.getMetadata().getName()) .withNamespace(resource.getMetadata().getNamespace()) .build()); - res.setStatus(new StatusPatchCacheStatus()); + res.setStatus(new StatusPatchCacheWithLockStatus()); return res; } diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockSpec.java similarity index 60% rename from operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java rename to operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockSpec.java index d1426fd943..ebbabd49a0 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheSpec.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockSpec.java @@ -1,6 +1,6 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.internal; +package io.javaoperatorsdk.operator.baseapi.statuscache; -public class StatusPatchCacheSpec { +public class StatusPatchCacheWithLockSpec { private int counter = 0; diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockStatus.java new file mode 100644 index 0000000000..5f2d8f5a6f --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/StatusPatchCacheWithLockStatus.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.baseapi.statuscache; + +public class StatusPatchCacheWithLockStatus { + + private Integer value = 0; + + public Integer getValue() { + return value; + } + + public StatusPatchCacheWithLockStatus setValue(Integer value) { + this.value = value; + return this; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java deleted file mode 100644 index 2a2d8b83fd..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheCustomResource.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.internal; - -import io.fabric8.kubernetes.api.model.Namespaced; -import io.fabric8.kubernetes.client.CustomResource; -import io.fabric8.kubernetes.model.annotation.Group; -import io.fabric8.kubernetes.model.annotation.ShortNames; -import io.fabric8.kubernetes.model.annotation.Version; - -@Group("sample.javaoperatorsdk") -@Version("v1") -@ShortNames("spcl") -public class StatusPatchCacheCustomResource - extends CustomResource implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java deleted file mode 100644 index 00bc4b6f04..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/internal/StatusPatchCacheStatus.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.internal; - -public class StatusPatchCacheStatus { - - private Integer value = 0; - - public Integer getValue() { - return value; - } - - public StatusPatchCacheStatus setValue(Integer value) { - this.value = value; - return this; - } -} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java deleted file mode 100644 index a884ec0758..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheIT.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; - -import java.time.Duration; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; -import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -class StatusPatchPrimaryCacheIT { - - public static final String TEST_1 = "test1"; - - @RegisterExtension - LocallyRunOperatorExtension extension = - LocallyRunOperatorExtension.builder() - .withReconciler(StatusPatchPrimaryCacheReconciler.class) - .build(); - - @Test - void testStatusAlwaysUpToDate() { - var reconciler = extension.getReconcilerOfType(StatusPatchPrimaryCacheReconciler.class); - - extension.create(testResource()); - - // the reconciliation is periodically triggered, the status values should be increasing - // monotonically - await() - .pollDelay(Duration.ofSeconds(1)) - .pollInterval(Duration.ofMillis(30)) - .untilAsserted( - () -> { - assertThat(reconciler.errorPresent).isFalse(); - assertThat(reconciler.latestValue).isGreaterThan(10); - }); - } - - StatusPatchPrimaryCacheCustomResource testResource() { - var res = new StatusPatchPrimaryCacheCustomResource(); - res.setMetadata(new ObjectMetaBuilder().withName(TEST_1).build()); - res.setSpec(new StatusPatchPrimaryCacheSpec()); - return res; - } -} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java deleted file mode 100644 index c25fcddfec..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheReconciler.java +++ /dev/null @@ -1,89 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; - -import java.util.List; - -import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; -import io.javaoperatorsdk.operator.api.reconciler.Cleaner; -import io.javaoperatorsdk.operator.api.reconciler.Context; -import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; -import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; -import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; -import io.javaoperatorsdk.operator.api.reconciler.PrimaryUpdateAndCacheUtils; -import io.javaoperatorsdk.operator.api.reconciler.Reconciler; -import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; -import io.javaoperatorsdk.operator.api.reconciler.support.PrimaryResourceCache; -import io.javaoperatorsdk.operator.baseapi.statuscache.PeriodicTriggerEventSource; -import io.javaoperatorsdk.operator.processing.event.source.EventSource; - -@ControllerConfiguration -public class StatusPatchPrimaryCacheReconciler - implements Reconciler, - Cleaner { - - public volatile int latestValue = 0; - public volatile boolean errorPresent = false; - - // We on purpose don't use the provided predicate to show what a custom one could look like. - private final PrimaryResourceCache cache = - new PrimaryResourceCache<>( - (statusPatchCacheCustomResourcePair, statusPatchCacheCustomResource) -> - statusPatchCacheCustomResource.getStatus().getValue() - >= statusPatchCacheCustomResourcePair.afterUpdate().getStatus().getValue()); - - @Override - public UpdateControl reconcile( - StatusPatchPrimaryCacheCustomResource primary, - Context context) { - - primary = cache.getFreshResource(primary); - - if (primary.getStatus() != null && primary.getStatus().getValue() != latestValue) { - errorPresent = true; - throw new IllegalStateException( - "status is not up to date. Latest value: " - + latestValue - + " status values: " - + primary.getStatus().getValue()); - } - - var freshCopy = createFreshCopy(primary); - freshCopy - .getStatus() - .setValue(primary.getStatus() == null ? 1 : primary.getStatus().getValue() + 1); - - var updated = - PrimaryUpdateAndCacheUtils.ssaPatchAndCacheStatus(primary, freshCopy, context, cache); - latestValue = updated.getStatus().getValue(); - - return UpdateControl.noUpdate(); - } - - @Override - public List> prepareEventSources( - EventSourceContext context) { - // periodic event triggering for testing purposes - return List.of(new PeriodicTriggerEventSource<>(context.getPrimaryCache())); - } - - private StatusPatchPrimaryCacheCustomResource createFreshCopy( - StatusPatchPrimaryCacheCustomResource resource) { - var res = new StatusPatchPrimaryCacheCustomResource(); - res.setMetadata( - new ObjectMetaBuilder() - .withName(resource.getMetadata().getName()) - .withNamespace(resource.getMetadata().getNamespace()) - .build()); - res.setStatus(new StatusPatchPrimaryCacheStatus()); - - return res; - } - - @Override - public DeleteControl cleanup( - StatusPatchPrimaryCacheCustomResource resource, - Context context) - throws Exception { - cache.cleanup(resource); - return DeleteControl.defaultDelete(); - } -} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java deleted file mode 100644 index 90630c1ae8..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheSpec.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; - -public class StatusPatchPrimaryCacheSpec { - - private boolean messageInStatus = true; - - public boolean isMessageInStatus() { - return messageInStatus; - } - - public StatusPatchPrimaryCacheSpec setMessageInStatus(boolean messageInStatus) { - this.messageInStatus = messageInStatus; - return this; - } -} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java deleted file mode 100644 index 0687d5576a..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache/primarycache/StatusPatchPrimaryCacheStatus.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.javaoperatorsdk.operator.baseapi.statuscache.primarycache; - -public class StatusPatchPrimaryCacheStatus { - - private Integer value = 0; - - public Integer getValue() { - return value; - } - - public StatusPatchPrimaryCacheStatus setValue(Integer value) { - this.value = value; - return this; - } -} From e3c828fd4137c254194a0b8b09c0b4c8630844c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 16 May 2025 15:56:01 +0200 Subject: [PATCH 38/45] improve: blocklist of problematic resources for previous version annotation (#2774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: blacklist of problematic resources for previous version annotation Signed-off-by: Attila Mészáros * rename to blocklist, added javadocs Signed-off-by: Attila Mészáros * wip Signed-off-by: Attila Mészáros * remove deamonSet since was not able to reproduce the behavior Signed-off-by: Attila Mészáros * Add integration test Signed-off-by: Attila Mészáros * explanation and using Set instead of list Signed-off-by: Attila Mészáros * fix: improve javadoc Signed-off-by: Chris Laprun * refactor: rename so that relation between methods is more explicit Signed-off-by: Chris Laprun * naming Signed-off-by: Attila Mészáros --------- Signed-off-by: Attila Mészáros Signed-off-by: Chris Laprun Co-authored-by: Chris Laprun --- .../api/config/ConfigurationService.java | 32 ++++++- .../config/ConfigurationServiceOverrider.java | 15 +++ .../KubernetesDependentResource.java | 18 +++- ...BasedGenericKubernetesResourceMatcher.java | 2 +- .../prevblocklist/DeploymentDependent.java | 95 +++++++++++++++++++ .../PrevAnnotationBlockCustomResource.java | 13 +++ .../PrevAnnotationBlockReconciler.java | 34 +++++++ .../PrevAnnotationBlockReconcilerIT.java | 50 ++++++++++ .../PrevAnnotationBlockSpec.java | 15 +++ 9 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/DeploymentDependent.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockCustomResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconcilerIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockSpec.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 3ffc913c5e..18e74d29a9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -13,6 +13,8 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.StatefulSet; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.CustomResource; @@ -442,12 +444,40 @@ default Set> defaultNonSSAResource() { * * @return if special annotation should be used for dependent resource to filter events * @since 4.5.0 - * @return if special annotation should be used for dependent resource to filter events */ default boolean previousAnnotationForDependentResourcesEventFiltering() { return true; } + /** + * For dependent resources, the framework can add an annotation to filter out events resulting + * directly from the framework's operation. There are, however, some resources that do not follow + * the Kubernetes API conventions that changes in metadata should not increase the generation of + * the resource (as recorded in the {@code generation} field of the resource's {@code metadata}). + * For these resources, this convention is not respected and results in a new event for the + * framework to process. If that particular case is not handled correctly in the resource matcher, + * the framework will consider that the resource doesn't match the desired state and therefore + * triggers an update, which in turn, will re-add the annotation, thus starting the loop again, + * infinitely. + * + *

As a workaround, we automatically skip adding previous annotation for those well-known + * resources. Note that if you are sure that the matcher works for your use case, and it should in + * most instances, you can remove the resource type from the blocklist. + * + *

The consequence of adding a resource type to the set is that the framework will not use + * event filtering to prevent events, initiated by changes made by the framework itself as a + * result of its processing of dependent resources, to trigger the associated reconciler again. + * + *

Note that this method only takes effect if annotating dependent resources to prevent + * dependent resources events from triggering the associated reconciler again is activated as + * controlled by {@link #previousAnnotationForDependentResourcesEventFiltering()} + * + * @return a Set of resource classes where the previous version annotation won't be used. + */ + default Set> withPreviousAnnotationForDependentResourcesBlocklist() { + return Set.of(Deployment.class, StatefulSet.class); + } + /** * If the event logic should parse the resourceVersion to determine the ordering of dependent * resource events. This is typically not needed. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index f420be0fff..636c664f6b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -40,6 +40,7 @@ public class ConfigurationServiceOverrider { private Boolean parseResourceVersions; private Boolean useSSAToPatchPrimaryResource; private Boolean cloneSecondaryResourcesWhenGettingFromCache; + private Set> previousAnnotationUsageBlocklist; @SuppressWarnings("rawtypes") private DependentResourceFactory dependentResourceFactory; @@ -188,6 +189,12 @@ public ConfigurationServiceOverrider withCloneSecondaryResourcesWhenGettingFromC return this; } + public ConfigurationServiceOverrider withPreviousAnnotationForDependentResourcesBlocklist( + Set> blocklist) { + this.previousAnnotationUsageBlocklist = blocklist; + return this; + } + public ConfigurationService build() { return new BaseConfigurationService(original.getVersion(), cloner, client) { @Override @@ -328,6 +335,14 @@ public boolean cloneSecondaryResourcesWhenGettingFromCache() { cloneSecondaryResourcesWhenGettingFromCache, ConfigurationService::cloneSecondaryResourcesWhenGettingFromCache); } + + @Override + public Set> + withPreviousAnnotationForDependentResourcesBlocklist() { + return overriddenValueOrDefault( + previousAnnotationUsageBlocklist, + ConfigurationService::withPreviousAnnotationForDependentResourcesBlocklist); + } }; } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java index ab6e4eaca4..ebd6089aa7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResource.java @@ -41,6 +41,7 @@ public abstract class KubernetesDependentResource kubernetesDependentResourceConfig; private volatile Boolean useSSA; + private volatile Boolean usePreviousAnnotationForEventFiltering; public KubernetesDependentResource() {} @@ -165,10 +166,19 @@ protected boolean useSSA(Context

context) { } private boolean usePreviousAnnotation(Context

context) { - return context - .getControllerConfiguration() - .getConfigurationService() - .previousAnnotationForDependentResourcesEventFiltering(); + if (usePreviousAnnotationForEventFiltering == null) { + usePreviousAnnotationForEventFiltering = + context + .getControllerConfiguration() + .getConfigurationService() + .previousAnnotationForDependentResourcesEventFiltering() + && !context + .getControllerConfiguration() + .getConfigurationService() + .withPreviousAnnotationForDependentResourcesBlocklist() + .contains(this.resourceType()); + } + return usePreviousAnnotationForEventFiltering; } @Override diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java index 3c051acfb4..3fc5dbbee6 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java @@ -179,7 +179,7 @@ private Optional checkIfFieldManagerExists(R actual, String } /** Correct for known issue with SSA */ - private void sanitizeState(R actual, R desired, Map actualMap) { + protected void sanitizeState(R actual, R desired, Map actualMap) { if (actual instanceof StatefulSet actualStatefulSet && desired instanceof StatefulSet desiredStatefulSet) { var actualSpec = actualStatefulSet.getSpec(); diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/DeploymentDependent.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/DeploymentDependent.java new file mode 100644 index 0000000000..5cfb66f67e --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/DeploymentDependent.java @@ -0,0 +1,95 @@ +package io.javaoperatorsdk.operator.dependent.prevblocklist; + +import java.util.Map; + +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.PodSpecBuilder; +import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; +import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; +import io.fabric8.kubernetes.api.model.apps.DeploymentSpecBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource; +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.GenericKubernetesResourceMatcher; +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent; +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.SSABasedGenericKubernetesResourceMatcher; + +@KubernetesDependent +public class DeploymentDependent + extends CRUDKubernetesDependentResource { + + public static final String RESOURCE_NAME = "test1"; + + public DeploymentDependent() { + super(Deployment.class); + } + + @Override + protected Deployment desired( + PrevAnnotationBlockCustomResource primary, + Context context) { + + return new DeploymentBuilder() + .withMetadata( + new ObjectMetaBuilder() + .withName(primary.getMetadata().getName()) + .withNamespace(primary.getMetadata().getNamespace()) + .build()) + .withSpec( + new DeploymentSpecBuilder() + .withReplicas(1) + .withSelector( + new LabelSelectorBuilder().withMatchLabels(Map.of("app", "nginx")).build()) + .withTemplate( + new PodTemplateSpecBuilder() + .withMetadata( + new ObjectMetaBuilder().withLabels(Map.of("app", "nginx")).build()) + .withSpec( + new PodSpecBuilder() + .withContainers( + new ContainerBuilder() + .withName("nginx") + .withImage("nginx:1.14.2") + .withResources( + new ResourceRequirementsBuilder() + .withLimits(Map.of("cpu", new Quantity("2000m"))) + .build()) + .build()) + .build()) + .build()) + .build()) + .build(); + } + + // for testing purposes replicating the matching logic but with the special matcher + @Override + public Result match( + Deployment actualResource, + Deployment desired, + PrevAnnotationBlockCustomResource primary, + Context context) { + final boolean matches; + addMetadata(true, actualResource, desired, primary, context); + if (useSSA(context)) { + matches = new SSAMatcherWithoutSanitization().matches(actualResource, desired, context); + } else { + matches = + GenericKubernetesResourceMatcher.match(desired, actualResource, false, false, context) + .matched(); + } + return Result.computed(matches, desired); + } + + // using this matcher, so we are able to reproduce issue with resource limits + static class SSAMatcherWithoutSanitization + extends SSABasedGenericKubernetesResourceMatcher { + + @Override + protected void sanitizeState(R actual, R desired, Map actualMap) {} + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockCustomResource.java new file mode 100644 index 0000000000..7aa3194672 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockCustomResource.java @@ -0,0 +1,13 @@ +package io.javaoperatorsdk.operator.dependent.prevblocklist; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("pabc") +public class PrevAnnotationBlockCustomResource extends CustomResource + implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconciler.java new file mode 100644 index 0000000000..7f3dab61fe --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconciler.java @@ -0,0 +1,34 @@ +package io.javaoperatorsdk.operator.dependent.prevblocklist; + +import java.util.concurrent.atomic.AtomicInteger; + +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.api.reconciler.Workflow; +import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent; +import io.javaoperatorsdk.operator.support.TestExecutionInfoProvider; + +@Workflow(dependents = {@Dependent(type = DeploymentDependent.class)}) +@ControllerConfiguration() +public class PrevAnnotationBlockReconciler + implements Reconciler, TestExecutionInfoProvider { + + private final AtomicInteger numberOfExecutions = new AtomicInteger(0); + + public PrevAnnotationBlockReconciler() {} + + @Override + public UpdateControl reconcile( + PrevAnnotationBlockCustomResource resource, + Context context) { + numberOfExecutions.getAndIncrement(); + + return UpdateControl.noUpdate(); + } + + public int getNumberOfExecutions() { + return numberOfExecutions.get(); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconcilerIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconcilerIT.java new file mode 100644 index 0000000000..137e2ba663 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockReconcilerIT.java @@ -0,0 +1,50 @@ +package io.javaoperatorsdk.operator.dependent.prevblocklist; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class PrevAnnotationBlockReconcilerIT { + + public static final String TEST_1 = "test1"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + // Removing resource from blocklist List would result in test failure + // .withConfigurationService( + // o -> o.previousAnnotationUsageBlocklist(Collections.emptyList())) + .withReconciler(PrevAnnotationBlockReconciler.class) + .build(); + + @Test + void doNotUsePrevAnnotationForDeploymentDependent() { + extension.create(testResource(TEST_1)); + + var reconciler = extension.getReconcilerOfType(PrevAnnotationBlockReconciler.class); + await() + .pollDelay(Duration.ofMillis(200)) + .untilAsserted( + () -> { + var deployment = extension.get(Deployment.class, TEST_1); + assertThat(deployment).isNotNull(); + assertThat(reconciler.getNumberOfExecutions()).isGreaterThan(0).isLessThan(10); + }); + } + + PrevAnnotationBlockCustomResource testResource(String name) { + var res = new PrevAnnotationBlockCustomResource(); + res.setMetadata(new ObjectMetaBuilder().withName(name).build()); + res.setSpec(new PrevAnnotationBlockSpec()); + res.getSpec().setValue("value"); + return res; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockSpec.java new file mode 100644 index 0000000000..9d80e14bc1 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/prevblocklist/PrevAnnotationBlockSpec.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.dependent.prevblocklist; + +public class PrevAnnotationBlockSpec { + + private String value; + + public String getValue() { + return value; + } + + public PrevAnnotationBlockSpec setValue(String value) { + this.value = value; + return this; + } +} From 520770084efbbcd8bed2fd6b29120712fc46998a Mon Sep 17 00:00:00 2001 From: Antonio <122279781+afalhambra-hivemq@users.noreply.github.com> Date: Fri, 16 May 2025 15:56:29 +0200 Subject: [PATCH 39/45] fix: infinite resource updates due empty EnvVars (#2803) Signed-off-by: Antonio Fernandez Alhambra --- ...zer.java => PodTemplateSpecSanitizer.java} | 87 ++++++-- ...BasedGenericKubernetesResourceMatcher.java | 10 +- ...java => PodTemplateSpecSanitizerTest.java} | 197 +++++++++++++++--- .../javaoperatorsdk/operator/statefulset.yaml | 7 + 4 files changed, 254 insertions(+), 47 deletions(-) rename operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/{ResourceRequirementsSanitizer.java => PodTemplateSpecSanitizer.java} (57%) rename operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/{ResourceRequirementsSanitizerTest.java => PodTemplateSpecSanitizerTest.java} (53%) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizer.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizer.java similarity index 57% rename from operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizer.java rename to operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizer.java index 7193085b63..962059961e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizer.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizer.java @@ -2,32 +2,34 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.GenericKubernetesResource; import io.fabric8.kubernetes.api.model.PodTemplateSpec; import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.api.model.ResourceRequirements; /** - * Sanitizes the {@link ResourceRequirements} in the containers of a pair of {@link PodTemplateSpec} - * instances. + * Sanitizes the {@link ResourceRequirements} and the {@link EnvVar} in the containers of a pair of + * {@link PodTemplateSpec} instances. * *

When the sanitizer finds a mismatch in the structure of the given templates, before it gets to - * the nested resource limits and requests, it returns early without fixing the actual map. This is - * an optimization because the given templates will anyway differ at this point. This means we do - * not have to attempt to sanitize the resources for these use cases, since there will anyway be an - * update of the K8s resource. + * the nested fields, it returns early without fixing the actual map. This is an optimization + * because the given templates will anyway differ at this point. This means we do not have to + * attempt to sanitize the fields for these use cases, since there will anyway be an update of the + * K8s resource. * *

The algorithm traverses the whole template structure because we need the actual and desired - * {@link Quantity} instances to compare their numerical amount. Using the {@link + * {@link Quantity} and {@link EnvVar} instances. Using the {@link * GenericKubernetesResource#get(Map, Object...)} shortcut would need to create new instances just * for the sanitization check. */ -class ResourceRequirementsSanitizer { +class PodTemplateSpecSanitizer { - static void sanitizeResourceRequirements( + static void sanitizePodTemplateSpec( final Map actualMap, final PodTemplateSpec actualTemplate, final PodTemplateSpec desiredTemplate) { @@ -37,19 +39,19 @@ static void sanitizeResourceRequirements( if (actualTemplate.getSpec() == null || desiredTemplate.getSpec() == null) { return; } - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actualTemplate.getSpec().getInitContainers(), desiredTemplate.getSpec().getInitContainers(), "initContainers"); - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actualTemplate.getSpec().getContainers(), desiredTemplate.getSpec().getContainers(), "containers"); } - private static void sanitizeResourceRequirements( + private static void sanitizePodTemplateSpec( final Map actualMap, final List actualContainers, final List desiredContainers, @@ -57,11 +59,17 @@ private static void sanitizeResourceRequirements( int containers = desiredContainers.size(); if (containers == actualContainers.size()) { for (int containerIndex = 0; containerIndex < containers; containerIndex++) { - var desiredContainer = desiredContainers.get(containerIndex); - var actualContainer = actualContainers.get(containerIndex); + final var desiredContainer = desiredContainers.get(containerIndex); + final var actualContainer = actualContainers.get(containerIndex); if (!desiredContainer.getName().equals(actualContainer.getName())) { return; } + sanitizeEnvVars( + actualMap, + actualContainer.getEnv(), + desiredContainer.getEnv(), + containerPath, + containerIndex); sanitizeResourceRequirements( actualMap, actualContainer.getResources(), @@ -121,7 +129,7 @@ private static void sanitizeQuantities( m -> actualResource.forEach( (key, actualQuantity) -> { - var desiredQuantity = desiredResource.get(key); + final var desiredQuantity = desiredResource.get(key); if (desiredQuantity == null) { return; } @@ -138,4 +146,53 @@ private static void sanitizeQuantities( } })); } + + @SuppressWarnings("unchecked") + private static void sanitizeEnvVars( + final Map actualMap, + final List actualEnvVars, + final List desiredEnvVars, + final String containerPath, + final int containerIndex) { + if (desiredEnvVars.isEmpty() || actualEnvVars.isEmpty()) { + return; + } + Optional.ofNullable( + GenericKubernetesResource.get( + actualMap, "spec", "template", "spec", containerPath, containerIndex, "env")) + .map(List.class::cast) + .ifPresent( + envVars -> + actualEnvVars.forEach( + actualEnvVar -> { + final var actualEnvVarName = actualEnvVar.getName(); + final var actualEnvVarValue = actualEnvVar.getValue(); + // check if the actual EnvVar value string is not null or the desired EnvVar + // already contains the same EnvVar name with a non empty EnvVar value + final var isDesiredEnvVarEmpty = + hasEnvVarNoEmptyValue(actualEnvVarName, desiredEnvVars); + if (actualEnvVarValue != null || isDesiredEnvVarEmpty) { + return; + } + envVars.stream() + .filter( + envVar -> + ((Map) envVar) + .get("name") + .equals(actualEnvVarName)) + // add the actual EnvVar value with an empty string to prevent a + // resource update + .forEach(envVar -> ((Map) envVar).put("value", "")); + })); + } + + private static boolean hasEnvVarNoEmptyValue( + final String envVarName, final List envVars) { + return envVars.stream() + .anyMatch( + envVar -> + Objects.equals(envVarName, envVar.getName()) + && envVar.getValue() != null + && !envVar.getValue().isEmpty()); + } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java index 3fc5dbbee6..4954dfd17a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/SSABasedGenericKubernetesResourceMatcher.java @@ -31,7 +31,7 @@ import com.github.difflib.DiffUtils; import com.github.difflib.UnifiedDiffUtils; -import static io.javaoperatorsdk.operator.processing.dependent.kubernetes.ResourceRequirementsSanitizer.sanitizeResourceRequirements; +import static io.javaoperatorsdk.operator.processing.dependent.kubernetes.PodTemplateSpecSanitizer.sanitizePodTemplateSpec; /** * Matches the actual state on the server vs the desired state. Based on the managedFields of SSA. @@ -203,22 +203,22 @@ protected void sanitizeState(R actual, R desired, Map actualMap) } } } - sanitizeResourceRequirements(actualMap, actualSpec.getTemplate(), desiredSpec.getTemplate()); + sanitizePodTemplateSpec(actualMap, actualSpec.getTemplate(), desiredSpec.getTemplate()); } else if (actual instanceof Deployment actualDeployment && desired instanceof Deployment desiredDeployment) { - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actualDeployment.getSpec().getTemplate(), desiredDeployment.getSpec().getTemplate()); } else if (actual instanceof ReplicaSet actualReplicaSet && desired instanceof ReplicaSet desiredReplicaSet) { - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actualReplicaSet.getSpec().getTemplate(), desiredReplicaSet.getSpec().getTemplate()); } else if (actual instanceof DaemonSet actualDaemonSet && desired instanceof DaemonSet desiredDaemonSet) { - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actualDaemonSet.getSpec().getTemplate(), desiredDaemonSet.getSpec().getTemplate()); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizerTest.java similarity index 53% rename from operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizerTest.java rename to operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizerTest.java index 79f3640883..091a1a666c 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/ResourceRequirementsSanitizerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/PodTemplateSpecSanitizerTest.java @@ -1,10 +1,14 @@ package io.javaoperatorsdk.operator.processing.dependent.kubernetes; +import java.util.List; import java.util.Map; +import org.assertj.core.api.ListAssert; import org.assertj.core.api.MapAssert; import org.junit.jupiter.api.Test; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; import io.fabric8.kubernetes.api.model.GenericKubernetesResource; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -15,17 +19,17 @@ import io.fabric8.kubernetes.client.utils.KubernetesSerialization; import io.javaoperatorsdk.operator.MockKubernetesClient; -import static io.javaoperatorsdk.operator.processing.dependent.kubernetes.ResourceRequirementsSanitizer.sanitizeResourceRequirements; +import static io.javaoperatorsdk.operator.processing.dependent.kubernetes.PodTemplateSpecSanitizer.sanitizePodTemplateSpec; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verifyNoInteractions; /** - * Tests the {@link ResourceRequirementsSanitizer} with combinations of matching and mismatching K8s - * resources, using a mix of containers and init containers, as well as resource requests and - * limits. + * Tests the {@link PodTemplateSpecSanitizer} with combinations of matching and mismatching K8s + * resources, using a mix of containers and init containers, as well as resource requests and limits + * along with environment variables. */ -class ResourceRequirementsSanitizerTest { +class PodTemplateSpecSanitizerTest { private final Map actualMap = mock(); @@ -33,26 +37,26 @@ class ResourceRequirementsSanitizerTest { private final KubernetesSerialization serialization = client.getKubernetesSerialization(); @Test - void testSanitizeResourceRequirements_whenTemplateIsNull_doNothing() { + void testSanitizePodTemplateSpec_whenTemplateIsNull_doNothing() { final var template = new PodTemplateSpecBuilder().build(); - sanitizeResourceRequirements(actualMap, null, template); - sanitizeResourceRequirements(actualMap, template, null); + sanitizePodTemplateSpec(actualMap, null, template); + sanitizePodTemplateSpec(actualMap, template, null); verifyNoInteractions(actualMap); } @Test - void testSanitizeResourceRequirements_whenTemplateSpecIsNull_doNothing() { + void testSanitizePodTemplateSpec_whenTemplateSpecIsNull_doNothing() { final var template = new PodTemplateSpecBuilder().withSpec(null).build(); final var templateWithSpec = new PodTemplateSpecBuilder().withNewSpec().endSpec().build(); - sanitizeResourceRequirements(actualMap, template, templateWithSpec); - sanitizeResourceRequirements(actualMap, templateWithSpec, template); + sanitizePodTemplateSpec(actualMap, template, templateWithSpec); + sanitizePodTemplateSpec(actualMap, templateWithSpec, template); verifyNoInteractions(actualMap); } @Test - void testSanitizeResourceRequirements_whenContainerSizeMismatch_doNothing() { + void testSanitizePodTemplateSpec_whenContainerSizeMismatch_doNothing() { final var template = new PodTemplateSpecBuilder() .withNewSpec() @@ -73,13 +77,13 @@ void testSanitizeResourceRequirements_whenContainerSizeMismatch_doNothing() { .endSpec() .build(); - sanitizeResourceRequirements(actualMap, template, templateWithTwoContainers); - sanitizeResourceRequirements(actualMap, templateWithTwoContainers, template); + sanitizePodTemplateSpec(actualMap, template, templateWithTwoContainers); + sanitizePodTemplateSpec(actualMap, templateWithTwoContainers, template); verifyNoInteractions(actualMap); } @Test - void testSanitizeResourceRequirements_whenContainerNameMismatch_doNothing() { + void testSanitizePodTemplateSpec_whenContainerNameMismatch_doNothing() { final var template = new PodTemplateSpecBuilder() .withNewSpec() @@ -97,13 +101,13 @@ void testSanitizeResourceRequirements_whenContainerNameMismatch_doNothing() { .endSpec() .build(); - sanitizeResourceRequirements(actualMap, template, templateWithNewContainerName); - sanitizeResourceRequirements(actualMap, templateWithNewContainerName, template); + sanitizePodTemplateSpec(actualMap, template, templateWithNewContainerName); + sanitizePodTemplateSpec(actualMap, templateWithNewContainerName, template); verifyNoInteractions(actualMap); } @Test - void testSanitizeResourceRequirements_whenResourceIsNull_doNothing() { + void testSanitizePodTemplateSpec_whenResourceIsNull_doNothing() { final var template = new PodTemplateSpecBuilder() .withNewSpec() @@ -123,8 +127,8 @@ void testSanitizeResourceRequirements_whenResourceIsNull_doNothing() { .endSpec() .build(); - sanitizeResourceRequirements(actualMap, template, templateWithResource); - sanitizeResourceRequirements(actualMap, templateWithResource, template); + sanitizePodTemplateSpec(actualMap, template, templateWithResource); + sanitizePodTemplateSpec(actualMap, templateWithResource, template); verifyNoInteractions(actualMap); } @@ -155,7 +159,7 @@ void testSanitizeResourceRequirements_whenResourceKeyMismatch_doNothing() { } @Test - void testSanitizeResourceRequirements_whenResourcesHaveSameAmountAndFormat_doNothing() { + void testSanitizePodTemplateSpec_whenResourcesHaveSameAmountAndFormat_doNothing() { final var actualMap = sanitizeRequestsAndLimits( ContainerType.CONTAINER, @@ -168,7 +172,7 @@ void testSanitizeResourceRequirements_whenResourcesHaveSameAmountAndFormat_doNot } @Test - void testSanitizeResourceRequirements_whenResourcesHaveNumericalAmountMismatch_doNothing() { + void testSanitizePodTemplateSpec_whenResourcesHaveNumericalAmountMismatch_doNothing() { final var actualMap = sanitizeRequestsAndLimits( ContainerType.INIT_CONTAINER, @@ -200,17 +204,139 @@ void testSanitizeResourceRequirements_whenResourcesHaveNumericalAmountMismatch_d assertContainerResources(actualMap, "limits").hasSize(1).containsEntry("cpu", "4000m"); } - @SuppressWarnings("unchecked") + @Test + void testSanitizePodTemplateSpec_whenEnvVarsIsEmpty_doNothing() { + final var template = + new PodTemplateSpecBuilder() + .withNewSpec() + .addNewContainer() + .withName("test") + .endContainer() + .endSpec() + .build(); + final var templateWithEnvVars = + new PodTemplateSpecBuilder() + .withNewSpec() + .addNewContainer() + .withName("test") + .withEnv(List.of(new EnvVarBuilder().withName("FOO").withValue("foobar").build())) + .endContainer() + .endSpec() + .build(); + + sanitizePodTemplateSpec(actualMap, template, templateWithEnvVars); + sanitizePodTemplateSpec(actualMap, templateWithEnvVars, template); + verifyNoInteractions(actualMap); + } + + @Test + void testSanitizePodTemplateSpec_whenActualEnvVarValueIsNotEmpty_doNothing() { + final var actualMap = + sanitizeEnvVars( + ContainerType.CONTAINER, + List.of( + new EnvVarBuilder().withName("FOO").withValue("foo").build(), + new EnvVarBuilder().withName("BAR").withValue("bar").build()), + List.of( + new EnvVarBuilder().withName("FOO").withValue("bar").build(), + new EnvVarBuilder().withName("BAR").withValue("foo").build())); + assertContainerEnvVars(actualMap) + .hasSize(2) + .containsExactly( + Map.of("name", "FOO", "value", "foo"), Map.of("name", "BAR", "value", "bar")); + } + + @Test + void testSanitizePodTemplateSpec_whenActualAndDesiredEnvVarsAreDifferent_doNothing() { + final var actualMap = + sanitizeEnvVars( + ContainerType.INIT_CONTAINER, + List.of(new EnvVarBuilder().withName("FOO").withValue("foo").build()), + List.of(new EnvVarBuilder().withName("BAR").withValue("bar").build())); + assertInitContainerEnvVars(actualMap) + .hasSize(1) + .containsExactly(Map.of("name", "FOO", "value", "foo")); + } + + @Test + void testSanitizePodTemplateSpec_whenActualEnvVarIsEmpty_doNothing() { + final var actualMap = + sanitizeEnvVars( + ContainerType.INIT_CONTAINER, + List.of( + new EnvVarBuilder().withName("FOO").withValue("").build(), + new EnvVarBuilder().withName("BAR").withValue("").build()), + List.of( + new EnvVarBuilder().withName("FOO").withValue("foo").build(), + new EnvVarBuilder().withName("BAR").withValue("").build())); + assertInitContainerEnvVars(actualMap) + .hasSize(2) + .containsExactly(Map.of("name", "FOO", "value", ""), Map.of("name", "BAR", "value", "")); + } + + @Test + void testSanitizePodTemplateSpec_whenActualEnvVarIsNull_doNothing() { + final var actualMap = + sanitizeEnvVars( + ContainerType.CONTAINER, + List.of( + new EnvVarBuilder().withName("FOO").withValue(null).build(), + new EnvVarBuilder().withName("BAR").withValue(null).build()), + List.of( + new EnvVarBuilder().withName("FOO").withValue("foo").build(), + new EnvVarBuilder().withName("BAR").withValue(" ").build())); + assertContainerEnvVars(actualMap) + .hasSize(2) + .containsExactly(Map.of("name", "FOO"), Map.of("name", "BAR")); + } + + @Test + void + testSanitizePodTemplateSpec_whenActualEnvVarIsNull_withDesiredEnvVarEmpty_thenSanitizeActualMap() { + final var actualMap = + sanitizeEnvVars( + ContainerType.CONTAINER, + List.of( + new EnvVarBuilder().withName("FOO").withValue(null).build(), + new EnvVarBuilder().withName("BAR").withValue(null).build()), + List.of( + new EnvVarBuilder().withName("FOO").withValue("").build(), + new EnvVarBuilder().withName("BAR").withValue("").build())); + assertContainerEnvVars(actualMap) + .hasSize(2) + .containsExactly(Map.of("name", "FOO", "value", ""), Map.of("name", "BAR", "value", "")); + } + private Map sanitizeRequestsAndLimits( final ContainerType type, final Map actualRequests, final Map desiredRequests, final Map actualLimits, final Map desiredLimits) { - final var actual = createStatefulSet(type, actualRequests, actualLimits); - final var desired = createStatefulSet(type, desiredRequests, desiredLimits); + return sanitize( + type, actualRequests, desiredRequests, actualLimits, desiredLimits, List.of(), List.of()); + } + + private Map sanitizeEnvVars( + final ContainerType type, + final List actualEnvVars, + final List desiredEnvVars) { + return sanitize(type, Map.of(), Map.of(), Map.of(), Map.of(), actualEnvVars, desiredEnvVars); + } + + @SuppressWarnings("unchecked") + private Map sanitize( + final ContainerType type, + final Map actualRequests, + final Map desiredRequests, + final Map actualLimits, + final Map desiredLimits, + final List actualEnvVars, + final List desiredEnvVars) { + final var actual = createStatefulSet(type, actualRequests, actualLimits, actualEnvVars); + final var desired = createStatefulSet(type, desiredRequests, desiredLimits, desiredEnvVars); final var actualMap = serialization.convertValue(actual, Map.class); - sanitizeResourceRequirements( + sanitizePodTemplateSpec( actualMap, actual.getSpec().getTemplate(), desired.getSpec().getTemplate()); return actualMap; } @@ -223,7 +349,8 @@ private enum ContainerType { private static StatefulSet createStatefulSet( final ContainerType type, final Map requests, - final Map limits) { + final Map limits, + final List envVars) { var builder = new StatefulSetBuilder().withNewSpec().withNewTemplate().withNewSpec(); if (type == ContainerType.CONTAINER) { builder = @@ -234,6 +361,7 @@ private static StatefulSet createStatefulSet( .withRequests(requests) .withLimits(limits) .endResources() + .withEnv(envVars) .endContainer(); } else { builder = @@ -244,6 +372,7 @@ private static StatefulSet createStatefulSet( .withRequests(requests) .withLimits(limits) .endResources() + .withEnv(envVars) .endInitContainer(); } return builder.endSpec().endTemplate().endSpec().build(); @@ -262,4 +391,18 @@ private static MapAssert assertInitContainerResources( GenericKubernetesResource.>get( actualMap, "spec", "template", "spec", "initContainers", 0, "resources", resourceName)); } + + private static ListAssert> assertContainerEnvVars( + final Map actualMap) { + return assertThat( + GenericKubernetesResource.>>get( + actualMap, "spec", "template", "spec", "containers", 0, "env")); + } + + private static ListAssert> assertInitContainerEnvVars( + final Map actualMap) { + return assertThat( + GenericKubernetesResource.>>get( + actualMap, "spec", "template", "spec", "initContainers", 0, "env")); + } } diff --git a/operator-framework/src/test/resources/io/javaoperatorsdk/operator/statefulset.yaml b/operator-framework/src/test/resources/io/javaoperatorsdk/operator/statefulset.yaml index f40fbeb607..bb8a2df04b 100644 --- a/operator-framework/src/test/resources/io/javaoperatorsdk/operator/statefulset.yaml +++ b/operator-framework/src/test/resources/io/javaoperatorsdk/operator/statefulset.yaml @@ -21,6 +21,13 @@ spec: ports: - containerPort: 80 name: web + env: + - name: APP1_HOST_NAME + value: "" + - name: APP2_HOST_NAME + value: "localhost" + - name: APP3_HOST_NAME + value: " " volumeMounts: - name: www mountPath: /usr/share/nginx/html From 7edd2573b5ba47d6ff7199c754a58eb51d7d05bc Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Fri, 16 May 2025 17:44:17 +0200 Subject: [PATCH 40/45] chore(build): update version to 5.1.0-SNAPSHOT (#2805) Signed-off-by: Chris Laprun --- bootstrapper-maven-plugin/pom.xml | 2 +- caffeine-bounded-cache-support/pom.xml | 2 +- micrometer-support/pom.xml | 2 +- operator-framework-bom/pom.xml | 2 +- operator-framework-core/pom.xml | 2 +- operator-framework-junit5/pom.xml | 2 +- operator-framework/pom.xml | 2 +- pom.xml | 2 +- sample-operators/controller-namespace-deletion/pom.xml | 2 +- sample-operators/leader-election/pom.xml | 2 +- sample-operators/mysql-schema/pom.xml | 2 +- sample-operators/pom.xml | 2 +- sample-operators/tomcat-operator/pom.xml | 2 +- sample-operators/webpage/pom.xml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bootstrapper-maven-plugin/pom.xml b/bootstrapper-maven-plugin/pom.xml index 44b50a3d81..0e28960832 100644 --- a/bootstrapper-maven-plugin/pom.xml +++ b/bootstrapper-maven-plugin/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT bootstrapper diff --git a/caffeine-bounded-cache-support/pom.xml b/caffeine-bounded-cache-support/pom.xml index 924164c6cb..da33ff0a6c 100644 --- a/caffeine-bounded-cache-support/pom.xml +++ b/caffeine-bounded-cache-support/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT caffeine-bounded-cache-support diff --git a/micrometer-support/pom.xml b/micrometer-support/pom.xml index 3c568e76fd..a48312b3cd 100644 --- a/micrometer-support/pom.xml +++ b/micrometer-support/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT micrometer-support diff --git a/operator-framework-bom/pom.xml b/operator-framework-bom/pom.xml index e1cff7980d..d6b33034c4 100644 --- a/operator-framework-bom/pom.xml +++ b/operator-framework-bom/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk operator-framework-bom - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT pom Operator SDK - Bill of Materials Java SDK for implementing Kubernetes operators diff --git a/operator-framework-core/pom.xml b/operator-framework-core/pom.xml index cad50ebc32..6f9bd02ec3 100644 --- a/operator-framework-core/pom.xml +++ b/operator-framework-core/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT ../pom.xml diff --git a/operator-framework-junit5/pom.xml b/operator-framework-junit5/pom.xml index 7e68616edf..20b6550f42 100644 --- a/operator-framework-junit5/pom.xml +++ b/operator-framework-junit5/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT operator-framework-junit-5 diff --git a/operator-framework/pom.xml b/operator-framework/pom.xml index d72f91d293..24a6181134 100644 --- a/operator-framework/pom.xml +++ b/operator-framework/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT operator-framework diff --git a/pom.xml b/pom.xml index 132f51e312..fd24b11a23 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT pom Operator SDK for Java Java SDK for implementing Kubernetes operators diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml index ee0d5bb3d2..831af2e40b 100644 --- a/sample-operators/controller-namespace-deletion/pom.xml +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-controller-namespace-deletion diff --git a/sample-operators/leader-election/pom.xml b/sample-operators/leader-election/pom.xml index 4b1088fa3b..3b2c32008e 100644 --- a/sample-operators/leader-election/pom.xml +++ b/sample-operators/leader-election/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-leader-election diff --git a/sample-operators/mysql-schema/pom.xml b/sample-operators/mysql-schema/pom.xml index 92a5cb5c45..25e079e8e2 100644 --- a/sample-operators/mysql-schema/pom.xml +++ b/sample-operators/mysql-schema/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-mysql-schema-operator diff --git a/sample-operators/pom.xml b/sample-operators/pom.xml index cbe10340fc..79f1d0d034 100644 --- a/sample-operators/pom.xml +++ b/sample-operators/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk java-operator-sdk - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-operators diff --git a/sample-operators/tomcat-operator/pom.xml b/sample-operators/tomcat-operator/pom.xml index cd340c525d..9ccc91e15b 100644 --- a/sample-operators/tomcat-operator/pom.xml +++ b/sample-operators/tomcat-operator/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-tomcat-operator diff --git a/sample-operators/webpage/pom.xml b/sample-operators/webpage/pom.xml index 6ae09c835d..47e134770a 100644 --- a/sample-operators/webpage/pom.xml +++ b/sample-operators/webpage/pom.xml @@ -5,7 +5,7 @@ io.javaoperatorsdk sample-operators - 5.0.5-SNAPSHOT + 5.1.0-SNAPSHOT sample-webpage-operator From 0aeb31473dbd1b5112156d6115fe675ab207bc60 Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Mon, 19 May 2025 10:47:03 +0200 Subject: [PATCH 41/45] fix: restore backwards compatibility (#2806) Signed-off-by: Chris Laprun --- .../kubernetes/KubernetesDependentResourceConfig.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java index bcfe2f9fe6..6f626d2628 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/kubernetes/KubernetesDependentResourceConfig.java @@ -12,6 +12,13 @@ public class KubernetesDependentResourceConfig { private final InformerConfiguration informerConfig; private final SSABasedGenericKubernetesResourceMatcher matcher; + public KubernetesDependentResourceConfig( + Boolean useSSA, + boolean createResourceOnlyIfNotExistingWithSSA, + InformerConfiguration informerConfig) { + this(useSSA, createResourceOnlyIfNotExistingWithSSA, informerConfig, null); + } + public KubernetesDependentResourceConfig( Boolean useSSA, boolean createResourceOnlyIfNotExistingWithSSA, From f6f8994c25fc0ac184debe59172c90968a7adca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 20 May 2025 12:50:34 +0200 Subject: [PATCH 42/45] fix: pool size configuration (#2810) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../config/ConfigurationServiceOverrider.java | 13 +++- .../ConfigurationServiceOverriderTest.java | 61 ++++++++++++------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index 636c664f6b..be86cbe312 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -254,13 +254,20 @@ public boolean closeClientOnStop() { @Override public ExecutorService getExecutorService() { - return overriddenValueOrDefault(executorService, ConfigurationService::getExecutorService); + if (executorService != null) { + return executorService; + } else { + return super.getExecutorService(); + } } @Override public ExecutorService getWorkflowExecutorService() { - return overriddenValueOrDefault( - workflowExecutorService, ConfigurationService::getWorkflowExecutorService); + if (workflowExecutorService != null) { + return workflowExecutorService; + } else { + return super.getWorkflowExecutorService(); + } } @Override diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index 2467df75aa..4f30458d68 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -3,12 +3,14 @@ import java.time.Duration; import java.util.Optional; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import org.junit.jupiter.api.Test; import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.monitoring.Metrics; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertNotEquals; class ConfigurationServiceOverriderTest { @@ -26,30 +28,32 @@ public R clone(R object) { } }; + final BaseConfigurationService config = + new BaseConfigurationService(null) { + @Override + public boolean checkCRDAndValidateLocalModel() { + return false; + } + + @Override + public Metrics getMetrics() { + return METRICS; + } + + @Override + public Cloner getResourceCloner() { + return CLONER; + } + + @Override + public Optional getLeaderElectionConfiguration() { + return Optional.of(LEADER_ELECTION_CONFIGURATION); + } + }; + @Test void overrideShouldWork() { - final var config = - new BaseConfigurationService(null) { - @Override - public boolean checkCRDAndValidateLocalModel() { - return false; - } - - @Override - public Metrics getMetrics() { - return METRICS; - } - - @Override - public Cloner getResourceCloner() { - return CLONER; - } - - @Override - public Optional getLeaderElectionConfiguration() { - return Optional.of(LEADER_ELECTION_CONFIGURATION); - } - }; + final var overridden = new ConfigurationServiceOverrider(config) .checkingCRDAndValidateLocalModel(true) @@ -86,4 +90,17 @@ public R clone(R object) { assertNotEquals( config.reconciliationTerminationTimeout(), overridden.reconciliationTerminationTimeout()); } + + @Test + void threadCountConfiguredProperly() { + final var overridden = + new ConfigurationServiceOverrider(config) + .withConcurrentReconciliationThreads(13) + .withConcurrentWorkflowExecutorThreads(14) + .build(); + assertThat(((ThreadPoolExecutor) overridden.getExecutorService()).getMaximumPoolSize()) + .isEqualTo(13); + assertThat(((ThreadPoolExecutor) overridden.getWorkflowExecutorService()).getMaximumPoolSize()) + .isEqualTo(14); + } } From 89d12ec0451bc481c1c999a4b1bf681dcae66336 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 08:06:41 +0200 Subject: [PATCH 43/45] chore(deps): bump fabric8-client.version from 7.3.0 to 7.3.1 (#2811) Bumps `fabric8-client.version` from 7.3.0 to 7.3.1. Updates `io.fabric8:kubernetes-client-bom` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:kubernetes-server-mock` from 7.3.0 to 7.3.1 Updates `io.fabric8:kubernetes-client-api` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:kube-api-test-client-inject` from 7.3.0 to 7.3.1 Updates `io.fabric8:kubernetes-httpclient-okhttp` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:kubernetes-httpclient-vertx` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:kubernetes-httpclient-jdk` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:kubernetes-httpclient-jetty` from 7.3.0 to 7.3.1 - [Release notes](https://github.com/fabric8io/kubernetes-client/releases) - [Changelog](https://github.com/fabric8io/kubernetes-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/fabric8io/kubernetes-client/compare/v7.3.0...v7.3.1) Updates `io.fabric8:crd-generator-maven-plugin` from 7.3.0 to 7.3.1 Updates `io.fabric8:java-generator-maven-plugin` from 7.3.0 to 7.3.1 --- updated-dependencies: - dependency-name: io.fabric8:kubernetes-client-bom dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-server-mock dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-client-api dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:kube-api-test-client-inject dependency-version: 7.3.1 dependency-type: direct:development update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-httpclient-okhttp dependency-version: 7.3.1 dependency-type: direct:development update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-httpclient-vertx dependency-version: 7.3.1 dependency-type: direct:development update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-httpclient-jdk dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:kubernetes-httpclient-jetty dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:crd-generator-maven-plugin dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.fabric8:java-generator-maven-plugin dependency-version: 7.3.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd24b11a23..b758814481 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ jdk 5.12.2 - 7.3.0 + 7.3.1 2.0.12 2.24.3 5.17.0 From 494e063a3536b5b3c5eac5a7e79b98fd2e6e1d27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 08:08:11 +0200 Subject: [PATCH 44/45] chore(deps): bump org.mockito:mockito-core from 5.17.0 to 5.18.0 (#2812) Bumps [org.mockito:mockito-core](https://github.com/mockito/mockito) from 5.17.0 to 5.18.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v5.17.0...v5.18.0) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-version: 5.18.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b758814481..4452238f32 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 7.3.1 2.0.12 2.24.3 - 5.17.0 + 5.18.0 3.17.0 0.21.0 1.13.0 From 7d86c296921c85be00235ad684b3d674c43a6c44 Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Wed, 21 May 2025 10:35:08 +0200 Subject: [PATCH 45/45] docs: document annotation-based dependent resource configuration (#2809) * docs: document annotation-based dependent resource configuration Fixes #2791 Signed-off-by: Chris Laprun * fix: remove now unneeded interface Signed-off-by: Chris Laprun * docs: expand KubernetesDependentResource example Signed-off-by: Chris Laprun --------- Signed-off-by: Chris Laprun --- .../en/docs/documentation/configuration.md | 53 ++++++++++++++++--- ...ependentResourceConfigurationProvider.java | 6 --- 2 files changed, 46 insertions(+), 13 deletions(-) delete mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationProvider.java diff --git a/docs/content/en/docs/documentation/configuration.md b/docs/content/en/docs/documentation/configuration.md index 052c0e0f19..06eda5f2a2 100644 --- a/docs/content/en/docs/documentation/configuration.md +++ b/docs/content/en/docs/documentation/configuration.md @@ -113,14 +113,53 @@ for this feature. ## DependentResource-level configuration -`DependentResource` implementations can implement the `DependentResourceConfigurator` interface -to pass information to the implementation. For example, the SDK -provides specific support for the `KubernetesDependentResource`, which can be configured via the -`@KubernetesDependent` annotation. This annotation is, in turn, converted into a -`KubernetesDependentResourceConfig` instance, which is then passed to the `configureWith` method -implementation. +It is possible to define custom annotations to configure custom `DependentResource` implementations. In order to provide +such a configuration mechanism for your own `DependentResource` implementations, they must be annotated with the +`@Configured` annotation. This annotation defines 3 fields that tie everything together: + +- `by`, which specifies which annotation class will be used to configure your dependents, +- `with`, which specifies the class holding the configuration object for your dependents and +- `converter`, which specifies the `ConfigurationConverter` implementation in charge of converting the annotation + specified by the `by` field into objects of the class specified by the `with` field. + +`ConfigurationConverter` instances implement a single `configFrom` method, which will receive, as expected, the +annotation instance annotating the dependent resource instance to be configured, but it can also extract information +from the `DependentResourceSpec` instance associated with the `DependentResource` class so that metadata from it can be +used in the configuration, as well as the parent `ControllerConfiguration`, if needed. The role of +`ConfigurationConverter` implementations is to extract the annotation information, augment it with metadata from the +`DependentResourceSpec` and the configuration from the parent controller on which the dependent is defined, to finally +create the configuration object that the `DependentResource` instances will use. + +However, one last element is required to finish the configuration process: the target `DependentResource` class must +implement the `ConfiguredDependentResource` interface, parameterized with the annotation class defined by the +`@Configured` annotation `by` field. This interface is called by the framework to inject the configuration at the +appropriate time and retrieve the configuration, if it's available. + +For example, `KubernetesDependentResource`, a core implementation that the framework provides, can be configured via the +`@KubernetesDependent` annotation. This set up is configured as follows: -TODO +```java + +@Configured( + by = KubernetesDependent.class, + with = KubernetesDependentResourceConfig.class, + converter = KubernetesDependentConverter.class) +public abstract class KubernetesDependentResource + extends AbstractEventSourceHolderDependentResource> + implements ConfiguredDependentResource> { + // code omitted +} +``` + +The `@Configured` annotation specifies that `KubernetesDependentResource` instances can be configured by using the +`@KubernetesDependent` annotation, which gets converted into a `KubernetesDependentResourceConfig` object by a +`KubernetesDependentConverter`. That configuration object is then injected by the framework in the +`KubernetesDependentResource` instance, after it's been created, because the class implements the +`ConfiguredDependentResource` interface, properly parameterized. + +For more information on how to use this feature, we recommend looking at how this mechanism is implemented for +`KubernetesDependentResource` in the core framework, `SchemaDependentResource` in the samples or `CustomAnnotationDep` +in the `BaseConfigurationServiceTest` test class. ## EventSource-level configuration diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationProvider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationProvider.java deleted file mode 100644 index a0c9dc67ae..0000000000 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/dependent/DependentResourceConfigurationProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.javaoperatorsdk.operator.api.config.dependent; - -public interface DependentResourceConfigurationProvider { - @SuppressWarnings("rawtypes") - Object getConfigurationFor(DependentResourceSpec spec); -}