COH 24183 Allow the Operator to resume Coherence services on Pod start-up (#506)

Allow the Operator to resume any suspended Coherence services on Pod start-up. The Operator will now only suspend Coherence services that have persistence enabled when deleting a Coherence resource, or scaling to zero.

Author: thegridman

api/v1/coherenceresourcespec_types.go

@@ -7,6 +7,9 @@
 package v1
 
 import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
 	"github.com/oracle/coherence-operator/pkg/operator"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -15,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	"strconv"
+	"strings"
 )
 
 // NOTE: This file is used to generate the CRDs use by the Operator. The CRD files should not be manually edited
@@ -99,6 +103,26 @@ type CoherenceResourceSpec struct {
 	// The default value if not specified is true.
 	// +optional
 	SuspendServicesOnShutdown *bool `json:"suspendServicesOnShutdown,omitempty"`
+	// ResumeServicesOnStartup allows the Operator to resume suspended Coherence services when
+	// the Coherence container is started. This only applies to storage enabled distributed cache
+	// services. This ensures that services that are suspended due to the shutdown of a storage
+	// tier, but those services are still running (albeit suspended) in other storage disabled
+	// deployments, will be resumed when storage comes back.
+	// Note that starting Pods with suspended partitioned cache services may stop the Pod reaching the ready state.
+	// The default value if not specified is true.
+	// +optional
+	ResumeServicesOnStartup *bool `json:"resumeServicesOnStartup,omitempty"`
+	// AutoResumeServices is a map of Coherence service names to allow more fine-grained control over
+	// which services may be auto-resumed by the operator when a Coherence Pod starts.
+	// The key to the map is the name of the Coherence service. This should be the fully qualified name
+	// if scoped services are being used in Coherence. The value is a bool, set to `true` to allow the
+	// service to be auto-resumed or `false` to not allow the service to be auto-resumed.
+	// Adding service names to this list will override any value set in `ResumeServicesOnStartup`, so if the
+	// `ResumeServicesOnStartup` field is `false` but there are service names in the `AutoResumeServices`, mapped
+	// to `true`, those services will still be resumed.
+	// Note that starting Pods with suspended partitioned cache services may stop the Pod reaching the ready state.
+	// +optional
+	AutoResumeServices map[string]bool `json:"autoResumeServices,omitempty"`
 	// SuspendServiceTimeout sets the number of seconds to wait for the service suspend
 	// call to return (the default is 60 seconds)
 	// +optional
@@ -117,7 +141,7 @@ type CoherenceResourceSpec struct {
 	// +listMapKey=name
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
-	// The extra labels to add to the all of the Pods in this deployments.
+	// The extra labels to add to the all the Pods in this deployment.
 	// Labels here will add to or override those defined for the cluster.
 	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
 	// +optional
@@ -257,14 +281,14 @@ type CoherenceResourceSpec struct {
 	// sets the serviceAccountName value in the Pod spec.
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
-	// Whether or not to auto-mount the Kubernetes API credentials for a service account
+	// Whether to auto-mount the Kubernetes API credentials for a service account
 	// +optional
 	AutomountServiceAccountToken *bool `json:"automountServiceAccountToken,omitempty"`
 	// The timeout to apply to REST requests made back to the Operator from Coherence Pods.
 	// These requests are typically to obtain site and rack information for the Pod.
 	// +optional
 	OperatorRequestTimeout *int32 `json:"operatorRequestTimeout,omitempty"`
-	// Whether or not to perform a StatusHA test on the cluster before performing an update or deletion.
+	// Whether to perform a StatusHA test on the cluster before performing an update or deletion.
 	// This field can be set to false to force through an update even when a Coherence deployment is in
 	// an unstable state.
 	// The default is true, to always check for StatusHA before updating a Coherence deployment.
@@ -871,6 +895,19 @@ func (in *CoherenceResourceSpec) CreateDefaultEnv(deployment *Coherence) []corev
 		env = append(env, corev1.EnvVar{Name: EnvVarCohIdentity, Value: deployment.Name + "@" + deployment.Namespace})
 	}
 
+	if deployment.Spec.ResumeServicesOnStartup != nil {
+		env = append(env, corev1.EnvVar{Name: EnvVarOperatorAllowResume, Value: BoolPtrToString(deployment.Spec.ResumeServicesOnStartup)})
+	}
+
+	if deployment.Spec.AutoResumeServices != nil {
+		b := new(bytes.Buffer)
+		for key, value := range deployment.Spec.AutoResumeServices {
+			_, _ = fmt.Fprintf(b, "\"%s\"=%t,", strings.Replace(key, "\"", "\\\"", -1), value)
+		}
+		value := base64.StdEncoding.EncodeToString(b.Bytes())
+		env = append(env, corev1.EnvVar{Name: EnvVarOperatorResumeServices, Value: value})
+	}
+
 	return env
 }
 

api/v1/constants.go

@@ -157,6 +157,8 @@ const (
 	EnvVarAppMainArgs                 = "COH_MAIN_ARGS"
 	EnvVarOperatorHost                = "OPERATOR_HOST"
 	EnvVarOperatorTimeout             = "OPERATOR_REQUEST_TIMEOUT"
+	EnvVarOperatorAllowResume         = "OPERATOR_ALLOW_RESUME"
+	EnvVarOperatorResumeServices      = "OPERATOR_RESUME_SERVICES"
 	EnvVarCoherenceHome               = "COHERENCE_HOME"
 	EnvVarCohDependencyModules        = "DEPENDENCY_MODULES"
 	EnvVarCohSkipVersionCheck         = "COH_SKIP_VERSION_CHECK"

api/v1/create_statefulset_coherencespec_test.go

@@ -303,3 +303,35 @@ func TestCreateStatefulSetWithCoherenceSpecWithWkaDifferentNamespace(t *testing.
 	// assert that the StatefulSet is as expected
 	assertStatefulSetCreation(t, deployment, stsExpected)
 }
+
+func TestCreateStatefulSetWithResumeServicesOnStartupTrue(t *testing.T) {
+
+	spec := coh.CoherenceResourceSpec{
+		ResumeServicesOnStartup: boolPtr(true),
+	}
+
+	// Create the test deployment
+	deployment := createTestDeployment(spec)
+	// Create expected StatefulSet
+	stsExpected := createMinimalExpectedStatefulSet(deployment)
+	addEnvVars(stsExpected, coh.ContainerNameCoherence, corev1.EnvVar{Name: coh.EnvVarOperatorAllowResume, Value: "true"})
+
+	// assert that the StatefulSet is as expected
+	assertStatefulSetCreation(t, deployment, stsExpected)
+}
+
+func TestCreateStatefulSetWithResumeServicesOnStartupFalse(t *testing.T) {
+
+	spec := coh.CoherenceResourceSpec{
+		ResumeServicesOnStartup: boolPtr(false),
+	}
+
+	// Create the test deployment
+	deployment := createTestDeployment(spec)
+	// Create expected StatefulSet
+	stsExpected := createMinimalExpectedStatefulSet(deployment)
+	addEnvVars(stsExpected, coh.ContainerNameCoherence, corev1.EnvVar{Name: coh.EnvVarOperatorAllowResume, Value: "false"})
+
+	// assert that the StatefulSet is as expected
+	assertStatefulSetCreation(t, deployment, stsExpected)
+}

api/v1/zz_generated.deepcopy.go

@@ -727,8 +727,24 @@ spec:
                       not set the application directory default value is "/app".
                     type: string
                 type: object
+              autoResumeServices:
+                additionalProperties:
+                  type: boolean
+                description: AutoResumeServices is a map of Coherence service names
+                  to allow more fine-grained control over which services may be auto-resumed
+                  by the operator when a Coherence Pod starts. The key to the map
+                  is the name of the Coherence service. This should be the fully qualified
+                  name if scoped services are being used in Coherence. The value is
+                  a bool, set to `true` to allow the service to be auto-resumed or
+                  `false` to not allow the service to be auto-resumed. Adding service
+                  names to this list will override any value set in `ResumeServicesOnStartup`,
+                  so if the `ResumeServicesOnStartup` field is `false` but there are
+                  service names in the `AutoResumeServices`, mapped to `true`, those
+                  services will still be resumed. Note that starting Pods with suspended
+                  partitioned cache services may stop the Pod reaching the ready state.
+                type: object
               automountServiceAccountToken:
-                description: Whether or not to auto-mount the Kubernetes API credentials
+                description: Whether to auto-mount the Kubernetes API credentials
                   for a service account
                 type: boolean
               cluster:
@@ -4770,11 +4786,11 @@ spec:
                 - name
                 x-kubernetes-list-type: map
               haBeforeUpdate:
-                description: Whether or not to perform a StatusHA test on the cluster
-                  before performing an update or deletion. This field can be set to
-                  false to force through an update even when a Coherence deployment
-                  is in an unstable state. The default is true, to always check for
-                  StatusHA before updating a Coherence deployment.
+                description: Whether to perform a StatusHA test on the cluster before
+                  performing an update or deletion. This field can be set to false
+                  to force through an update even when a Coherence deployment is in
+                  an unstable state. The default is true, to always check for StatusHA
+                  before updating a Coherence deployment.
                 type: boolean
               healthPort:
                 description: The port that the health check endpoint will bind to.
@@ -7582,8 +7598,8 @@ spec:
               labels:
                 additionalProperties:
                   type: string
-                description: 'The extra labels to add to the all of the Pods in this
-                  deployments. Labels here will add to or override those defined for
+                description: 'The extra labels to add to the all the Pods in this
+                  deployment. Labels here will add to or override those defined for
                   the cluster. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/'
                 type: object
               livenessProbe:
@@ -8603,6 +8619,17 @@ spec:
                       to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                     type: object
                 type: object
+              resumeServicesOnStartup:
+                description: ResumeServicesOnStartup allows the Operator to resume
+                  suspended Coherence services when the Coherence container is started.
+                  This only applies to storage enabled distributed cache services.
+                  This ensures that services that are suspended due to the shutdown
+                  of a storage tier, but those services are still running (albeit
+                  suspended) in other storage disabled deployments, will be resumed
+                  when storage comes back. Note that starting Pods with suspended
+                  partitioned cache services may stop the Pod reaching the ready state.
+                  The default value if not specified is true.
+                type: boolean
               role:
                 description: The name of the role that this deployment represents
                   in a Coherence cluster. This value will be used to set the Coherence

config/crd/bases/coherence.oracle.com_coherence.yaml

@@ -84,10 +84,12 @@ m| ports | Ports specifies additional port mappings for the Pod and additional S
 m| scaling | The configuration to control safe scaling. m| &#42;<<ScalingSpec,ScalingSpec>> | false
 m| suspendProbe | The configuration of the probe used to signal that services must be suspended before a deployment is stopped. m| &#42;<<Probe,Probe>> | false
 m| suspendServicesOnShutdown | A flag controlling whether storage enabled cache services in this deployment will be suspended before the deployment is shutdown or scaled to zero. The action of suspending storage enabled services when the whole deployment is being stopped ensures that cache services with persistence enabled will shutdown cleanly without the possibility of Coherence trying to recover and re-balance partitions as Pods are stopped. The default value if not specified is true. m| &#42;bool | false
+m| resumeServicesOnStartup | ResumeServicesOnStartup allows the Operator to resume suspended Coherence services when the Coherence container is started. This only applies to storage enabled distributed cache services. This ensures that services that are suspended due to the shutdown of a storage tier, but those services are still running (albeit suspended) in other storage disabled deployments, will be resumed when storage comes back. Note that starting Pods with suspended partitioned cache services may stop the Pod reaching the ready state. The default value if not specified is true. m| &#42;bool | false
+m| autoResumeServices | AutoResumeServices is a map of Coherence service names to allow more fine-grained control over which services may be auto-resumed by the operator when a Coherence Pod starts. The key to the map is the name of the Coherence service. This should be the fully qualified name if scoped services are being used in Coherence. The value is a bool, set to `true` to allow the service to be auto-resumed or `false` to not allow the service to be auto-resumed. Adding service names to this list will override any value set in `ResumeServicesOnStartup`, so if the `ResumeServicesOnStartup` field is `false` but there are service names in the `AutoResumeServices`, mapped to `true`, those services will still be resumed. Note that starting Pods with suspended partitioned cache services may stop the Pod reaching the ready state. m| map[string]bool | false
 m| suspendServiceTimeout | SuspendServiceTimeout sets the number of seconds to wait for the service suspend call to return (the default is 60 seconds) m| &#42;int | false
 m| startQuorum | StartQuorum controls the start-up order of this Coherence resource in relation to other Coherence resources. m| []<<StartQuorum,StartQuorum>> | false
 m| env | Env is additional environment variable mappings that will be passed to the Coherence container in the Pod. To specify extra variables add them as name value pairs the same as they would be added to a Pod containers spec. m| []https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvar-v1-core[corev1.EnvVar] | false
-m| labels | The extra labels to add to the all of the Pods in this deployments. Labels here will add to or override those defined for the cluster. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ m| map[string]string | false
+m| labels | The extra labels to add to the all the Pods in this deployment. Labels here will add to or override those defined for the cluster. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ m| map[string]string | false
 m| annotations | Annotations are free-form yaml that will be added to the store release as annotations Any annotations should be placed BELOW this annotations: key. For example if we wanted to include annotations for Prometheus it would look like this: +
  +
 annotations: +
@@ -139,9 +141,9 @@ m| hostIPC | Use the host's ipc namespace. Optional: Default to false. m| &#42;b
 m| network | Configure various networks and DNS settings for Pods in this rolw. m| &#42;<<NetworkSpec,NetworkSpec>> | false
 m| coherenceUtils | The configuration for the Coherence utils image m| &#42;<<ImageSpec,ImageSpec>> | false
 m| serviceAccountName | The name to use for the service account to use when RBAC is enabled The role bindings must already have been created as this chart does not create them it just sets the serviceAccountName value in the Pod spec. m| string | false
-m| automountServiceAccountToken | Whether or not to auto-mount the Kubernetes API credentials for a service account m| &#42;bool | false
+m| automountServiceAccountToken | Whether to auto-mount the Kubernetes API credentials for a service account m| &#42;bool | false
 m| operatorRequestTimeout | The timeout to apply to REST requests made back to the Operator from Coherence Pods. These requests are typically to obtain site and rack information for the Pod. m| &#42;int32 | false
-m| haBeforeUpdate | Whether or not to perform a StatusHA test on the cluster before performing an update or deletion. This field can be set to false to force through an update even when a Coherence deployment is in an unstable state. The default is true, to always check for StatusHA before updating a Coherence deployment. m| &#42;bool | false
+m| haBeforeUpdate | Whether to perform a StatusHA test on the cluster before performing an update or deletion. This field can be set to false to force through an update even when a Coherence deployment is in an unstable state. The default is true, to always check for StatusHA before updating a Coherence deployment. m| &#42;bool | false
 |===
 
 <<Table of Contents,Back to TOC>>

docs/about/04_coherence_spec.adoc

@@ -74,6 +74,13 @@
     </dependencies>
 
     <build>
+        <resources>
+            <resource>
+                <directory>${project.basedir}/src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+
         <!-- Use the assembly plugin to pull together the files to use to build the
              docker image under the target/docker folder -->
         <plugins>

java/coherence-operator/pom.xml

@@ -6,6 +6,10 @@
 
 package com.oracle.coherence.k8s;
 
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.Properties;
+
 import com.tangosol.net.CacheFactory;
 
 /**
@@ -18,6 +22,8 @@ public class CoherenceOperator
 
     private static final String NA = "n/a";
 
+    private static Properties properties;
+
     private String identity = NA;
 
     /**
@@ -39,4 +45,39 @@ public String getIdentity() {
     public int getNodeId() {
         return CacheFactory.getCluster().getLocalMember().getId();
     }
+
+    /**
+     * Returns the operator version.
+     *
+     * @return the operator version
+     */
+    public static String getVersion() {
+        return ensureProperties().getProperty("version", NA);
+    }
+
+    /**
+     * Print the Operator banner.
+     *
+     * @param out  the {@link PrintStream} to print he banner on
+     */
+    public static void printBanner(PrintStream out) {
+        out.printf("CoherenceOperator: Java Runner version %s\n", getVersion());
+    }
+
+    private static synchronized Properties ensureProperties() {
+        if (properties == null) {
+            Properties props = new Properties();
+            try {
+                URL url = CoherenceOperator.class.getResource("/META-INF/operator.properties");
+                if (url != null) {
+                    props.load(url.openStream());
+                }
+            }
+            catch (Throwable t) {
+                t.printStackTrace();
+            }
+            properties = props;
+        }
+        return properties;
+    }
 }