Enhanced Cassandra health checks

build.gradle

@@ -9,6 +9,7 @@ allprojects {
 
 	repositories {
 		mavenCentral()
+		maven { url "https://repo.sjc.dsinternal.org/artifactory/datastax-public-releases-local" }
 		if (!version.endsWith('RELEASE')) {
 			maven { url "https://repo.spring.io/milestone" }
 		}

buildSrc/src/main/resources/effective-bom-settings.xml

@@ -18,6 +18,10 @@
 					<id>spring-milestone</id>
 					<url>https://repo.spring.io/milestone</url>
 				</repository>
+				<repository>
+					<id>datastax</id>
+					<url>https://repo.sjc.dsinternal.org/artifactory/datastax-public-releases-local</url>
+				</repository>
 			</repositories>
 		</profile>
 	</profiles>

spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/cassandra/CassandraHealthChecker.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.actuate.cassandra;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import com.datastax.oss.driver.api.core.ConsistencyLevel;
+import com.datastax.oss.driver.api.core.CqlSession;
+import com.datastax.oss.driver.api.core.DefaultConsistencyLevel;
+import com.datastax.oss.driver.api.core.config.DefaultDriverOption;
+import com.datastax.oss.driver.api.core.config.DriverExecutionProfile;
+import com.datastax.oss.driver.api.core.metadata.diagnostic.TokenRingDiagnostic;
+import com.datastax.oss.driver.api.core.metadata.diagnostic.TopologyDiagnostic;
+import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata;
+import com.datastax.oss.driver.internal.core.context.InternalDriverContext;
+
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.util.Assert;
+
+public class CassandraHealthChecker {
+
+	private final CqlSession session;
+
+	public CassandraHealthChecker(CqlSession session) {
+		Assert.notNull(session, "session must not be null");
+		this.session = session;
+	}
+
+	public void doHealthCheck(Health.Builder builder) {
+		TopologyDiagnostic topologyDiagnostic = createTopologyDiagnostic();
+		com.datastax.oss.driver.api.core.metadata.diagnostic.Status status = topologyDiagnostic.getStatus();
+		Map<String, Object> details = new LinkedHashMap<>();
+		details.put("topology", topologyDiagnostic.getDetails());
+		Optional<TokenRingDiagnostic> tokenRingDiagnostic = createTokenRingDiagnostic();
+		if (tokenRingDiagnostic.isPresent()) {
+			status = status.mergeWith(tokenRingDiagnostic.get().getStatus());
+			details.put("ring", tokenRingDiagnostic.get().getDetails());
+		}
+		switch (status) {
+		case AVAILABLE:
+		case PARTIALLY_AVAILABLE:
+			builder.up();
+			break;
+		default:
+			builder.down();
+			break;
+		}
+		builder.withDetails(details);
+	}
+
+	protected TopologyDiagnostic createTopologyDiagnostic() {
+		return this.session.getMetadata().generateTopologyDiagnostic();
+	}
+
+	protected Optional<TokenRingDiagnostic> createTokenRingDiagnostic() {
+		return getKeyspace().flatMap((keyspace) -> this.session.getMetadata().generateTokenRingDiagnostic(keyspace,
+				getConsistencyLevel(), getDatacenter().orElse(null)));
+	}
+
+	protected Optional<KeyspaceMetadata> getKeyspace() {
+		return this.session.getKeyspace()
+				.flatMap((keyspaceName) -> this.session.getMetadata().getKeyspace(keyspaceName));
+	}
+
+	protected ConsistencyLevel getConsistencyLevel() {
+		DriverExecutionProfile defaultProfile = this.session.getContext().getConfig().getDefaultProfile();
+		return DefaultConsistencyLevel.valueOf(defaultProfile.getString(DefaultDriverOption.REQUEST_CONSISTENCY));
+	}
+
+	protected Optional<String> getDatacenter() {
+		// local DC was set programmatically
+		InternalDriverContext context = (InternalDriverContext) this.session.getContext();
+		DriverExecutionProfile defaultProfile = context.getConfig().getDefaultProfile();
+		String localDatacenter = context.getLocalDatacenter(defaultProfile.getName());
+		if (localDatacenter == null) {
+			// local DC was specified in the config
+			localDatacenter = defaultProfile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER, null);
+		}
+		return Optional.ofNullable(localDatacenter);
+	}
+
+}

spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/cassandra/CassandraHealthIndicator.java

@@ -16,30 +16,49 @@
 
 package org.springframework.boot.actuate.cassandra;
 
-import com.datastax.oss.driver.api.core.ConsistencyLevel;
 import com.datastax.oss.driver.api.core.CqlSession;
-import com.datastax.oss.driver.api.core.cql.Row;
-import com.datastax.oss.driver.api.core.cql.SimpleStatement;
 
 import org.springframework.boot.actuate.health.AbstractHealthIndicator;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.boot.actuate.health.Status;
 import org.springframework.util.Assert;
 
 /**
  * Simple implementation of a {@link HealthIndicator} returning status information for
  * Cassandra data stores.
  *
+ * <p>
+ * The health report is comprised of two main sections:
+ *
+ * <ol>
+ * <li>Cluster topology report: checks the nodes in the cluster, and reports their state
+ * (up or down);</li>
+ * <li>Token ring availability report: checks the token ranges in the ring, and reports
+ * their state (available or unavailable, according to the configured keyspace and
+ * consistency level).</li>
+ * </ol>
+ *
+ * <p>
+ * The health status will be switched to {@link Status#DOWN} if:
+ *
+ * <ol>
+ * <li>The entire cluster is down; or</li>
+ * <li>There is at least one unavailable token range.</li>
+ * </ol>
+ *
+ * Note: cluster topology and ring availability reports are based solely on Gossip events
+ * received by the driver. But Gossip events are not 100% reliable, and therefore, the
+ * accuracy of reported health statuses should be considered best-effort only, and are not
+ * meant to replace a proper operational surveillance tool.
+ *
  * @author Julien Dubois
  * @author Alexandre Dutra
  * @since 2.0.0
  */
 public class CassandraHealthIndicator extends AbstractHealthIndicator {
 
-	private static final SimpleStatement SELECT = SimpleStatement
-			.newInstance("SELECT release_version FROM system.local").setConsistencyLevel(ConsistencyLevel.LOCAL_ONE);
-
-	private final CqlSession session;
+	private final CassandraHealthChecker cassandraHealthChecker;
 
 	/**
 	 * Create a new {@link CassandraHealthIndicator} instance.
@@ -48,16 +67,12 @@ public class CassandraHealthIndicator extends AbstractHealthIndicator {
 	public CassandraHealthIndicator(CqlSession session) {
 		super("Cassandra health check failed");
 		Assert.notNull(session, "session must not be null");
-		this.session = session;
+		this.cassandraHealthChecker = new CassandraHealthChecker(session);
 	}
 
 	@Override
 	protected void doHealthCheck(Health.Builder builder) throws Exception {
-		Row row = this.session.execute(SELECT).one();
-		builder.up();
-		if (row != null && !row.isNull(0)) {
-			builder.withDetail("version", row.getString(0));
-		}
+		this.cassandraHealthChecker.doHealthCheck(builder);
 	}
 
 }

spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/cassandra/CassandraReactiveHealthIndicator.java

@@ -15,29 +15,49 @@
  */
 package org.springframework.boot.actuate.cassandra;
 
-import com.datastax.oss.driver.api.core.ConsistencyLevel;
 import com.datastax.oss.driver.api.core.CqlSession;
-import com.datastax.oss.driver.api.core.cql.SimpleStatement;
 import reactor.core.publisher.Mono;
 
 import org.springframework.boot.actuate.health.AbstractReactiveHealthIndicator;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.ReactiveHealthIndicator;
+import org.springframework.boot.actuate.health.Status;
 import org.springframework.util.Assert;
 
 /**
  * A {@link ReactiveHealthIndicator} for Cassandra.
  *
+ * <p>
+ * The health report is comprised of two main sections:
+ *
+ * <ol>
+ * <li>Cluster topology report: checks the nodes in the cluster, and reports their state
+ * (up or down);</li>
+ * <li>Token ring availability report: checks the token ranges in the ring, and reports
+ * their state (available or unavailable, according to the configured keyspace and
+ * consistency level).</li>
+ * </ol>
+ *
+ * <p>
+ * The health status will be switched to {@link Status#DOWN} if:
+ *
+ * <ol>
+ * <li>The entire cluster is down; or</li>
+ * <li>There is at least one unavailable token range.</li>
+ * </ol>
+ *
+ * Note: cluster topology and ring availability reports are based solely on Gossip events
+ * received by the driver. But Gossip events are not 100% reliable, and therefore, the
+ * accuracy of reported health statuses should be considered best-effort only, and are not
+ * meant to replace a proper operational surveillance tool.
+ *
  * @author Artsiom Yudovin
  * @author Alexandre Dutra
  * @since 2.1.0
  */
 public class CassandraReactiveHealthIndicator extends AbstractReactiveHealthIndicator {
 
-	private static final SimpleStatement SELECT = SimpleStatement
-			.newInstance("SELECT release_version FROM system.local").setConsistencyLevel(ConsistencyLevel.LOCAL_ONE);
-
-	private final CqlSession session;
+	private final CassandraHealthChecker cassandraHealthChecker;
 
 	/**
 	 * Create a new {@link CassandraHealthIndicator} instance.
@@ -46,13 +66,13 @@ public class CassandraReactiveHealthIndicator extends AbstractReactiveHealthIndi
 	public CassandraReactiveHealthIndicator(CqlSession session) {
 		super("Cassandra health check failed");
 		Assert.notNull(session, "session must not be null");
-		this.session = session;
+		this.cassandraHealthChecker = new CassandraHealthChecker(session);
 	}
 
 	@Override
 	protected Mono<Health> doHealthCheck(Health.Builder builder) {
-		return Mono.from(this.session.executeReactive(SELECT))
-				.map((row) -> builder.up().withDetail("version", row.getString(0)).build());
+		this.cassandraHealthChecker.doHealthCheck(builder);
+		return Mono.just(builder.build());
 	}
 
 }

spring-boot-project/spring-boot-dependencies/build.gradle

@@ -159,7 +159,7 @@ bom {
 			]
 		}
 	}
-	library("Cassandra Driver", "4.5.1") {
+	library("Cassandra Driver", "4.6.0-alpha3") {
 		group("com.datastax.oss") {
 			imports = [
 				"java-driver-bom"