WL#15524 Patch #12 MGM TLS in API Nodes, mysqld, and Cluster/J

For the Ndb_cluster_connection::configure_tls() method added in
wl#15135, this patch adds the MGM TLS level as a second argument.

When some data node requires TLS, but an API node does not have
a certificate, the API node will fail quickly.

In mysqld, this adds the --ndb-mgm-tls option.

In Cluster/J, a new property com.mysql.clusterj.tls.strict
that takes numeric values 0 or 1 and defaults to 0.

Add an NDBAPI test, testMgmd -n ApiWithoutCertificate

Change-Id: If62349c288b0bef2b4594662ade3befa4dc5ed8a

Author: jdduncan

mysql-test/suite/ndb/r/ndb_basic.result

@@ -184,6 +184,7 @@ ndb_log_updated_only	#
 ndb_metadata_check	#
 ndb_metadata_check_interval	#
 ndb_metadata_sync	#
+ndb_mgm_tls	#
 ndb_mgmd_host	#
 ndb_nodeid	#
 ndb_optimization_delay	#

mysql-test/suite/ndb/r/ndb_basic_3rpl.result

@@ -184,6 +184,7 @@ ndb_log_updated_only	#
 ndb_metadata_check	#
 ndb_metadata_check_interval	#
 ndb_metadata_sync	#
+ndb_mgm_tls	#
 ndb_mgmd_host	#
 ndb_nodeid	#
 ndb_optimization_delay	#

mysql-test/suite/ndb/r/ndb_basic_4rpl.result

@@ -184,6 +184,7 @@ ndb_log_updated_only	#
 ndb_metadata_check	#
 ndb_metadata_check_interval	#
 ndb_metadata_sync	#
+ndb_mgm_tls	#
 ndb_mgmd_host	#
 ndb_nodeid	#
 ndb_optimization_delay	#

mysql-test/suite/ndb/r/ndb_basic_ndbd.result

@@ -184,6 +184,7 @@ ndb_log_updated_only	#
 ndb_metadata_check	#
 ndb_metadata_check_interval	#
 ndb_metadata_sync	#
+ndb_mgm_tls	#
 ndb_mgmd_host	#
 ndb_nodeid	#
 ndb_optimization_delay	#

storage/ndb/clusterj/clusterj-api/src/main/java/com/mysql/clusterj/Constants.java

@@ -46,6 +46,12 @@ public interface Constants {
     /** The default value of the TLS Search Path property. */
     static final String DEFAULT_PROPERTY_TLS_SEARCH_PATH = "$HOME/ndb-tls";
 
+    /*** The name of the boolean strict MGM TLS property. */
+    static final String PROPERTY_MGM_STRICT_TLS = "com.mysql.clusterj.tls.strict";
+
+    /*** The default value of the MGM TLS level property */
+    static final int DEFAULT_PROPERTY_MGM_STRICT_TLS = 0;
+
     /** The name of the initial timeout for cluster connection to connect to MGM before connecting to data nodes
      * <a href="http://dev.mysql.com/doc/ndbapi/en/ndb-ndb-cluster-connection.html#ndb-ndb-cluster-connection-set-timeout">Ndb_cluster_connection::set_timeout()</a>
      */

storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/SessionFactoryImpl.java

@@ -76,6 +76,7 @@ public class SessionFactoryImpl implements SessionFactory, Constants {
     String CLUSTER_CONNECTION_SERVICE;
     String CLUSTER_CONNECT_STRING;
     String CLUSTER_TLS_SEARCH_PATH;
+    int CLUSTER_STRICT_TLS;
     int CLUSTER_CONNECT_TIMEOUT_MGM;
     int CLUSTER_CONNECT_RETRIES;
     int CLUSTER_CONNECT_DELAY;
@@ -192,6 +193,8 @@ protected SessionFactoryImpl(Map<?, ?> props) {
         CLUSTER_CONNECT_STRING = getRequiredStringProperty(props, PROPERTY_CLUSTER_CONNECTSTRING);
         CLUSTER_TLS_SEARCH_PATH = getStringProperty(props, PROPERTY_TLS_SEARCH_PATH,
                 Constants.DEFAULT_PROPERTY_TLS_SEARCH_PATH);
+        CLUSTER_STRICT_TLS = getIntProperty(props, PROPERTY_MGM_STRICT_TLS,
+                Constants.DEFAULT_PROPERTY_MGM_STRICT_TLS);
         CLUSTER_CONNECT_RETRIES = getIntProperty(props, PROPERTY_CLUSTER_CONNECT_RETRIES,
                 Constants.DEFAULT_PROPERTY_CLUSTER_CONNECT_RETRIES);
         CLUSTER_CONNECT_TIMEOUT_MGM = getIntProperty(props, PROPERTY_CLUSTER_CONNECT_TIMEOUT_MGM,
@@ -364,7 +367,7 @@ protected ClusterConnection createClusterConnection(
         try {
             result = service.create(CLUSTER_CONNECT_STRING, nodeId, CLUSTER_CONNECT_TIMEOUT_MGM);
             result.setByteBufferPoolSizes(CLUSTER_BYTE_BUFFER_POOL_SIZES);
-            result.configureTls(CLUSTER_TLS_SEARCH_PATH, 0);
+            result.configureTls(CLUSTER_TLS_SEARCH_PATH, CLUSTER_STRICT_TLS);
             result.connect(CLUSTER_CONNECT_RETRIES, CLUSTER_CONNECT_DELAY,true);
             result.waitUntilReady(CLUSTER_CONNECT_TIMEOUT_BEFORE,CLUSTER_CONNECT_TIMEOUT_AFTER);
             // Cluster connection successful.

storage/ndb/include/ndbapi/ndb_cluster_connection.hpp

@@ -116,17 +116,21 @@ class Ndb_cluster_connection {
    * may contain absolute directories, relative directories, and environment
    * variables which will be expanded.
    *
-   * The second (int) parameter will be introduced in WL#15524.
+   * mgm_tls_level is a value 0 or 1 specifying the requirement for TLS
+   * to secure the MGM protocol connection between this node and the NDB
+   * Management server.
+   *   0 = Relaxed TLS (attempt to use TLS, but failure is OK)
+   *   1 = Strict TLS (failure to establish TLS is treated as an error)
    *
    * If the node finds active NDB TLS node keys and certificates in the seach
    * path, it will be able to connect securely to other nodes. These keys and
    * certificates can be created using the ndb_sign_keys tool.
    *
-   * If configure_tls() is not called for a connection, the search path used
-   * will be the compile-time default NDB_TLS_SEARCH_PATH.
-
+   * If configure_tls() is not called for a connection, the search path
+   * used will be the compile-time default NDB_TLS_SEARCH_PATH, and the
+   * mgm_tls_level will be 0 (relaxed).
    */
-   void configure_tls(const char * tls_search_path, int);
+   void configure_tls(const char * tls_search_path, int mgm_tls_level);
 
   /**
    * For each Ndb_cluster_connection, NDB publishes a URI in the ndbinfo

storage/ndb/plugin/ha_ndbcluster.cc

@@ -146,6 +146,7 @@ static bool opt_ndb_fully_replicated;
 static ulong opt_ndb_row_checksum;
 
 char *opt_ndb_tls_search_path;
+ulong opt_ndb_mgm_tls_level;
 
 // The version where ndbcluster uses DYNAMIC by default when creating columns
 static const ulong NDB_VERSION_DYNAMIC_IS_DEFAULT = 50711;
@@ -17614,6 +17615,16 @@ static MYSQL_SYSVAR_STR(tls_search_path,         /* name */
                         "Directory containing NDB Cluster TLS Private Keys",
                         NULL, NULL, NDB_TLS_SEARCH_PATH);
 
+static const char *tls_req_levels[] = {"relaxed", "strict", NullS};
+
+static TYPELIB mgm_tls_typelib = {array_elements(tls_req_levels) - 1, "",
+                                  tls_req_levels, nullptr};
+
+static MYSQL_SYSVAR_ENUM(mgm_tls, opt_ndb_mgm_tls_level,
+                         PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+                         "MGM TLS Requirement level", nullptr, nullptr, 0,
+                         &mgm_tls_typelib);
+
 static const int MIN_ACTIVATION_THRESHOLD = 0;
 static const int MAX_ACTIVATION_THRESHOLD = 16;
 
@@ -18451,6 +18462,7 @@ static SYS_VAR *system_variables[] = {
     MYSQL_SYSVAR(index_stat_enable),
     MYSQL_SYSVAR(index_stat_option),
     MYSQL_SYSVAR(tls_search_path),
+    MYSQL_SYSVAR(mgm_tls),
     MYSQL_SYSVAR(table_no_logging),
     MYSQL_SYSVAR(table_temporary),
     MYSQL_SYSVAR(log_bin),

storage/ndb/plugin/ha_ndbcluster_connection.cc

@@ -60,6 +60,7 @@ static uint g_pool_pos = 0;
 static mysql_mutex_t g_pool_mutex;
 
 extern char *opt_ndb_tls_search_path;
+extern ulong opt_ndb_mgm_tls_level;
 
 /**
    @brief Parse the --ndb-cluster-connection-pool-nodeids=nodeid[,nodeidN]
@@ -247,7 +248,8 @@ int ndbcluster_connect(ulong wait_connected,  // Timeout in seconds
     char buf[128];
     snprintf(buf, sizeof(buf), "mysqld --server-id=%lu", server_id);
     g_ndb_cluster_connection->set_name(buf);
-    g_ndb_cluster_connection->configure_tls(opt_ndb_tls_search_path, 0);
+    g_ndb_cluster_connection->configure_tls(opt_ndb_tls_search_path,
+                                            opt_ndb_mgm_tls_level);
     snprintf(buf, sizeof(buf), "%s%s", processinfo_path, server_id_string);
     g_ndb_cluster_connection->set_service_uri("mysql", processinfo_host,
                                               processinfo_port, buf);
@@ -309,7 +311,8 @@ int ndbcluster_connect(ulong wait_connected,  // Timeout in seconds
         snprintf(buf, sizeof(buf), "mysqld --server-id=%lu (connection %u)",
                  server_id, i + 1);
         g_pool[i]->set_name(buf);
-        g_pool[i]->configure_tls(opt_ndb_tls_search_path, 0);
+        g_pool[i]->configure_tls(opt_ndb_tls_search_path,
+                                 opt_ndb_mgm_tls_level);
         const char *uri_sep = server_id ? ";" : "?";
         snprintf(buf, sizeof(buf), "%s%s%sconnection=%u", processinfo_path,
                  server_id_string, uri_sep, i + 1);

storage/ndb/src/ndbapi/TransporterFacade.cpp

@@ -499,7 +499,8 @@ void copy(Uint32*& /*insertPtr*/, class SectionSegmentPool& /*thePool*/,
 
 int
 TransporterFacade::start_instance(NodeId nodeId,
-                                  const ndb_mgm_configuration* conf)
+                                  const ndb_mgm_configuration* conf,
+                                  bool tls_required)
 {
   DBUG_ENTER("TransporterFacade::start_instance");
 
@@ -527,6 +528,13 @@ TransporterFacade::start_instance(NodeId nodeId,
                                    m_tls_primary_api,
                                    m_mgm_tls_level);
 
+  if(tls_required && ! theTransporterRegistry->getTlsKeyManager()->ctx())
+  {
+    delete theTransporterRegistry;
+    theTransporterRegistry = nullptr;
+    DBUG_RETURN(-2);
+  }
+
   if (theClusterMgr == nullptr)
   {
     theClusterMgr = new ClusterMgr(*this);

storage/ndb/src/ndbapi/TransporterFacade.hpp

@@ -70,7 +70,7 @@ class TransporterFacade :
   TransporterFacade(GlobalDictCache *cache);
   ~TransporterFacade() override;
 
-  int start_instance(NodeId, const ndb_mgm_configuration*);
+  int start_instance(NodeId, const ndb_mgm_configuration*, bool tls_req=false);
   void stop_instance();
 
   void configure_tls(const char *, int type, bool primary, int mgmLevel);

storage/ndb/src/ndbapi/ndb_cluster_connection.cpp

@@ -942,14 +942,12 @@ Ndb_cluster_connection_impl::set_data_node_neighbour(Uint32 node)
 }
 
 void
-Ndb_cluster_connection_impl::configure_tls(const char * searchPath)
+Ndb_cluster_connection_impl::configure_tls(const char * searchPath,
+                                           int mgm_level)
 {
   bool isPrimary = ! (bool) m_main_connection;
   m_tls_search_path = strdup(searchPath);
 
-  /* A later patch will make mgm_level configurable */
-  static constexpr int mgm_level = CLIENT_TLS_RELAXED;
-
   m_config_retriever->init_mgm_tls(m_tls_search_path, ::Node::Type::Client,
                                    mgm_level);
   m_transporter_facade->api_configure_tls(m_tls_search_path, isPrimary,
@@ -1008,7 +1006,7 @@ Ndb_cluster_connection_impl::init_nodes_vector(Uint32 nodeid,
 
   for(iter.first(); iter.valid(); iter.next())
   {
-    Uint32 nodeid1, nodeid2, remoteNodeId, group= 5;
+    Uint32 nodeid1, nodeid2, tlsReq, remoteNodeId, group= 5;
     const char *remoteHostName = nullptr;
     if(iter.get(CFG_CONNECTION_NODE_1, &nodeid1)) continue;
     if(iter.get(CFG_CONNECTION_NODE_2, &nodeid2)) continue;
@@ -1042,6 +1040,10 @@ Ndb_cluster_connection_impl::init_nodes_vector(Uint32 nodeid,
        * decrease it by 1.
        */
     case CONNECTION_TYPE_TCP:{
+      // check for TLS requirement
+      iter.get(CFG_TCP_REQUIRE_TLS, &tlsReq);
+      if(tlsReq) m_tls_requirement = true;
+
       // connecting through localhost
       // check if config_hostname is local
       // check if in same location domain
@@ -1408,9 +1410,10 @@ Ndb_cluster_connection_impl::do_test()
   delete[] nodes;
 }
 
-void Ndb_cluster_connection::configure_tls(const char * searchPath, int)
+void Ndb_cluster_connection::configure_tls(const char * searchPath,
+                                           int mgmTlsLevel)
 {
-  m_impl.configure_tls(searchPath);
+  m_impl.configure_tls(searchPath, mgmTlsLevel);
 }
 
 void Ndb_cluster_connection::set_data_node_neighbour(Uint32 node)
@@ -1494,10 +1497,21 @@ int Ndb_cluster_connection_impl::connect(int no_retries,
       DBUG_RETURN(-1);
     }
 
-    if (m_transporter_facade->start_instance(nodeId, config.get()) < 0)
+    int r_start = m_transporter_facade->start_instance(nodeId,
+                                                       config.get(),
+                                                       m_tls_requirement);
+    if (r_start == -2)
+    {
+      m_latest_error = 2;
+      m_latest_error_msg.assign(
+        "TLS is required but this node does not have a valid certificate");
+    }
+
+    if (r_start < 0)
     {
       DBUG_RETURN(-1);
     }
+
     // NOTE! The config generation used by this API node could also be sent to
     // the cluster in same way as other properties with setProcessInfoUri()
     m_transporter_facade->theClusterMgr->setProcessInfoUri(m_uri_scheme.c_str(),

storage/ndb/src/ndbapi/ndb_cluster_connection_impl.hpp

@@ -134,7 +134,7 @@ class Ndb_cluster_connection_impl : public Ndb_cluster_connection
   int configure(Uint32 nodeid, const ndb_mgm_configuration *config);
   void connect_thread();
   void set_name(const char *name);
-  void configure_tls(const char * search_path);
+  void configure_tls(const char * search_path, int mgm_tls_level);
   int set_service_uri(const char *, const char *, int, const char *);
   void set_data_node_neighbour(Uint32 neighbour_node);
   void adjust_node_proximity(Uint32 node_id, Int32 adjustment);
@@ -212,6 +212,9 @@ class Ndb_cluster_connection_impl : public Ndb_cluster_connection
 
   // TLS Certificate Search Path
   const char * m_tls_search_path {nullptr};
+
+  // Some connection requires TLS
+  bool m_tls_requirement{false};
 };
 
 #endif

storage/ndb/test/ndbapi/testMgmd.cpp

@@ -1964,6 +1964,44 @@ runTestMgmdWithoutCert(NDBT_Context* ctx, NDBT_Step* step)
   return NDBT_OK;
 }
 
+int runTestApiWithoutCert(NDBT_Context* ctx, NDBT_Step* step)
+{
+  NDBT_Workingdir wd("test_tls"); // temporary working directory
+
+  BaseString cfg_path = path(wd.path(), "config.ini", nullptr);
+  Properties config = ConfigFactory::create();
+  CHECK(ConfigFactory::put(config, "ndbd", 2, "RequireTls", "true"));
+  CHECK(ConfigFactory::write_config_ini(config, cfg_path.c_str()));
+
+  CHECK(sign_tls_keys(wd));
+
+  Mgmd mgmd(1);
+  Ndbd ndbd(2);
+
+  NdbProcess::Args mgmdArgs;
+  mgmd.common_args(mgmdArgs, wd.path());
+
+  CHECK(mgmd.start(wd.path(), mgmdArgs));          // Start management node
+  CHECK(mgmd.connect(config));                     // Connect to management node
+  CHECK(mgmd.wait_confirmed_config());             // Wait for configuration
+
+  ndbd.args().add("--ndb-tls-search-path=", wd.path());
+  ndbd.start(wd.path(), mgmd.connectstring(config)); // Start data node
+  NdbMgmHandle handle = mgmd.handle() ;
+  CHECK(ndbd.wait_started(handle));
+
+  /* API has no TLS context and should fail to connect */
+  Ndb_cluster_connection con(mgmd.connectstring(config).c_str());
+  con.set_name("api_without_cert");
+  int r = con.connect(0,0,1);
+  CHECK(r == -1);
+  printf("ERROR %d: %s\n", con.get_latest_error(), con.get_latest_error_msg());
+
+  ndbd.stop();
+  mgmd.stop();
+  return NDBT_OK;
+}
+
 int
 runTestNdbdWithoutCert(NDBT_Context* ctx, NDBT_Step* step)
 {
@@ -2292,6 +2330,12 @@ TESTCASE("NdbdWithoutCertificate",
   INITIALIZER(runTestNdbdWithoutCert)
 }
 
+TESTCASE("ApiWithoutCertificate",
+         "Test API node without certificate where TRP TLS is required")
+{
+  INITIALIZER(runTestApiWithoutCert)
+}
+
 TESTCASE("NdbdWithExpiredCertificate",
         "Test data node startup with expired certificate")
 {