Add server-side metrics for TLS handshakes on side threads and on the main thread.

Rename existing (just-added) metrics to indicate that they are client-side.

Author: Gideon Glass

flow/Net2.actor.cpp

@@ -418,8 +418,10 @@ class Net2 final : public INetwork, public INetworkConnections {
 	DoubleMetricHandle countLaunchTime;
 	DoubleMetricHandle countReactTime;
 	BoolMetricHandle awakeMetric;
-	Int64MetricHandle countTLSHandshakesOnSideThreads;
-	Int64MetricHandle countTLSHandshakesOnMainThread;
+	Int64MetricHandle countClientTLSHandshakesOnSideThreads;
+	Int64MetricHandle countClientTLSHandshakesOnMainThread;
+	Int64MetricHandle countServerTLSHandshakesOnSideThreads;
+	Int64MetricHandle countServerTLSHandshakesOnMainThread;
 
 	EventMetricHandle<SlowTask> slowTaskMetric;
 
@@ -1062,8 +1064,11 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			});
 
 			// If the background handshakers are not all busy, use one
+
+			// FIXME: see comment elsewhere about making this the only path.
 			if (FLOW_KNOBS->TLS_HANDSHAKE_ALWAYS_BACKGROUND ||
 			    N2::g_net2->sslPoolHandshakesInProgress < N2::g_net2->sslHandshakerThreadsStarted) {
+				g_net2->countServerTLSHandshakesOnSideThreads++;
 				holder = Hold(&N2::g_net2->sslPoolHandshakesInProgress);
 				auto handshake =
 				    new SSLHandshakerThread::Handshake(self->ssl_sock, boost::asio::ssl::stream_base::server);
@@ -1073,6 +1078,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 				N2::g_net2->sslHandshakerPool->post(handshake);
 			} else {
 				// Otherwise use flow network thread
+				g_net2->countServerTLSHandshakesOnMainThread++;
 				BindPromise p("N2_AcceptHandshakeError"_audit, self->id);
 				p.setPeerAddr(self->getPeerAddress());
 				onHandshook = p.getFuture();
@@ -1177,9 +1183,10 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			// unpredictable system performance and reliability) is
 			// much, much higher than the cost a few hundred or
 			// thousand incremental threads.
+
 			if (FLOW_KNOBS->TLS_HANDSHAKE_ALWAYS_BACKGROUND ||
 			    N2::g_net2->sslPoolHandshakesInProgress < N2::g_net2->sslHandshakerThreadsStarted) {
-				g_net2->countTLSHandshakesOnSideThreads++;
+				g_net2->countClientTLSHandshakesOnSideThreads++;
 				holder = Hold(&N2::g_net2->sslPoolHandshakesInProgress);
 				auto handshake =
 				    new SSLHandshakerThread::Handshake(self->ssl_sock, boost::asio::ssl::stream_base::client);
@@ -1189,7 +1196,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 				N2::g_net2->sslHandshakerPool->post(handshake);
 			} else {
 				// Otherwise use flow network thread
-				g_net2->countTLSHandshakesOnMainThread++;
+				g_net2->countClientTLSHandshakesOnMainThread++;
 				BindPromise p("N2_ConnectHandshakeError"_audit, self->id);
 				p.setPeerAddr(self->getPeerAddress());
 				onHandshook = p.getFuture();
@@ -1622,8 +1629,10 @@ void Net2::initMetrics() {
 	slowTaskMetric.init("Net2.SlowTask"_sr);
 	countLaunchTime.init("Net2.CountLaunchTime"_sr);
 	countReactTime.init("Net2.CountReactTime"_sr);
-	countTLSHandshakesOnSideThreads.init("Net2.CountTLSHandshakesOnSideThreads"_sr);
-	countTLSHandshakesOnMainThread.init("Net2.CountTLSHandshakesOnMainThread"_sr);
+	countClientTLSHandshakesOnSideThreads.init("Net2.CountClientTLSHandshakesOnSideThreads"_sr);
+	countClientTLSHandshakesOnMainThread.init("Net2.CountClientTLSHandshakesOnMainThread"_sr);
+	countServerTLSHandshakesOnSideThreads.init("Net2.CountServerTLSHandshakesOnSideThreads"_sr);
+	countServerTLSHandshakesOnMainThread.init("Net2.CountServerTLSHandshakesOnMainThread"_sr);
 	taskQueue.initMetrics();
 }
 

flow/SystemMonitor.cpp

@@ -182,14 +182,23 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 			    .detail("TLSPolicyFailures",
 			            (netData.countTLSPolicyFailures - statState->networkState.countTLSPolicyFailures) /
 			                currentStats.elapsed)
-			    .detail("TLSHandshakesOnSideThreads",
-			            (netData.countTLSHandshakesOnSideThreads -
-			             statState->networkState.countTLSHandshakesOnSideThreads) /
+			    .detail("ClientTLSHandshakesOnSideThreads",
+			            (netData.countClientTLSHandshakesOnSideThreads -
+			             statState->networkState.countClientTLSHandshakesOnSideThreads) /
 			                currentStats.elapsed)
-			    .detail(
-			        "TLSHandshakesOnMainThread",
-			        (netData.countTLSHandshakesOnMainThread - statState->networkState.countTLSHandshakesOnMainThread) /
-			            currentStats.elapsed)
+			    .detail("ClientTLSHandshakesOnMainThread",
+			            (netData.countClientTLSHandshakesOnMainThread -
+			             statState->networkState.countClientTLSHandshakesOnMainThread) /
+			                currentStats.elapsed)
+			    .detail("ServerTLSHandshakesOnSideThreads",
+			            (netData.countServerTLSHandshakesOnSideThreads -
+			             statState->networkState.countServerTLSHandshakesOnSideThreads) /
+			                currentStats.elapsed)
+			    .detail("ServerTLSHandshakesOnMainThread",
+			            (netData.countServerTLSHandshakesOnMainThread -
+			             statState->networkState.countServerTLSHandshakesOnMainThread) /
+			                currentStats.elapsed)
+
 			    .trackLatest(eventName);
 
 			TraceEvent("MemoryMetrics")

flow/include/flow/SystemMonitor.h

@@ -95,8 +95,10 @@ struct NetworkData {
 	int64_t countTLSPolicyFailures;
 	double countLaunchTime;
 	double countReactTime;
-	int64_t countTLSHandshakesOnSideThreads;
-	int64_t countTLSHandshakesOnMainThread;
+	int64_t countClientTLSHandshakesOnSideThreads;
+	int64_t countClientTLSHandshakesOnMainThread;
+	int64_t countServerTLSHandshakesOnSideThreads;
+	int64_t countServerTLSHandshakesOnMainThread;
 
 	void init() {
 		bytesSent = Int64Metric::getValueOrDefault("Net2.BytesSent"_sr);
@@ -139,8 +141,14 @@ struct NetworkData {
 		countFilePageCacheHits = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsHit"_sr);
 		countFilePageCacheMisses = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMissed"_sr);
 		countFilePageCacheEvictions = Int64Metric::getValueOrDefault("EvictablePageCache.CacheEvictions"_sr);
-		countTLSHandshakesOnSideThreads = Int64Metric::getValueOrDefault("Net2.CountTLSHandshakesOnSideThreads"_sr);
-		countTLSHandshakesOnMainThread = Int64Metric::getValueOrDefault("Net2.CountTLSHandshakesOnMainThread"_sr);
+		countClientTLSHandshakesOnSideThreads =
+		    Int64Metric::getValueOrDefault("Net2.CountClientTLSHandshakesOnSideThreads"_sr);
+		countClientTLSHandshakesOnMainThread =
+		    Int64Metric::getValueOrDefault("Net2.CountClientTLSHandshakesOnMainThread"_sr);
+		countServerTLSHandshakesOnSideThreads =
+		    Int64Metric::getValueOrDefault("Net2.CountServerTLSHandshakesOnSideThreads"_sr);
+		countServerTLSHandshakesOnMainThread =
+		    Int64Metric::getValueOrDefault("Net2.CountServerTLSHandshakesOnMainThread"_sr);
 	}
 };