improve tls benchmark and inject tls verify delay

Author: kakaiu

fdbserver/networktest.actor.cpp

@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "flow/IRandom.h"
 #include "fmt/format.h"
 #include "fdbserver/NetworkTest.h"
 #include "flow/Knobs.h"
@@ -358,6 +359,10 @@ struct P2PNetworkTest {
 	// Random delay before socket writes
 	RandomIntRange waitWriteMilliseconds;
 
+	double randomCloseMaxDelay = -1; // Maximum delay before closing a connection
+
+	double probabilityNotCloseConn = 0;
+
 	double startTime;
 	int64_t bytesSent;
 	int64_t bytesReceived;
@@ -400,10 +405,13 @@ struct P2PNetworkTest {
 	               RandomIntRange requests,
 	               RandomIntRange idleMilliseconds,
 	               RandomIntRange waitReadMilliseconds,
-	               RandomIntRange waitWriteMilliseconds)
+	               RandomIntRange waitWriteMilliseconds,
+	               double randomCloseMaxDelay,
+	               double probabilityNotCloseConn)
 	  : connectionsOut(connectionsOut), requestBytes(sendMsgBytes), replyBytes(recvMsgBytes), requests(requests),
 	    idleMilliseconds(idleMilliseconds), waitReadMilliseconds(waitReadMilliseconds),
-	    waitWriteMilliseconds(waitWriteMilliseconds) {
+	    waitWriteMilliseconds(waitWriteMilliseconds), randomCloseMaxDelay(randomCloseMaxDelay),
+	    probabilityNotCloseConn(probabilityNotCloseConn) {
 		bytesSent = 0;
 		bytesReceived = 0;
 		sessionsIn = 0;
@@ -494,18 +502,30 @@ struct P2PNetworkTest {
 		return Void();
 	}
 
+	ACTOR static Future<Void> randomDelayedConnClose(P2PNetworkTest* self, Reference<IConnection> conn) {
+		wait(delay(deterministicRandom()->random01() * self->randomCloseMaxDelay / 1000.0));
+		conn->close();
+		return Void();
+	}
+
 	ACTOR static Future<Void> doSession(P2PNetworkTest* self, Reference<IConnection> conn, bool incoming) {
 		state int numRequests;
+		state Future<Void> randomClose;
 
 		try {
 			if (incoming) {
+				if (self->randomCloseMaxDelay > -1) {
+					randomClose = randomDelayedConnClose(self, conn);
+				}
 				wait(conn->acceptHandshake());
-
 				// Read the number of requests for the session
 				Standalone<StringRef> buf = wait(readMsg(self, conn));
 				ASSERT(buf.size() == sizeof(int));
 				numRequests = *(int*)buf.begin();
 			} else {
+				if (self->randomCloseMaxDelay > -1) {
+					randomClose = randomDelayedConnClose(self, conn);
+				}
 				wait(conn->connectHandshake());
 
 				// Pick the number of requests for the session and send it to remote
@@ -532,7 +552,9 @@ struct P2PNetworkTest {
 			}
 
 			wait(delay(self->idleMilliseconds.get() / 1e3));
-			conn->close();
+			if (deterministicRandom()->random01() > self->probabilityNotCloseConn) {
+				conn->close();
+			}
 
 			if (incoming) {
 				++self->sessionsIn;
@@ -653,7 +675,9 @@ TEST_CASE(":/network/p2ptest") {
 	                         params.get("requests").orDefault("10:10000"),
 	                         params.get("idleMilliseconds").orDefault("0"),
 	                         params.get("waitReadMilliseconds").orDefault("0"),
-	                         params.get("waitWriteMilliseconds").orDefault("0"));
+	                         params.get("waitWriteMilliseconds").orDefault("0"),
+	                         params.getDouble("randomCloseMaxDelay").orDefault(-1),
+	                         params.getDouble("probabilityNotCloseConn").orDefault(0.0));
 
 	wait(p2p.run());
 	return Void();

flow/Knobs.cpp

@@ -107,6 +107,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 
 	init( TLS_HANDSHAKE_ALWAYS_BACKGROUND,                   false );
 	init( INJECT_TLS_HANDSHAKE_BUSYNESS_SEC,                   0.0 );
+	init( INJECT_TLS_HANDSHAKE_VERIFY_SEC,                     0.0 );
 
 	//FlowTransport
 	init( CONNECTION_REJECTED_MESSAGE_DELAY,                   1.0 );

flow/TLSConfig.actor.cpp

@@ -42,6 +42,7 @@ TLSPolicy::~TLSPolicy() {}
 #include <sstream>
 #include <utility>
 #include <boost/asio/ssl/context.hpp>
+#include <thread>
 
 #include "flow/Platform.h"
 #include "flow/IAsyncFile.h"
@@ -147,6 +148,9 @@ void ConfigureSSLStream(Reference<TLSPolicy> policy,
                         std::function<void(bool)> callback) {
 	try {
 		stream.set_verify_callback([policy, callback](bool preverified, boost::asio::ssl::verify_context& ctx) {
+			if (FLOW_KNOBS->INJECT_TLS_HANDSHAKE_VERIFY_SEC > 0) {
+				std::this_thread::sleep_for(std::chrono::duration<double>(FLOW_KNOBS->INJECT_TLS_HANDSHAKE_VERIFY_SEC));
+			}
 			bool success = policy->verify_peer(preverified, ctx.native_handle());
 			if (!success) {
 				if (policy->on_failure)

flow/include/flow/Knobs.h

@@ -128,6 +128,7 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 
 	bool TLS_HANDSHAKE_ALWAYS_BACKGROUND;
 	double INJECT_TLS_HANDSHAKE_BUSYNESS_SEC;
+	double INJECT_TLS_HANDSHAKE_VERIFY_SEC;
 
 	int DISABLE_ASSERTS;
 	double QUEUE_MODEL_SMOOTHING_AMOUNT;