runtime: sandbox working on testnet full client

src/app/firedancer/config/default.toml

@@ -1509,9 +1509,7 @@ user = ""
     # This option cannot be enabled in production.  In development, you
     # can also launch Firedancer as a single process for with the
     # `--no-clone` argument to `firedancer-dev`.
-    #
-    # TODO: No clone should not be on by default, fix sandbox.
-    no_clone = true
+    no_clone = false
 
     # As part of security sandboxing, the dumpable bit of each process
     # is cleared with prctl( PR_SET_DUMPABLE, 0 ).  This prevents other

src/app/firedancer/topology.c

@@ -737,10 +737,10 @@ fd_topo_initialize( config_t * config ) {
   for( ulong i=0UL; i<bank_tile_cnt; i++ ) {
     fd_topo_obj_t * busy_obj = fd_topob_obj( topo, "fseq", "bank_busy" );
 
-    fd_topo_tile_t * poh_tile = &topo->tiles[ fd_topo_find_tile( topo, "poh", 0UL ) ];
     fd_topo_tile_t * pack_tile = &topo->tiles[ fd_topo_find_tile( topo, "pack", 0UL ) ];
-    fd_topob_tile_uses( topo, poh_tile, busy_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+    fd_topo_tile_t * bank_tile = &topo->tiles[ fd_topo_find_tile( topo, "bank", i ) ];
     fd_topob_tile_uses( topo, pack_tile, busy_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
+    fd_topob_tile_uses( topo, bank_tile, busy_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
     FD_TEST( fd_pod_insertf_ulong( topo->props, busy_obj->id, "bank_busy.%lu", i ) );
   }
 

src/app/shared/commands/run/run.c

@@ -256,6 +256,7 @@ main_pid_namespace( void * _args ) {
 
   pid_t child_pids[ FD_TOPO_MAX_TILES+1 ];
   char  child_names[ FD_TOPO_MAX_TILES+1 ][ 32 ];
+  ulong child_idxs[ FD_TOPO_MAX_TILES+1 ];
   struct pollfd fds[ FD_TOPO_MAX_TILES+2 ];
 
   int config_memfd = fd_config_to_memfd( config );
@@ -310,6 +311,7 @@ main_pid_namespace( void * _args ) {
     if( FD_UNLIKELY( pipe2( pipefd, O_CLOEXEC ) ) ) FD_LOG_ERR(( "pipe2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     fds[ child_cnt ] = (struct pollfd){ .fd = pipefd[ 0 ], .events = 0 };
     child_pids[ child_cnt ] = execve_tile( tile, floating_cpu_set, save_priority, config_memfd, pipefd[ 1 ] );
+    child_idxs[ child_cnt ] = i;
     if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     strncpy( child_names[ child_cnt ], tile->name, 32 );
     child_cnt++;
@@ -409,8 +411,7 @@ main_pid_namespace( void * _args ) {
         }
 
         char * tile_name = child_names[ i ];
-        ulong  tile_idx = 0UL;
-        if( FD_LIKELY( i>0UL ) ) tile_idx = (!config->is_firedancer && config->development.no_agave) ? i : i-1UL;
+        ulong  tile_idx = child_idxs[ i ];
         ulong  tile_id = config->topo.tiles[ tile_idx ].kind_id;
 
         /* Child process died, reap it to figure out exit code. */

src/disco/stem/fd_stem.c

@@ -382,7 +382,7 @@ STEM_(run1)( ulong                        in_cnt,
   for(;;) {
 
 #ifdef STEM_CALLBACK_SHOULD_SHUTDOWN
-    if( FD_UNLIKELY( STEM_CALLBACK_SHOULD_SHUTDOWN( ctx ) ) ) break;
+    if( FD_UNLIKELY( STEM_CALLBACK_SHOULD_SHUTDOWN( ctx ) ) ) { break; }
 #endif
 
     /* Do housekeeping at a low rate in the background */

src/discof/genesis/fd_genesi_tile.c

@@ -1,6 +1,5 @@
 #include "../../disco/topo/fd_topo.h"
 #include "../../funk/fd_funk.h"
-#include "generated/fd_genesi_tile_seccomp.h"
 #include "../../flamenco/types/fd_types.h"
 #include "../../ballet/lthash/fd_lthash.h"
 #include "../../ballet/sha256/fd_sha256.h"
@@ -12,6 +11,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "generated/fd_genesi_tile_seccomp.h"
 struct fd_genesi_tile {
   int fd;
 
@@ -241,10 +241,13 @@ populate_allowed_seccomp( fd_topo_t const *      topo,
                           fd_topo_tile_t const * tile,
                           ulong                  out_cnt,
                           struct sock_filter *   out ) {
-  (void)topo;
-  (void)tile;
 
-  populate_sock_filter_policy_fd_genesi_tile( out_cnt, out, (uint)fd_log_private_logfile_fd() );
+  void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
+
+  FD_SCRATCH_ALLOC_INIT( l, scratch );
+  fd_genesi_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_genesi_tile_t ), sizeof( fd_genesi_tile_t ) );
+
+  populate_sock_filter_policy_fd_genesi_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->fd );
   return sock_filter_policy_fd_genesi_tile_instr_cnt;
 }
 
@@ -253,15 +256,18 @@ populate_allowed_fds( fd_topo_t const *      topo,
                       fd_topo_tile_t const * tile,
                       ulong                  out_fds_cnt,
                       int *                  out_fds ) {
-  (void)topo;
-  (void)tile;
+  void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
+
+  FD_SCRATCH_ALLOC_INIT( l, scratch );
+  fd_genesi_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_genesi_tile_t ), sizeof( fd_genesi_tile_t ) );
 
   if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
 
   ulong out_cnt = 0UL;
   out_fds[ out_cnt++ ] = 2; /* stderr */
   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
+  out_fds[ out_cnt++ ] = ctx->fd;
   return out_cnt;
 }
 

src/discof/genesis/fd_genesi_tile.seccomppolicy

@@ -1,6 +1,6 @@
 # logfile_fd: It can be disabled by configuration, but typically tiles
 #             will open a log file on boot and write all messages there.
-unsigned int logfile_fd
+unsigned int logfile_fd, unsigned int genesis_fd
 
 # logging: all log messages are written to a file and/or pipe
 #
@@ -16,3 +16,25 @@ write: (or (eq (arg 0) 2)
 #
 # arg 0 is the file descriptor to fsync.
 fsync: (eq (arg 0) logfile_fd)
+
+# genesis
+#
+# arg 0 is the file descriptor for the genesis file
+fstat: (eq (arg 0) genesis_fd)
+
+# genesis: close the file after we are done reading from it
+#
+# arg 0 is the file descriptor for the genesis file
+close: (eq (arg 0) genesis_fd)
+
+# read: read data in from the genesis file
+#
+# arg 0 is the file descriptor for the genesis file
+read:  (eq (arg 0) genesis_fd)
+
+# shutdown: exit is called on shutdown
+#
+# arg 0 is the exit code for close
+exit
+
+exit_group

src/discof/genesis/generated/fd_genesi_tile_seccomp.h

@@ -21,34 +21,56 @@
 #else
 # error "Target architecture is unsupported by seccomp."
 #endif
-static const unsigned int sock_filter_policy_fd_genesi_tile_instr_cnt = 14;
+static const unsigned int sock_filter_policy_fd_genesi_tile_instr_cnt = 25;
 
-static void populate_sock_filter_policy_fd_genesi_tile( ulong out_cnt, struct sock_filter * out, unsigned int logfile_fd ) {
-  FD_TEST( out_cnt >= 14 );
-  struct sock_filter filter[14] = {
+static void populate_sock_filter_policy_fd_genesi_tile( ulong out_cnt, struct sock_filter * out, unsigned int logfile_fd, unsigned int genesis_fd ) {
+  FD_TEST( out_cnt >= 25 );
+  struct sock_filter filter[25] = {
     /* Check: Jump to RET_KILL_PROCESS if the script's arch != the runtime arch */
     BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) ) ),
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 10 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 21 ),
     /* loading syscall number in accumulator */
     BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, nr ) ) ),
     /* allow write based on expression */
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 2, 0 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 7, 0 ),
     /* allow fsync based on expression */
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 5, 0 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 10, 0 ),
+    /* allow fstat based on expression */
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fstat, /* check_fstat */ 11, 0 ),
+    /* allow close based on expression */
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_close, /* check_close */ 12, 0 ),
+    /* allow read based on expression */
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_read, /* check_read */ 13, 0 ),
+    /* simply allow exit */
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_exit, /* RET_ALLOW */ 15, 0 ),
+    /* simply allow exit_group */
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_exit_group, /* RET_ALLOW */ 14, 0 ),
     /* none of the syscalls matched */
-    { BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 6 },
+    { BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 12 },
 //  check_write:
     /* load syscall argument 0 in accumulator */
     BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 5, /* lbl_1 */ 0 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 11, /* lbl_1 */ 0 ),
 //  lbl_1:
     /* load syscall argument 0 in accumulator */
     BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 9, /* RET_KILL_PROCESS */ 8 ),
 //  check_fsync:
     /* load syscall argument 0 in accumulator */
     BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
-    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 1, /* RET_KILL_PROCESS */ 0 ),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 7, /* RET_KILL_PROCESS */ 6 ),
+//  check_fstat:
+    /* load syscall argument 0 in accumulator */
+    BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, genesis_fd, /* RET_ALLOW */ 5, /* RET_KILL_PROCESS */ 4 ),
+//  check_close:
+    /* load syscall argument 0 in accumulator */
+    BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, genesis_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ),
+//  check_read:
+    /* load syscall argument 0 in accumulator */
+    BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
+    BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, genesis_fd, /* RET_ALLOW */ 1, /* RET_KILL_PROCESS */ 0 ),
 //  RET_KILL_PROCESS:
     /* KILL_PROCESS is placed before ALLOW since it's the fallthrough case. */
     BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ),

src/discof/ipecho/fd_ipecho_client.c

@@ -157,10 +157,12 @@ write_conn( fd_ipecho_client_t * client,
     '\n',                   /* End of request */
   };
 
-  long written = send( client->pollfds[ conn_idx ].fd,
-                       request+peer->request_bytes_sent,
-                       sizeof(request)-peer->request_bytes_sent,
-                       MSG_NOSIGNAL );
+  long written = sendto( client->pollfds[ conn_idx ].fd,
+                         request+peer->request_bytes_sent,
+                         sizeof(request)-peer->request_bytes_sent,
+                         MSG_NOSIGNAL,
+                         NULL,
+                         0 );
   if( FD_UNLIKELY( -1==written && errno==EAGAIN ) ) return; /* No data was written, continue. */
   else if( FD_UNLIKELY( -1==written ) ) {
     close_one( client, conn_idx );
@@ -181,12 +183,12 @@ read_conn( fd_ipecho_client_t * client,
   fd_ipecho_client_peer_t * peer = &client->peers[ conn_idx ];
 
   if( FD_UNLIKELY( peer->writing ) ) return 1;
-
-  long read = recv( client->pollfds[ conn_idx ].fd,
-                    peer->response+peer->response_bytes_read,
-                    sizeof(peer->response)-peer->response_bytes_read,
-                    0 );
-
+  long read = recvfrom( client->pollfds[ conn_idx ].fd,
+                        peer->response+peer->response_bytes_read,
+                        sizeof(peer->response)-peer->response_bytes_read,
+                        0,
+                        NULL,
+                        NULL );
   if( FD_UNLIKELY( -1==read && (errno==EAGAIN || errno==EINTR) ) ) return 1;
   else if( FD_UNLIKELY( -1==read ) ) {
     close_one( client, conn_idx );
@@ -216,7 +218,7 @@ fd_ipecho_client_poll( fd_ipecho_client_t * client,
     return -1;
   }
 
-  int nfds = fd_syscall_poll( client->pollfds, 16U, 0 );
+  int nfds = fd_syscall_poll( client->pollfds, (uint)client->peer_cnt, 0 );
   if( FD_UNLIKELY( 0==nfds ) ) return 1;
   else if( FD_UNLIKELY( -1==nfds && errno==EINTR ) ) return 1;
   else if( FD_UNLIKELY( -1==nfds ) ) FD_LOG_ERR(( "poll() failed (%i-%s)", errno, strerror( errno ) ));
@@ -238,3 +240,8 @@ fd_ipecho_client_poll( fd_ipecho_client_t * client,
 
   return 1;
 }
+
+struct pollfd const *
+fd_ipecho_client_get_pollfds( fd_ipecho_client_t * client ) {
+  return client->pollfds;
+}

src/discof/ipecho/fd_ipecho_client.h

@@ -32,4 +32,7 @@ fd_ipecho_client_poll( fd_ipecho_client_t * client,
                        ushort *             shred_version,
                        int *                charge_busy );
 
+struct pollfd const *
+fd_ipecho_client_get_pollfds( fd_ipecho_client_t * client );
+
 #endif

src/discof/ipecho/fd_ipecho_server.c

@@ -44,6 +44,8 @@ typedef struct fd_ipecho_server_connection fd_ipecho_server_connection_t;
 #include "../../util/tmpl/fd_pool.c"
 
 struct fd_ipecho_server {
+  int sockfd;
+
   ushort shred_version;
 
   ulong evict_idx;
@@ -138,11 +140,11 @@ fd_ipecho_server_init( fd_ipecho_server_t * server,
                        ushort               shred_version ) {
   server->shred_version = shred_version;
 
-  int sockfd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK, 0 );
-  if( FD_UNLIKELY( -1==sockfd ) ) FD_LOG_ERR(( "socket() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
+  server->sockfd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK, 0 );
+  if( FD_UNLIKELY( -1==server->sockfd ) ) FD_LOG_ERR(( "socket() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
 
   int optval = 1;
-  if( FD_UNLIKELY( -1==setsockopt( sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof( optval ) ) ) )
+  if( FD_UNLIKELY( -1==setsockopt( server->sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof( optval ) ) ) )
     FD_LOG_ERR(( "setsockopt failed (%i-%s)", errno, strerror( errno ) ));
 
   struct sockaddr_in addr = {
@@ -151,14 +153,14 @@ fd_ipecho_server_init( fd_ipecho_server_t * server,
     .sin_addr.s_addr = address,
   };
 
-  if( FD_UNLIKELY( -1==bind( sockfd, fd_type_pun( &addr ), sizeof( addr ) ) ) ) {
+  if( FD_UNLIKELY( -1==bind( server->sockfd, fd_type_pun( &addr ), sizeof( addr ) ) ) ) {
     FD_LOG_ERR(( "bind(%i,AF_INET," FD_IP4_ADDR_FMT ":%u) failed (%i-%s)",
-                 sockfd, FD_IP4_ADDR_FMT_ARGS( address ), port,
+                 server->sockfd, FD_IP4_ADDR_FMT_ARGS( address ), port,
                  errno, fd_io_strerror( errno ) ));
   }
-  if( FD_UNLIKELY( -1==listen( sockfd, (int)server->max_connection_cnt ) ) ) FD_LOG_ERR(( "listen() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
+  if( FD_UNLIKELY( -1==listen( server->sockfd, (int)server->max_connection_cnt ) ) ) FD_LOG_ERR(( "listen() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
 
-  server->pollfds[ server->max_connection_cnt ] = (struct pollfd){ .fd = sockfd, .events = POLLIN, .revents = 0 };
+  server->pollfds[ server->max_connection_cnt ] = (struct pollfd){ .fd = server->sockfd, .events = POLLIN, .revents = 0 };
 }
 
 static inline int
@@ -291,7 +293,7 @@ write_conn( fd_ipecho_server_t * server,
 
   if( FD_LIKELY( conn->state==STATE_READING ) ) return;
 
-  long sz = send( server->pollfds[ conn_idx ].fd, conn->response_bytes+conn->response_bytes_written, sizeof(conn->response_bytes)-conn->response_bytes_written, MSG_NOSIGNAL );
+  long sz = sendto( server->pollfds[ conn_idx ].fd, conn->response_bytes+conn->response_bytes_written, sizeof(conn->response_bytes)-conn->response_bytes_written, MSG_NOSIGNAL, NULL, 0 );
   if( FD_UNLIKELY( -1==sz && errno==EAGAIN ) ) return; /* No data was written, continue. */
   if( FD_UNLIKELY( -1==sz && is_expected_network_error( errno ) ) ) {
     close_conn( server, conn_idx, CLOSE_PEER_RESET );
@@ -310,7 +312,19 @@ void
 fd_ipecho_server_poll( fd_ipecho_server_t * server,
                        int *                charge_busy,
                        int                  timeout_ms ) {
-  int nfds = fd_syscall_poll( server->pollfds, (uint)( server->max_connection_cnt+1UL ), timeout_ms );
+
+  /* Will look for first fd==-1.  fd_syscall_poll fails if any of the
+     fds passed in are ==-1.
+     TODO: There is probably a better way to do this. */
+  ulong valid_fds = 0UL;
+  for( ulong i=0UL; i<server->max_connection_cnt+1UL; i++ ) {
+    if( FD_UNLIKELY( -1==server->pollfds[ i ].fd ) ) {
+      valid_fds = i;
+      break;
+    }
+  }
+
+  int nfds = fd_syscall_poll( server->pollfds, (uint)valid_fds, timeout_ms );
   if( FD_UNLIKELY( 0==nfds ) ) return;
   else if( FD_UNLIKELY( -1==nfds && errno==EINTR ) ) return;
   else if( FD_UNLIKELY( -1==nfds ) ) FD_LOG_ERR(( "poll() failed (%i-%s)", errno, strerror( errno ) ));
@@ -334,3 +348,8 @@ fd_ipecho_server_metrics_t *
 fd_ipecho_server_metrics( fd_ipecho_server_t * server ) {
   return server->metrics;
 }
+
+int
+fd_ipecho_server_sockfd( fd_ipecho_server_t * server ) {
+  return server->sockfd;
+}

src/discof/ipecho/fd_ipecho_server.h

@@ -45,4 +45,7 @@ fd_ipecho_server_poll( fd_ipecho_server_t * server,
 fd_ipecho_server_metrics_t *
 fd_ipecho_server_metrics( fd_ipecho_server_t * server );
 
+int
+fd_ipecho_server_sockfd( fd_ipecho_server_t * server );
+
 #endif /* HEADER_fd_src_discof_ipecho_fd_ipecho_server_h */