Bazel Remote Output Service

Bazel since 7.2.0 supports a remote output service. A remote output service implements a virtual filesystem for bazel-out/ with FUSE on Linux and NFS on macOS. The output service shows outputs of remote actions as if they were built locally but only fetches their contents from the server if they are read. This is an alternative to Builds Without the Bytes.

Using the bb-clientd remote output service with EngFlow

bb-clientd provides an implementation of the remote output service for Linux and macOS that is compatible with EngFlow.

1. Download a bb_clientd binary. Click the most recent workflow run from bb-clientd’s GitHub actions and download an artifact matching your operating system and architecture.
2. Create a bb-clientd configuration file, bb_client.jsonnet. It is written in the Jsonnet configuration language. Here is a simple example, where example.cluster.engflow.com, PATH_TO_ENGFLOW_TLS_CERTIFICATE, and PATH_TO_ENGFLOW_TLS_PRIVATE_KEY need to be replaced with the appropriate values for your cluster:

   local blobstoreConfig = {
     grpc: {
       address: 'example.cluster.engflow.com:443',
       tls: {
         clientKeyPair: {
           files: {
             certificatePath: 'PATH_TO_ENGFLOW_TLS_CERTIFICATE',
             privateKeyPath: 'PATH_TO_ENGFLOW_TLS_PRIVATE_KEY',
             refreshInterval: '10000s',
           },
         },
       },
       keepalive: {
         time: '60s',
         timeout: '30s',
       },
     },
   };
   
   local homeDirectory = std.extVar('HOME');
   local cacheDirectory = homeDirectory + '/.cache/bb_clientd';
   
   {
     casKeyLocationMapSizeBytes:: 512 * 1024 * 1024,
     casBlocksSizeBytes:: 100 * 1024 * 1024 * 1024,
     filePoolSizeBytes:: 100 * 1024 * 1024 * 1024,
   
     maximumMessageSizeBytes: 16 * 1024 * 1024,
     maximumTreeSizeBytes: 256 * 1024 * 1024,
   
     useNFSv4:: std.extVar('OS') == 'Darwin',
   
     blobstore: {
         actionCache: blobstoreConfig,
       contentAddressableStorage: { withLabels: {
         backend: { demultiplexing: { instanceNamePrefixes: {
           'local': { backend: { label: 'localCAS' } },
           '': { backend: { readCaching: {
             slow: {
               existenceCaching: {
                 backend: { label: 'clustersCAS' },
                 // Assume that if FindMissingBlobs() reports a blob as being
                 // present, it's going to stay around for five more minutes.
                 // This significantly reduces the combined size of
                 // FindMissingBlobs() calls generated by Bazel.
                 existenceCache: {
                   cacheSize: 1000 * 1000,
                   cacheDuration: '300s',
                   cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
                 },
               },
             },
             // On-disk cache to speed up access to recently used objects.
             fast: { label: 'localCAS' },
             replicator: {
               deduplicating: {
                 // Bazel's -j flag not only affects the number of actions
                 // executed concurrently, it also influences the concurrency
                 // of ByteStream requests. Prevent starvation by limiting
                 // the number of requests that are forwarded when cache
                 // misses occur.
                 concurrencyLimiting: {
                   base: { 'local': {} },
                   maximumConcurrency: 100,
                 },
               },
             },
           } } },
         } } },
         labels: {
           // Let the local CAS consume up to 100 GiB of disk space. A 64
           // MiB index is large enough to accommodate approximately one
           // million objects.
           localCAS: { 'local': {
             keyLocationMapOnBlockDevice: { file: {
               path: cacheDirectory + '/cas/key_location_map',
               sizeBytes: $.casKeyLocationMapSizeBytes,
             } },
             keyLocationMapMaximumGetAttempts: 8,
             keyLocationMapMaximumPutAttempts: 32,
             oldBlocks: 1,
             currentBlocks: 5,
             newBlocks: 1,
             blocksOnBlockDevice: {
               source: { file: {
                 path: cacheDirectory + '/cas/blocks',
                 sizeBytes: $.casBlocksSizeBytes,
               } },
               spareBlocks: 1,
               dataIntegrityValidationCache: {
                 cacheSize: 100000,
                 cacheDuration: '14400s',
                 cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
               },
             },
             persistent: {
               stateDirectoryPath: cacheDirectory + '/cas/persistent_state',
               minimumEpochInterval: '300s',
             },
           } },
           clustersCAS: blobstoreConfig,
         },
       } },
     },
   
     grpcServers: [{
       listenPaths: [cacheDirectory + '/grpc'],
       authenticationPolicy: { allow: {} },
       maximumReceivedMessageSizeBytes: 16 * 1024 * 1024,
     }],
   
     mount: if $.useNFSv4 then {
       mountPath: homeDirectory + '/bb_clientd',
       nfsv4: {
         enforcedLeaseTime: '120s',
         announcedLeaseTime: '60s',
         darwin: { socketPath: cacheDirectory + '/nfsv4' },
       },
     } else {
       mountPath: homeDirectory + '/bb_clientd',
       fuse: {
         directoryEntryValidity: '300s',
         inodeAttributeValidity: '300s',
       },
     },
   
     filePool: {
       blockDevice: { file: {
         path: cacheDirectory + '/filepool',
         sizeBytes: $.filePoolSizeBytes,
       } },
     },
   
     outputPathPersistency: {
       stateDirectoryPath: cacheDirectory + '/outputs',
       maximumStateFileSizeBytes: 1024 * 1024 * 1024,
       maximumStateFileAge: '604800s',
     },
   
     directoryCache: {
       maximumCount: 10000,
       maximumSizeBytes: 1024 * self.maximumCount,
       cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
     },
   
     maximumFileSystemRetryDelay: '300s',
   
     global: {
       logPaths: [cacheDirectory + '/log'],
     },
   }
   

3. Run the bb_clientd daemon:

   $ mkdir -p ~/.cache/bb_clientd ~/.cache/bb_clientd/cas/persistent_state ~/.cache/bb_clientd/ac/persistent_state ~/.cache/bb_clientd/outputs ~/bb_clientd
   $ bb_clientd bb_clientd.jsonnet
   

4. Run a build with Bazel using the remote output service:

   $ bazel build //foo --experimental_remote_output_service="unix://$HOME/.cache/bb_clientd/grpc" --experimental_remote_output_service_output_path_prefix="$HOME/bb_clientd/outputs"
   

Note it may be necessary to run umount ~/bb_clientd to rerun bb_clientd after the daemon exits.