libbinder: RPC cap transaction size at 100KB

Why?
- Android code uses -fno-exceptions and generally doesn't check for OOM
  conditions (unlike the Linux kernel itself!). Even if we check for
  allocation success, a successful allocation here may mean even a 1
  byte allocation on another thread or by the server will cause a
  failure.
- kernel binder can have by default 1MB of concurrent transactions
  at a time. A transaction of size 100KB is already exceedingly
  dangerous to the runtime, since in a big process, this could cause
  other processes to reach the limit.

In the future, we could increase this cap (lowering is potentially
difficult) or make it customizable.

Bug: 167966510
Test: binderRpcTest, binderRpcBenchmark, binder_rpc_fuzzer
Change-Id: Ia215f1a00412654ce08e6bced14d4da4a0a46987
diff --git a/libs/binder/RpcState.cpp b/libs/binder/RpcState.cpp
index 20fdbfe..2ba9fa2 100644
--- a/libs/binder/RpcState.cpp
+++ b/libs/binder/RpcState.cpp
@@ -182,6 +182,27 @@
     }
 }
 
+RpcState::CommandData::CommandData(size_t size) : mSize(size) {
+    // The maximum size for regular binder is 1MB for all concurrent
+    // transactions. A very small proportion of transactions are even
+    // larger than a page, but we need to avoid allocating too much
+    // data on behalf of an arbitrary client, or we could risk being in
+    // a position where a single additional allocation could run out of
+    // memory.
+    //
+    // Note, this limit may not reflect the total amount of data allocated for a
+    // transaction (in some cases, additional fixed size amounts are added),
+    // though for rough consistency, we should avoid cases where this data type
+    // is used for multiple dynamic allocations for a single transaction.
+    constexpr size_t kMaxTransactionAllocation = 100 * 1000;
+    if (size == 0) return;
+    if (size > kMaxTransactionAllocation) {
+        ALOGW("Transaction requested too much data allocation %zu", size);
+        return;
+    }
+    mData.reset(new (std::nothrow) uint8_t[size]);
+}
+
 bool RpcState::rpcSend(const base::unique_fd& fd, const char* what, const void* data, size_t size) {
     LOG_RPC_DETAIL("Sending %s on fd %d: %s", what, fd.get(), hexString(data, size).c_str());
 
@@ -326,7 +347,7 @@
             .asyncNumber = asyncNumber,
     };
 
-    ByteVec transactionData(sizeof(RpcWireTransaction) + data.dataSize());
+    CommandData transactionData(sizeof(RpcWireTransaction) + data.dataSize());
     if (!transactionData.valid()) {
         return NO_MEMORY;
     }
@@ -383,7 +404,7 @@
         if (status != OK) return status;
     }
 
-    ByteVec data(command.bodySize);
+    CommandData data(command.bodySize);
     if (!data.valid()) {
         return NO_MEMORY;
     }
@@ -469,7 +490,7 @@
                                    const RpcWireHeader& command) {
     LOG_ALWAYS_FATAL_IF(command.command != RPC_COMMAND_TRANSACT, "command: %d", command.command);
 
-    ByteVec transactionData(command.bodySize);
+    CommandData transactionData(command.bodySize);
     if (!transactionData.valid()) {
         return NO_MEMORY;
     }
@@ -490,7 +511,7 @@
 }
 
 status_t RpcState::processTransactInternal(const base::unique_fd& fd, const sp<RpcSession>& session,
-                                           ByteVec transactionData) {
+                                           CommandData transactionData) {
     if (transactionData.size() < sizeof(RpcWireTransaction)) {
         ALOGE("Expecting %zu but got %zu bytes for RpcWireTransaction. Terminating!",
               sizeof(RpcWireTransaction), transactionData.size());
@@ -640,7 +661,7 @@
                 // justification for const_cast (consider avoiding priority_queue):
                 // - AsyncTodo operator< doesn't depend on 'data' object
                 // - gotta go fast
-                ByteVec data = std::move(
+                CommandData data = std::move(
                         const_cast<BinderNode::AsyncTodo&>(it->second.asyncTodo.top()).data);
                 it->second.asyncTodo.pop();
                 _l.unlock();
@@ -654,7 +675,7 @@
             .status = replyStatus,
     };
 
-    ByteVec replyData(sizeof(RpcWireReply) + reply.dataSize());
+    CommandData replyData(sizeof(RpcWireReply) + reply.dataSize());
     if (!replyData.valid()) {
         return NO_MEMORY;
     }
@@ -684,7 +705,7 @@
 status_t RpcState::processDecStrong(const base::unique_fd& fd, const RpcWireHeader& command) {
     LOG_ALWAYS_FATAL_IF(command.command != RPC_COMMAND_DEC_STRONG, "command: %d", command.command);
 
-    ByteVec commandData(command.bodySize);
+    CommandData commandData(command.bodySize);
     if (!commandData.valid()) {
         return NO_MEMORY;
     }
diff --git a/libs/binder/RpcState.h b/libs/binder/RpcState.h
index 83d0344..31f8a22 100644
--- a/libs/binder/RpcState.h
+++ b/libs/binder/RpcState.h
@@ -101,10 +101,10 @@
      */
     void terminate();
 
-    // alternative to std::vector<uint8_t> that doesn't abort on too big of allocations
-    struct ByteVec {
-        explicit ByteVec(size_t size)
-              : mData(size > 0 ? new (std::nothrow) uint8_t[size] : nullptr), mSize(size) {}
+    // Alternative to std::vector<uint8_t> that doesn't abort on allocation failure and caps
+    // large allocations to avoid being requested from allocating too much data.
+    struct CommandData {
+        explicit CommandData(size_t size);
         bool valid() { return mSize == 0 || mData != nullptr; }
         size_t size() { return mSize; }
         uint8_t* data() { return mData.get(); }
@@ -128,7 +128,7 @@
                                            const RpcWireHeader& command);
     [[nodiscard]] status_t processTransactInternal(const base::unique_fd& fd,
                                                    const sp<RpcSession>& session,
-                                                   ByteVec transactionData);
+                                                   CommandData transactionData);
     [[nodiscard]] status_t processDecStrong(const base::unique_fd& fd,
                                             const RpcWireHeader& command);
 
@@ -163,7 +163,7 @@
 
         // async transaction queue, _only_ for local binder
         struct AsyncTodo {
-            ByteVec data;
+            CommandData data;
             uint64_t asyncNumber = 0;
 
             bool operator<(const AsyncTodo& o) const {