Reduce stack usage for overflow checks

This reduces the stack space reserved for overflow checks to 12K, split
into an 8K gap and a 4K protected region.  GC needs over 8K when running
in a stack overflow situation.

Also prevents signal runaway by detecting a signal inside code that
resulted from a signal handler invokation.  And adds a max signal count to
the SignalTest to prevent it running forever.

Also reduces the number of iterations for the InterfaceTest as this was
taking (almost) forever with the --trace option on run-test.

Bug: 15435566

Change-Id: Id4fd46f22d52d42a9eb431ca07948673e8fda694
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 8e6da74..7aabfce 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -76,8 +76,7 @@
 bool Thread::is_started_ = false;
 pthread_key_t Thread::pthread_key_self_;
 ConditionVariable* Thread::resume_cond_ = nullptr;
-const size_t Thread::kStackOverflowImplicitCheckSize = kStackOverflowProtectedSize +
-    GetStackOverflowReservedBytes(kRuntimeISA);
+const size_t Thread::kStackOverflowImplicitCheckSize = GetStackOverflowReservedBytes(kRuntimeISA);
 
 static const char* kThreadNameDuringStartup = "<native thread without managed peer>";
 
@@ -238,92 +237,48 @@
 byte dont_optimize_this;
 
 // Install a protected region in the stack.  This is used to trigger a SIGSEGV if a stack
-// overflow is detected.  It is located right below the stack_end_.  Just below that
-// is the StackOverflow reserved region used when creating the StackOverflow
-// exception.
+// overflow is detected.  It is located right below the stack_begin_.
 //
-// There is a little complexity here that deserves a special mention.  When running on the
-// host (glibc), the process's main thread's stack is allocated with a special flag
+// There is a little complexity here that deserves a special mention.  On some
+// architectures, the stack created using a VM_GROWSDOWN flag
 // to prevent memory being allocated when it's not needed.  This flag makes the
 // kernel only allocate memory for the stack by growing down in memory.  Because we
 // want to put an mprotected region far away from that at the stack top, we need
 // to make sure the pages for the stack are mapped in before we call mprotect.  We do
 // this by reading every page from the stack bottom (highest address) to the stack top.
 // We then madvise this away.
-void Thread::InstallImplicitProtection(bool is_main_stack) {
-  byte* pregion = tlsPtr_.stack_end;
-  byte* stack_lowmem = tlsPtr_.stack_begin;
-  byte* stack_top = reinterpret_cast<byte*>(reinterpret_cast<uintptr_t>(&pregion) &
+void Thread::InstallImplicitProtection() {
+  byte* pregion = tlsPtr_.stack_begin - kStackOverflowProtectedSize;
+  byte* stack_himem = tlsPtr_.stack_end;
+  byte* stack_top = reinterpret_cast<byte*>(reinterpret_cast<uintptr_t>(&stack_himem) &
       ~(kPageSize - 1));    // Page containing current top of stack.
 
-  const bool running_on_intel = (kRuntimeISA == kX86) || (kRuntimeISA == kX86_64);
+  // First remove the protection on the protected region as will want to read and
+  // write it.  This may fail (on the first attempt when the stack is not mapped)
+  // but we ignore that.
+  UnprotectStack();
 
-  if (running_on_intel) {
-    // On Intel, we need to map in the main stack.  This must be done by reading from the
-    // current stack pointer downwards as the stack is mapped using VM_GROWSDOWN
-    // in the kernel.  Any access more than a page below the current SP will cause
-    // a segv.
-    if (is_main_stack) {
-      // First we need to unprotect the protected region because this may
-      // be called more than once for a particular stack and we will crash
-      // if we try to read the protected page.
-      mprotect(pregion - kStackOverflowProtectedSize, kStackOverflowProtectedSize, PROT_READ);
+  // Map in the stack.  This must be done by reading from the
+  // current stack pointer downwards as the stack may be mapped using VM_GROWSDOWN
+  // in the kernel.  Any access more than a page below the current SP might cause
+  // a segv.
 
-      // Read every page from the high address to the low.
-      for (byte* p = stack_top; p > stack_lowmem; p -= kPageSize) {
-        dont_optimize_this = *p;
-      }
-    }
+  // Read every page from the high address to the low.
+  for (byte* p = stack_top; p > pregion; p -= kPageSize) {
+    dont_optimize_this = *p;
   }
 
-  // Check and place a marker word at the lowest usable address in the stack.  This
-  // is used to prevent a double protection.
-  constexpr uint32_t kMarker = 0xdadadada;
-  uintptr_t *marker = reinterpret_cast<uintptr_t*>(pregion);
-  if (*marker == kMarker) {
-    // The region has already been set up.  But on the main stack on the host we have
-    // removed the protected region in order to read the stack memory.  We need to put
-    // this back again.
-    if (is_main_stack && running_on_intel) {
-      mprotect(pregion - kStackOverflowProtectedSize, kStackOverflowProtectedSize, PROT_NONE);
-      madvise(stack_lowmem, stack_top - stack_lowmem, MADV_DONTNEED);
-    }
-    return;
-  }
-  // Add marker so that we can detect a second attempt to do this.
-  *marker = kMarker;
-
-  if (!running_on_intel) {
-    // Running on !Intel, stacks are mapped cleanly.  The protected region for the
-    // main stack just needs to be mapped in.  We do this by writing one byte per page.
-    for (byte* p = pregion - kStackOverflowProtectedSize;  p < pregion; p += kPageSize) {
-      *p = 0;
-    }
-  }
-
-  pregion -= kStackOverflowProtectedSize;
-
   VLOG(threads) << "installing stack protected region at " << std::hex <<
       static_cast<void*>(pregion) << " to " <<
       static_cast<void*>(pregion + kStackOverflowProtectedSize - 1);
 
-
-  if (mprotect(pregion, kStackOverflowProtectedSize, PROT_NONE) == -1) {
-    LOG(FATAL) << "Unable to create protected region in stack for implicit overflow check. Reason:"
-        << strerror(errno) << kStackOverflowProtectedSize;
-  }
+  // Protect the bottom of the stack to prevent read/write to it.
+  ProtectStack();
 
   // Tell the kernel that we won't be needing these pages any more.
   // NB. madvise will probably write zeroes into the memory (on linux it does).
-  if (is_main_stack) {
-    if (running_on_intel) {
-      // On the host, it's the whole stack (minus a page to prevent overwrite of stack top).
-      madvise(stack_lowmem, stack_top - stack_lowmem - kPageSize, MADV_DONTNEED);
-    } else {
-      // On Android, just the protected region.
-      madvise(pregion, kStackOverflowProtectedSize, MADV_DONTNEED);
-    }
-  }
+  uint32_t unwanted_size = stack_top - pregion - kPageSize;
+  madvise(pregion, unwanted_size, MADV_DONTNEED);
 }
 
 void Thread::CreateNativeThread(JNIEnv* env, jobject java_peer, size_t stack_size, bool is_daemon) {
@@ -538,7 +493,13 @@
   tlsPtr_.stack_begin = reinterpret_cast<byte*>(read_stack_base);
   tlsPtr_.stack_size = read_stack_size;
 
-  if (read_stack_size <= GetStackOverflowReservedBytes(kRuntimeISA)) {
+  // The minimum stack size we can cope with is the overflow reserved bytes (typically
+  // 8K) + the protected region size (4K) + another page (4K).  Typically this will
+  // be 8+4+4 = 16K.  The thread won't be able to do much with this stack even the GC takes
+  // between 8K and 12K.
+  uint32_t min_stack = GetStackOverflowReservedBytes(kRuntimeISA) + kStackOverflowProtectedSize
+    + 4 * KB;
+  if (read_stack_size <= min_stack) {
     LOG(FATAL) << "Attempt to attach a thread with a too-small stack (" << read_stack_size
         << " bytes)";
   }
@@ -582,20 +543,19 @@
 
   // Install the protected region if we are doing implicit overflow checks.
   if (implicit_stack_check) {
-    if (is_main_thread) {
-      size_t guardsize;
-      pthread_attr_t attributes;
-      CHECK_PTHREAD_CALL(pthread_attr_init, (&attributes), "guard size query");
-      CHECK_PTHREAD_CALL(pthread_attr_getguardsize, (&attributes, &guardsize), "guard size query");
-      CHECK_PTHREAD_CALL(pthread_attr_destroy, (&attributes), "guard size query");
-      // The main thread might have protected region at the bottom.  We need
-      // to install our own region so we need to move the limits
-      // of the stack to make room for it.
-      tlsPtr_.stack_begin += guardsize;
-      tlsPtr_.stack_end += guardsize;
-      tlsPtr_.stack_size -= guardsize;
-    }
-    InstallImplicitProtection(is_main_thread);
+    size_t guardsize;
+    pthread_attr_t attributes;
+    CHECK_PTHREAD_CALL(pthread_attr_init, (&attributes), "guard size query");
+    CHECK_PTHREAD_CALL(pthread_attr_getguardsize, (&attributes, &guardsize), "guard size query");
+    CHECK_PTHREAD_CALL(pthread_attr_destroy, (&attributes), "guard size query");
+    // The thread might have protected region at the bottom.  We need
+    // to install our own region so we need to move the limits
+    // of the stack to make room for it.
+    tlsPtr_.stack_begin += guardsize;
+    tlsPtr_.stack_end += guardsize;
+    tlsPtr_.stack_size -= guardsize;
+
+    InstallImplicitProtection();
   }
 
   // Sanity check.
@@ -2266,6 +2226,14 @@
   }
 
   tlsPtr_.stack_end = tlsPtr_.stack_begin;
+
+  // Remove the stack overflow protection if is it set up.
+  bool implicit_stack_check = !Runtime::Current()->ExplicitStackOverflowChecks();
+  if (implicit_stack_check) {
+    if (!UnprotectStack()) {
+      LOG(ERROR) << "Unable to remove stack protection for stack overflow";
+    }
+  }
 }
 
 void Thread::SetTlab(byte* start, byte* end) {
@@ -2291,4 +2259,21 @@
   return os;
 }
 
+void Thread::ProtectStack() {
+  void* pregion = tlsPtr_.stack_begin - kStackOverflowProtectedSize;
+  VLOG(threads) << "Protecting stack at " << pregion;
+  if (mprotect(pregion, kStackOverflowProtectedSize, PROT_NONE) == -1) {
+    LOG(FATAL) << "Unable to create protected region in stack for implicit overflow check. "
+        "Reason: "
+        << strerror(errno) << " size:  " << kStackOverflowProtectedSize;
+  }
+}
+
+bool Thread::UnprotectStack() {
+  void* pregion = tlsPtr_.stack_begin - kStackOverflowProtectedSize;
+  VLOG(threads) << "Unprotecting stack at " << pregion;
+  return mprotect(pregion, kStackOverflowProtectedSize, PROT_READ|PROT_WRITE) == 0;
+}
+
+
 }  // namespace art