diff --git a/Android.bp b/Android.bp
index fd6ef1d..1646c42 100644
--- a/Android.bp
+++ b/Android.bp
@@ -28,6 +28,7 @@
     name: "vold_default_libs",
 
     static_libs: [
+        "libasync_safe",
         "libavb",
         "libbootloader_message",
         "libdm",
@@ -43,6 +44,7 @@
     shared_libs: [
         "android.hardware.keymaster@3.0",
         "android.hardware.keymaster@4.0",
+        "android.hardware.keymaster@4.1",
         "android.hardware.boot@1.0",
         "libbase",
         "libbinder",
@@ -57,6 +59,7 @@
         "libincfs",
         "libhidlbase",
         "libkeymaster4support",
+        "libkeymaster4_1support",
         "libkeyutils",
         "liblog",
         "liblogwrap",
@@ -231,10 +234,13 @@
 
         "android.hardware.keymaster@3.0",
         "android.hardware.keymaster@4.0",
+        "android.hardware.keymaster@4.1",
         "libhardware",
         "libhardware_legacy",
         "libhidlbase",
         "libkeymaster4support",
+        "libkeymaster4_1support",
+        "libutils",
     ],
 }
 
@@ -271,6 +277,7 @@
     srcs: [
         "binder/android/os/IVold.aidl",
         "binder/android/os/IVoldListener.aidl",
+        "binder/android/os/IVoldMountCallback.aidl",
         "binder/android/os/IVoldTaskListener.aidl",
     ],
     path: "binder",
diff --git a/FsCrypt.cpp b/FsCrypt.cpp
index c101ba8..603362f 100644
--- a/FsCrypt.cpp
+++ b/FsCrypt.cpp
@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <map>
+#include <optional>
 #include <set>
 #include <sstream>
 #include <string>
@@ -88,8 +89,6 @@
 // Map user ids to key references
 std::map<userid_t, std::string> s_de_key_raw_refs;
 std::map<userid_t, std::string> s_ce_key_raw_refs;
-// TODO abolish this map, per b/26948053
-std::map<userid_t, KeyBuffer> s_ce_keys;
 
 }  // namespace
 
@@ -243,7 +242,6 @@
     if (!read_and_fixate_user_ce_key(user_id, auth, &ce_key)) return false;
     std::string ce_raw_ref;
     if (!install_data_key(ce_key, &ce_raw_ref)) return false;
-    s_ce_keys[user_id] = std::move(ce_key);
     s_ce_key_raw_refs[user_id] = ce_raw_ref;
     LOG(DEBUG) << "Installed ce key for user " << user_id;
     return true;
@@ -296,7 +294,6 @@
     s_de_key_raw_refs[user_id] = de_raw_ref;
     std::string ce_raw_ref;
     if (!install_data_key(ce_key, &ce_raw_ref)) return false;
-    s_ce_keys[user_id] = ce_key;
     s_ce_key_raw_refs[user_id] = ce_raw_ref;
     LOG(DEBUG) << "Created keys for user " << user_id;
     return true;
@@ -468,7 +465,6 @@
 }
 
 static bool evict_ce_key(userid_t user_id) {
-    s_ce_keys.erase(user_id);
     bool success = true;
     std::string raw_ref;
     // If we haven't loaded the CE key, no need to evict it.
@@ -549,6 +545,18 @@
     return true;
 }
 
+static std::optional<android::vold::KeyAuthentication> authentication_from_hex(
+        const std::string& token_hex, const std::string& secret_hex) {
+    std::string token, secret;
+    if (!parse_hex(token_hex, &token)) return std::optional<android::vold::KeyAuthentication>();
+    if (!parse_hex(secret_hex, &secret)) return std::optional<android::vold::KeyAuthentication>();
+    if (secret.empty()) {
+        return kEmptyAuthentication;
+    } else {
+        return android::vold::KeyAuthentication(token, secret);
+    }
+}
+
 static std::string volkey_path(const std::string& misc_path, const std::string& volume_uuid) {
     return misc_path + "/vold/volume_keys/" + volume_uuid + "/default";
 }
@@ -592,30 +600,51 @@
     return android::vold::destroyKey(path);
 }
 
+static bool fscrypt_rewrap_user_key(userid_t user_id, int serial,
+                                    const android::vold::KeyAuthentication& retrieve_auth,
+                                    const android::vold::KeyAuthentication& store_auth) {
+    if (s_ephemeral_users.count(user_id) != 0) return true;
+    auto const directory_path = get_ce_key_directory_path(user_id);
+    KeyBuffer ce_key;
+    std::string ce_key_current_path = get_ce_key_current_path(directory_path);
+    if (android::vold::retrieveKey(ce_key_current_path, retrieve_auth, &ce_key)) {
+        LOG(DEBUG) << "Successfully retrieved key";
+        // TODO(147732812): Remove this once Locksettingservice is fixed.
+        // Currently it calls fscrypt_clear_user_key_auth with a secret when lockscreen is
+        // changed from swipe to none or vice-versa
+    } else if (android::vold::retrieveKey(ce_key_current_path, kEmptyAuthentication, &ce_key)) {
+        LOG(DEBUG) << "Successfully retrieved key with empty auth";
+    } else {
+        LOG(ERROR) << "Failed to retrieve key for user " << user_id;
+        return false;
+    }
+    auto const paths = get_ce_key_paths(directory_path);
+    std::string ce_key_path;
+    if (!get_ce_key_new_path(directory_path, paths, &ce_key_path)) return false;
+    if (!android::vold::storeKeyAtomically(ce_key_path, user_key_temp, store_auth, ce_key))
+        return false;
+    if (!android::vold::FsyncDirectory(directory_path)) return false;
+    return true;
+}
+
 bool fscrypt_add_user_key_auth(userid_t user_id, int serial, const std::string& token_hex,
                                const std::string& secret_hex) {
     LOG(DEBUG) << "fscrypt_add_user_key_auth " << user_id << " serial=" << serial
                << " token_present=" << (token_hex != "!");
     if (!fscrypt_is_native()) return true;
-    if (s_ephemeral_users.count(user_id) != 0) return true;
-    std::string token, secret;
-    if (!parse_hex(token_hex, &token)) return false;
-    if (!parse_hex(secret_hex, &secret)) return false;
-    auto auth =
-        secret.empty() ? kEmptyAuthentication : android::vold::KeyAuthentication(token, secret);
-    auto it = s_ce_keys.find(user_id);
-    if (it == s_ce_keys.end()) {
-        LOG(ERROR) << "Key not loaded into memory, can't change for user " << user_id;
-        return false;
-    }
-    const auto& ce_key = it->second;
-    auto const directory_path = get_ce_key_directory_path(user_id);
-    auto const paths = get_ce_key_paths(directory_path);
-    std::string ce_key_path;
-    if (!get_ce_key_new_path(directory_path, paths, &ce_key_path)) return false;
-    if (!android::vold::storeKeyAtomically(ce_key_path, user_key_temp, auth, ce_key)) return false;
-    if (!android::vold::FsyncDirectory(directory_path)) return false;
-    return true;
+    auto auth = authentication_from_hex(token_hex, secret_hex);
+    if (!auth) return false;
+    return fscrypt_rewrap_user_key(user_id, serial, kEmptyAuthentication, *auth);
+}
+
+bool fscrypt_clear_user_key_auth(userid_t user_id, int serial, const std::string& token_hex,
+                                 const std::string& secret_hex) {
+    LOG(DEBUG) << "fscrypt_clear_user_key_auth " << user_id << " serial=" << serial
+               << " token_present=" << (token_hex != "!");
+    if (!fscrypt_is_native()) return true;
+    auto auth = authentication_from_hex(token_hex, secret_hex);
+    if (!auth) return false;
+    return fscrypt_rewrap_user_key(user_id, serial, *auth, kEmptyAuthentication);
 }
 
 bool fscrypt_fixate_newest_user_key_auth(userid_t user_id) {
@@ -642,11 +671,9 @@
             LOG(WARNING) << "Tried to unlock already-unlocked key for user " << user_id;
             return true;
         }
-        std::string token, secret;
-        if (!parse_hex(token_hex, &token)) return false;
-        if (!parse_hex(secret_hex, &secret)) return false;
-        android::vold::KeyAuthentication auth(token, secret);
-        if (!read_and_install_user_ce_key(user_id, auth)) {
+        auto auth = authentication_from_hex(token_hex, secret_hex);
+        if (!auth) return false;
+        if (!read_and_install_user_ce_key(user_id, *auth)) {
             LOG(ERROR) << "Couldn't read key for " << user_id;
             return false;
         }
diff --git a/FsCrypt.h b/FsCrypt.h
index 03ec2e1..641991a 100644
--- a/FsCrypt.h
+++ b/FsCrypt.h
@@ -25,6 +25,8 @@
 bool fscrypt_destroy_user_key(userid_t user_id);
 bool fscrypt_add_user_key_auth(userid_t user_id, int serial, const std::string& token,
                                const std::string& secret);
+bool fscrypt_clear_user_key_auth(userid_t user_id, int serial, const std::string& token,
+                                 const std::string& secret);
 bool fscrypt_fixate_newest_user_key_auth(userid_t user_id);
 
 bool fscrypt_unlock_user_key(userid_t user_id, int serial, const std::string& token,
diff --git a/KeyStorage.cpp b/KeyStorage.cpp
index d5ac7d0..dbf190d 100644
--- a/KeyStorage.cpp
+++ b/KeyStorage.cpp
@@ -43,8 +43,8 @@
 #include <cutils/properties.h>
 
 #include <hardware/hw_auth_token.h>
-#include <keymasterV4_0/authorization_set.h>
-#include <keymasterV4_0/keymaster_utils.h>
+#include <keymasterV4_1/authorization_set.h>
+#include <keymasterV4_1/keymaster_utils.h>
 
 extern "C" {
 
diff --git a/Keymaster.cpp b/Keymaster.cpp
index aad4387..a3853f9 100644
--- a/Keymaster.cpp
+++ b/Keymaster.cpp
@@ -17,8 +17,8 @@
 #include "Keymaster.h"
 
 #include <android-base/logging.h>
-#include <keymasterV4_0/authorization_set.h>
-#include <keymasterV4_0/keymaster_utils.h>
+#include <keymasterV4_1/authorization_set.h>
+#include <keymasterV4_1/keymaster_utils.h>
 
 namespace android {
 namespace vold {
diff --git a/Keymaster.h b/Keymaster.h
index 42a2b5d..049a741 100644
--- a/Keymaster.h
+++ b/Keymaster.h
@@ -24,13 +24,24 @@
 #include <utility>
 
 #include <android-base/macros.h>
-#include <keymasterV4_0/Keymaster.h>
-#include <keymasterV4_0/authorization_set.h>
+#include <keymasterV4_1/Keymaster.h>
+#include <keymasterV4_1/authorization_set.h>
 
 namespace android {
 namespace vold {
 
-namespace km = ::android::hardware::keymaster::V4_0;
+namespace km {
+
+using namespace ::android::hardware::keymaster::V4_1;
+
+// Surprisingly -- to me, at least -- this is totally fine.  You can re-define symbols that were
+// brought in via a using directive (the "using namespace") above.  In general this seems like a
+// dangerous thing to rely on, but in this case its implications are simple and straightforward:
+// km::ErrorCode refers to the 4.0 ErrorCode, though we pull everything else from 4.1.
+using ErrorCode = ::android::hardware::keymaster::V4_0::ErrorCode;
+
+}  // namespace km
+
 using KmDevice = km::support::Keymaster;
 
 // C++ wrappers to the Keymaster hidl interface.
@@ -115,7 +126,7 @@
     bool isSecure();
 
   private:
-    std::unique_ptr<KmDevice> mDevice;
+    sp<KmDevice> mDevice;
     DISALLOW_COPY_AND_ASSIGN(Keymaster);
     static bool hmacKeyGenerated;
 };
diff --git a/Utils.cpp b/Utils.cpp
index 1616d80..202b98d 100644
--- a/Utils.cpp
+++ b/Utils.cpp
@@ -55,6 +55,7 @@
 
 using namespace std::chrono_literals;
 using android::base::ReadFileToString;
+using android::base::StartsWith;
 using android::base::StringPrintf;
 
 namespace android {
@@ -114,6 +115,27 @@
     }
 }
 
+int PrepareDirsFromRoot(std::string path, std::string root, mode_t mode, uid_t uid, gid_t gid) {
+    int ret = 0;
+    if (!StartsWith(path, root)) {
+        return -1;
+    }
+    std::string to_create_from_root = path.substr(root.length());
+
+    size_t pos = 0;
+    while ((pos = to_create_from_root.find('/')) != std::string::npos) {
+        auto component = to_create_from_root.substr(0, pos);
+        to_create_from_root.erase(0, pos + 1);
+        root = root + component + "/";
+        ret = fs_prepare_dir(root.c_str(), mode, uid, gid);
+        if (ret) {
+            break;
+        }
+    }
+
+    return ret;
+}
+
 status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid) {
     std::lock_guard<std::mutex> lock(kSecurityLock);
     const char* cpath = path.c_str();
@@ -165,7 +187,7 @@
     if (!umount2(cpath, UMOUNT_NOFOLLOW) || errno == EINVAL || errno == ENOENT) {
         return OK;
     }
-
+    PLOG(INFO) << "ForceUnmount failed";
     return -errno;
 }
 
@@ -985,5 +1007,143 @@
     return true;
 }
 
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd) {
+    std::string pre_fuse_path(StringPrintf("/mnt/user/%d", user_id));
+    std::string fuse_path(
+            StringPrintf("%s/%s", pre_fuse_path.c_str(), relative_upper_path.c_str()));
+
+    std::string pre_pass_through_path(StringPrintf("/mnt/pass_through/%d", user_id));
+    std::string pass_through_path(
+            StringPrintf("%s/%s", pre_pass_through_path.c_str(), relative_upper_path.c_str()));
+
+    // Ensure that /mnt/user is 0700. With FUSE, apps don't need access to /mnt/user paths directly.
+    // Without FUSE however, apps need /mnt/user access so /mnt/user in init.rc is 0755 until here
+    auto result = PrepareDir("/mnt/user", 0700, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory /mnt/user";
+        return -1;
+    }
+
+    // Shell is neither AID_ROOT nor AID_EVERYBODY. Since it equally needs 'execute' access to
+    // /mnt/user/0 to 'adb shell ls /sdcard' for instance, we set the uid bit of /mnt/user/0 to
+    // AID_SHELL. This gives shell access along with apps running as group everybody (user 0 apps)
+    // These bits should be consistent with what is set in zygote in
+    // com_android_internal_os_Zygote#MountEmulatedStorage on volume bind mount during app fork
+    result = PrepareDir(pre_fuse_path, 0710, user_id ? AID_ROOT : AID_SHELL,
+                             multiuser_get_uid(user_id, AID_EVERYBODY));
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(fuse_path, 0700, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(pre_pass_through_path, 0755, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_pass_through_path;
+        return -1;
+    }
+
+    result = PrepareDir(pass_through_path, 0755, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pass_through_path;
+        return -1;
+    }
+
+    if (relative_upper_path == "emulated") {
+        std::string linkpath(StringPrintf("/mnt/user/%d/self", user_id));
+        result = PrepareDir(linkpath, 0755, AID_ROOT, AID_ROOT);
+        if (result != android::OK) {
+            PLOG(ERROR) << "Failed to prepare directory " << linkpath;
+            return -1;
+        }
+        linkpath += "/primary";
+        Symlink("/storage/emulated/" + std::to_string(user_id), linkpath);
+
+        std::string pass_through_linkpath(StringPrintf("/mnt/pass_through/%d/self", user_id));
+        result = PrepareDir(pass_through_linkpath, 0755, AID_ROOT, AID_ROOT);
+        if (result != android::OK) {
+            PLOG(ERROR) << "Failed to prepare directory " << pass_through_linkpath;
+            return -1;
+        }
+        pass_through_linkpath += "/primary";
+        Symlink("/storage/emulated/" + std::to_string(user_id), pass_through_linkpath);
+    }
+
+    // Open fuse fd.
+    fuse_fd->reset(open("/dev/fuse", O_RDWR | O_CLOEXEC));
+    if (fuse_fd->get() == -1) {
+        PLOG(ERROR) << "Failed to open /dev/fuse";
+        return -1;
+    }
+
+    // Note: leaving out default_permissions since we don't want kernel to do lower filesystem
+    // permission checks before routing to FUSE daemon.
+    const auto opts = StringPrintf(
+        "fd=%i,"
+        "rootmode=40000,"
+        "allow_other,"
+        "user_id=0,group_id=0,",
+        fuse_fd->get());
+
+    result = TEMP_FAILURE_RETRY(mount("/dev/fuse", fuse_path.c_str(), "fuse",
+                                      MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_NOATIME | MS_LAZYTIME,
+                                      opts.c_str()));
+    if (result != 0) {
+        PLOG(ERROR) << "Failed to mount " << fuse_path;
+        return -errno;
+    }
+
+    if (IsFilesystemSupported("sdcardfs")) {
+        std::string sdcardfs_path(
+                StringPrintf("/mnt/runtime/full/%s", relative_upper_path.c_str()));
+
+        LOG(INFO) << "Bind mounting " << sdcardfs_path << " to " << pass_through_path;
+        return BindMount(sdcardfs_path, pass_through_path);
+    } else {
+        LOG(INFO) << "Bind mounting " << absolute_lower_path << " to " << pass_through_path;
+        return BindMount(absolute_lower_path, pass_through_path);
+    }
+}
+
+status_t UnmountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path) {
+    std::string fuse_path(StringPrintf("/mnt/user/%d/%s", user_id, relative_upper_path.c_str()));
+    std::string pass_through_path(
+            StringPrintf("/mnt/pass_through/%d/%s", user_id, relative_upper_path.c_str()));
+
+    // Best effort unmount pass_through path
+    sSleepOnUnmount = false;
+    LOG(INFO) << "Unmounting pass_through_path " << pass_through_path;
+    auto status = ForceUnmount(pass_through_path);
+    if (status != android::OK) {
+        LOG(ERROR) << "Failed to unmount " << pass_through_path;
+    }
+    rmdir(pass_through_path.c_str());
+
+    LOG(INFO) << "Unmounting fuse path " << fuse_path;
+    android::status_t result = ForceUnmount(fuse_path);
+    sSleepOnUnmount = true;
+    if (result != android::OK) {
+        // TODO(b/135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
+        // Figure out why we get EBUSY and remove this special casing if possible.
+        PLOG(ERROR) << "Failed to unmount. Trying MNT_DETACH " << fuse_path << " ...";
+        if (umount2(fuse_path.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) && errno != EINVAL &&
+            errno != ENOENT) {
+            PLOG(ERROR) << "Failed to unmount with MNT_DETACH " << fuse_path;
+            return -errno;
+        }
+        result = android::OK;
+    }
+    rmdir(fuse_path.c_str());
+
+    return result;
+}
+
 }  // namespace vold
 }  // namespace android
diff --git a/Utils.h b/Utils.h
index af4e401..056a635 100644
--- a/Utils.h
+++ b/Utils.h
@@ -20,6 +20,7 @@
 #include "KeyBuffer.h"
 
 #include <android-base/macros.h>
+#include <android-base/unique_fd.h>
 #include <cutils/multiuser.h>
 #include <selinux/selinux.h>
 #include <utils/Errors.h>
@@ -33,6 +34,8 @@
 namespace android {
 namespace vold {
 
+static const char* kPropFuse = "persist.sys.fuse";
+
 /* SELinux contexts used depending on the block device type */
 extern security_context_t sBlkidContext;
 extern security_context_t sBlkidUntrustedContext;
@@ -45,6 +48,12 @@
 status_t CreateDeviceNode(const std::string& path, dev_t dev);
 status_t DestroyDeviceNode(const std::string& path);
 
+/*
+ * Recursively calls fs_prepare_dir() on all components in 'path', starting at 'root'.
+ * 'path' must start with 'root'
+ */
+int PrepareDirsFromRoot(std::string path, std::string root, mode_t mode, uid_t uid, gid_t gid);
+
 /* fs_prepare_dir wrapper that creates with SELinux context */
 status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
 
@@ -147,6 +156,13 @@
 bool FsyncDirectory(const std::string& dirname);
 
 bool writeStringToFile(const std::string& payload, const std::string& filename);
+
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd);
+
+status_t UnmountUserFuse(userid_t userId, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path);
+
 }  // namespace vold
 }  // namespace android
 
diff --git a/VoldNativeService.cpp b/VoldNativeService.cpp
index cc32820..f8ed61c 100644
--- a/VoldNativeService.cpp
+++ b/VoldNativeService.cpp
@@ -108,7 +108,7 @@
         return exception(binder::Status::EX_ILLEGAL_ARGUMENT, "Missing ID");
     }
     for (const char& c : id) {
-        if (!std::isalnum(c) && c != ':' && c != ',') {
+        if (!std::isalnum(c) && c != ':' && c != ',' && c != ';') {
             return exception(binder::Status::EX_ILLEGAL_ARGUMENT,
                              StringPrintf("ID %s is malformed", id.c_str()));
         }
@@ -326,8 +326,9 @@
     return translate(VolumeManager::Instance()->forgetPartition(partGuid, fsUuid));
 }
 
-binder::Status VoldNativeService::mount(const std::string& volId, int32_t mountFlags,
-                                        int32_t mountUserId) {
+binder::Status VoldNativeService::mount(
+        const std::string& volId, int32_t mountFlags, int32_t mountUserId,
+        const android::sp<android::os::IVoldMountCallback>& callback) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_ID(volId);
     ACQUIRE_LOCK;
@@ -340,10 +341,14 @@
     vol->setMountFlags(mountFlags);
     vol->setMountUserId(mountUserId);
 
+    vol->setMountCallback(callback);
     int res = vol->mount();
+    vol->setMountCallback(nullptr);
+
     if (res != OK) {
         return translate(res);
     }
+
     if ((mountFlags & MOUNT_FLAG_PRIMARY) != 0) {
         res = VolumeManager::Instance()->setPrimary(vol);
         if (res != OK) {
@@ -451,12 +456,14 @@
     return translate(VolumeManager::Instance()->remountUid(uid, remountMode));
 }
 
-binder::Status VoldNativeService::mkdirs(const std::string& path) {
+binder::Status VoldNativeService::setupAppDir(const std::string& path,
+                                              const std::string& appDirRoot, int32_t appUid) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_PATH(path);
+    CHECK_ARGUMENT_PATH(appDirRoot);
     ACQUIRE_LOCK;
 
-    return translate(VolumeManager::Instance()->mkdirs(path));
+    return translate(VolumeManager::Instance()->setupAppDir(path, appDirRoot, appUid));
 }
 
 binder::Status VoldNativeService::createObb(const std::string& sourcePath,
@@ -761,6 +768,15 @@
     return translateBool(fscrypt_add_user_key_auth(userId, userSerial, token, secret));
 }
 
+binder::Status VoldNativeService::clearUserKeyAuth(int32_t userId, int32_t userSerial,
+                                                   const std::string& token,
+                                                   const std::string& secret) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_CRYPT_LOCK;
+
+    return translateBool(fscrypt_clear_user_key_auth(userId, userSerial, token, secret));
+}
+
 binder::Status VoldNativeService::fixateNewestUserKeyAuth(int32_t userId) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_CRYPT_LOCK;
diff --git a/VoldNativeService.h b/VoldNativeService.h
index 0718263..2967fae 100644
--- a/VoldNativeService.h
+++ b/VoldNativeService.h
@@ -52,7 +52,8 @@
     binder::Status partition(const std::string& diskId, int32_t partitionType, int32_t ratio);
     binder::Status forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
-    binder::Status mount(const std::string& volId, int32_t mountFlags, int32_t mountUserId);
+    binder::Status mount(const std::string& volId, int32_t mountFlags, int32_t mountUserId,
+                         const android::sp<android::os::IVoldMountCallback>& callback);
     binder::Status unmount(const std::string& volId);
     binder::Status format(const std::string& volId, const std::string& fsType);
     binder::Status benchmark(const std::string& volId,
@@ -64,7 +65,8 @@
 
     binder::Status remountUid(int32_t uid, int32_t remountMode);
 
-    binder::Status mkdirs(const std::string& path);
+    binder::Status setupAppDir(const std::string& path, const std::string& appDirRoot,
+                               int32_t appUid);
 
     binder::Status createObb(const std::string& sourcePath, const std::string& sourceKey,
                              int32_t ownerGid, std::string* _aidl_return);
@@ -112,6 +114,8 @@
 
     binder::Status addUserKeyAuth(int32_t userId, int32_t userSerial, const std::string& token,
                                   const std::string& secret);
+    binder::Status clearUserKeyAuth(int32_t userId, int32_t userSerial, const std::string& token,
+                                    const std::string& secret);
     binder::Status fixateNewestUserKeyAuth(int32_t userId);
 
     binder::Status unlockUserKey(int32_t userId, int32_t userSerial, const std::string& token,
diff --git a/VolumeManager.cpp b/VolumeManager.cpp
index 44bff5a..d8b1e32 100644
--- a/VolumeManager.cpp
+++ b/VolumeManager.cpp
@@ -40,6 +40,7 @@
 #include <android-base/properties.h>
 #include <android-base/stringprintf.h>
 #include <android-base/strings.h>
+#include <async_safe/log.h>
 
 #include <cutils/fs.h>
 #include <utils/Trace.h>
@@ -67,6 +68,7 @@
 #include "fs/Vfat.h"
 #include "model/EmulatedVolume.h"
 #include "model/ObbVolume.h"
+#include "model/PrivateVolume.h"
 #include "model/StubVolume.h"
 
 using android::OK;
@@ -79,10 +81,13 @@
 using android::vold::CreateDir;
 using android::vold::DeleteDirContents;
 using android::vold::DeleteDirContentsAndDir;
+using android::vold::PrepareDirsFromRoot;
+using android::vold::PrivateVolume;
 using android::vold::Symlink;
 using android::vold::Unlink;
 using android::vold::UnmountTree;
 using android::vold::VoldNativeService;
+using android::vold::VolumeBase;
 
 static const char* kPathUserMount = "/mnt/user";
 static const char* kPathVirtualDisk = "/data/misc/vold/virtual_disk";
@@ -174,10 +179,13 @@
 
     // Assume that we always have an emulated volume on internal
     // storage; the framework will decide if it should be mounted.
-    CHECK(mInternalEmulated == nullptr);
-    mInternalEmulated = std::shared_ptr<android::vold::VolumeBase>(
-        new android::vold::EmulatedVolume("/data/media"));
-    mInternalEmulated->create();
+    CHECK(mInternalEmulatedVolumes.empty());
+
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", 0));
+    vol->setMountUserId(0);
+    vol->create();
+    mInternalEmulatedVolumes.push_back(vol);
 
     // Consider creating a virtual disk
     updateVirtualDisk();
@@ -186,9 +194,12 @@
 }
 
 int VolumeManager::stop() {
-    CHECK(mInternalEmulated != nullptr);
-    mInternalEmulated->destroy();
-    mInternalEmulated = nullptr;
+    CHECK(!mInternalEmulatedVolumes.empty());
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
+    }
+    mInternalEmulatedVolumes.clear();
+
     return 0;
 }
 
@@ -253,10 +264,17 @@
 void VolumeManager::handleDiskAdded(const std::shared_ptr<android::vold::Disk>& disk) {
     // For security reasons, if secure keyguard is showing, wait
     // until the user unlocks the device to actually touch it
+    // Additionally, wait until user 0 is actually started, since we need
+    // the user to be up before we can mount a FUSE daemon to handle the disk.
+    bool userZeroStarted = mStartedUsers.find(0) != mStartedUsers.end();
     if (mSecureKeyguardShowing) {
         LOG(INFO) << "Found disk at " << disk->getEventPath()
                   << " but delaying scan due to secure keyguard";
         mPendingDisks.push_back(disk);
+    } else if (!userZeroStarted) {
+        LOG(INFO) << "Found disk at " << disk->getEventPath()
+                  << " but delaying scan due to user zero not having started";
+        mPendingDisks.push_back(disk);
     } else {
         disk->create();
         mDisks.push_back(disk);
@@ -310,11 +328,10 @@
 }
 
 std::shared_ptr<android::vold::VolumeBase> VolumeManager::findVolume(const std::string& id) {
-    // Vold could receive "mount" after "shutdown" command in the extreme case.
-    // If this happens, mInternalEmulated will equal nullptr and
-    // we need to deal with it in order to avoid null pointer crash.
-    if (mInternalEmulated != nullptr && mInternalEmulated->getId() == id) {
-        return mInternalEmulated;
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        if (vol->getId() == id) {
+            return vol;
+        }
     }
     for (const auto& disk : mDisks) {
         auto vol = disk->findVolume(id);
@@ -365,60 +382,145 @@
 }
 
 int VolumeManager::linkPrimary(userid_t userId) {
-    std::string source(mPrimary->getPath());
-    if (mPrimary->isEmulated()) {
-        source = StringPrintf("%s/%d", source.c_str(), userId);
-        fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
-    }
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
+        std::string source(mPrimary->getPath());
+        if (mPrimary->isEmulated()) {
+            source = StringPrintf("%s/%d", source.c_str(), userId);
+            fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
+        }
 
-    std::string target(StringPrintf("/mnt/user/%d/primary", userId));
-    LOG(DEBUG) << "Linking " << source << " to " << target;
-    Symlink(source, target);
+        std::string target(StringPrintf("/mnt/user/%d/primary", userId));
+        LOG(DEBUG) << "Linking " << source << " to " << target;
+        Symlink(source, target);
+    }
     return 0;
 }
 
+void VolumeManager::destroyEmulatedVolumesForUser(userid_t userId) {
+    // Destroy and remove all unstacked EmulatedVolumes for the user
+    auto i = mInternalEmulatedVolumes.begin();
+    while (i != mInternalEmulatedVolumes.end()) {
+        auto vol = *i;
+        if (vol->getMountUserId() == userId) {
+            vol->destroy();
+            i = mInternalEmulatedVolumes.erase(i);
+        } else {
+            i++;
+        }
+    }
+
+    // Destroy and remove all stacked EmulatedVolumes for the user on each mounted private volume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        std::list<std::shared_ptr<VolumeBase>> vols_to_remove;
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            for (const auto& vol : pvol->getVolumes()) {
+                if (vol->getMountUserId() == userId) {
+                    vols_to_remove.push_back(vol);
+                }
+            }
+            for (const auto& vol : vols_to_remove) {
+                vol->destroy();
+                pvol->removeVolume(vol);
+            }
+        }  // else EmulatedVolumes will be destroyed on VolumeBase#unmount
+    }
+}
+
+void VolumeManager::createEmulatedVolumesForUser(userid_t userId) {
+    // Create unstacked EmulatedVolumes for the user
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", userId));
+    vol->setMountUserId(userId);
+    mInternalEmulatedVolumes.push_back(vol);
+    vol->create();
+
+    // Create stacked EmulatedVolumes for the user on each PrivateVolume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            auto evol =
+                    std::shared_ptr<android::vold::VolumeBase>(new android::vold::EmulatedVolume(
+                            pvol->getPath() + "/media", pvol->getRawDevice(), pvol->getFsUuid(),
+                            userId));
+            evol->setMountUserId(userId);
+            pvol->addVolume(evol);
+            evol->create();
+        }  // else EmulatedVolumes will be created per user when on PrivateVolume#doMount
+    }
+}
+
 int VolumeManager::onUserAdded(userid_t userId, int userSerialNumber) {
+    LOG(INFO) << "onUserAdded: " << userId;
+
     mAddedUsers[userId] = userSerialNumber;
     return 0;
 }
 
 int VolumeManager::onUserRemoved(userid_t userId) {
+    LOG(INFO) << "onUserRemoved: " << userId;
+
+    onUserStopped(userId);
     mAddedUsers.erase(userId);
     return 0;
 }
 
 int VolumeManager::onUserStarted(userid_t userId) {
-    LOG(VERBOSE) << "onUserStarted: " << userId;
-    // Note that sometimes the system will spin up processes from Zygote
-    // before actually starting the user, so we're okay if Zygote
-    // already created this directory.
-    std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
-    fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+    LOG(INFO) << "onUserStarted: " << userId;
+
+    if (mStartedUsers.find(userId) == mStartedUsers.end()) {
+        createEmulatedVolumesForUser(userId);
+    }
+
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
+        // Note that sometimes the system will spin up processes from Zygote
+        // before actually starting the user, so we're okay if Zygote
+        // already created this directory.
+        std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
+        fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+
+        if (mPrimary) {
+            linkPrimary(userId);
+        }
+    }
 
     mStartedUsers.insert(userId);
-    if (mPrimary) {
-        linkPrimary(userId);
-    }
+
+    createPendingDisksIfNeeded();
     return 0;
 }
 
 int VolumeManager::onUserStopped(userid_t userId) {
     LOG(VERBOSE) << "onUserStopped: " << userId;
+
+    if (mStartedUsers.find(userId) != mStartedUsers.end()) {
+        destroyEmulatedVolumesForUser(userId);
+    }
+
     mStartedUsers.erase(userId);
     return 0;
 }
 
-int VolumeManager::onSecureKeyguardStateChanged(bool isShowing) {
-    mSecureKeyguardShowing = isShowing;
-    if (!mSecureKeyguardShowing) {
-        // Now that secure keyguard has been dismissed, process
-        // any pending disks
+void VolumeManager::createPendingDisksIfNeeded() {
+    bool userZeroStarted = mStartedUsers.find(0) != mStartedUsers.end();
+    if (!mSecureKeyguardShowing && userZeroStarted) {
+        // Now that secure keyguard has been dismissed and user 0 has
+        // started, process any pending disks
         for (const auto& disk : mPendingDisks) {
             disk->create();
             mDisks.push_back(disk);
         }
         mPendingDisks.clear();
     }
+}
+
+int VolumeManager::onSecureKeyguardStateChanged(bool isShowing) {
+    mSecureKeyguardShowing = isShowing;
+    createPendingDisksIfNeeded();
     return 0;
 }
 
@@ -430,7 +532,56 @@
     return 0;
 }
 
+// This code is executed after a fork so it's very important that the set of
+// methods we call here is strictly limited.
+//
+// TODO: Get rid of this guesswork altogether and instead exec a process
+// immediately after fork to do our bindding for us.
+static bool childProcess(const char* storageSource, const char* userSource, int nsFd,
+                         struct dirent* de) {
+    if (setns(nsFd, CLONE_NEWNS) != 0) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns for %s :%s", de->d_name,
+                              strerror(errno));
+        return false;
+    }
+
+    // NOTE: Inlined from vold::UnmountTree here to avoid using PLOG methods and
+    // to also protect against future changes that may cause issues across a
+    // fork.
+    if (TEMP_FAILURE_RETRY(umount2("/storage/", MNT_DETACH)) < 0 && errno != EINVAL &&
+        errno != ENOENT) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to unmount /storage/ :%s",
+                              strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(storageSource, "/storage", NULL, MS_BIND | MS_REC, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
+                              storageSource, de->d_name, strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(NULL, "/storage", NULL, MS_REC | MS_SLAVE, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                              "Failed to set MS_SLAVE to /storage for %s :%s", de->d_name,
+                              strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(userSource, "/storage/self", NULL, MS_BIND, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
+                              userSource, de->d_name, strerror(errno));
+        return false;
+    }
+
+    return true;
+}
+
 int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
+    if (GetBoolProperty(android::vold::kPropFuse, false)) {
+        // TODO(135341433): Implement fuse specific logic.
+        return 0;
+    }
     std::string mode;
     switch (mountMode) {
         case VoldNativeService::REMOUNT_MODE_NONE:
@@ -450,6 +601,9 @@
         case VoldNativeService::REMOUNT_MODE_FULL:
             mode = "full";
             break;
+        case VoldNativeService::REMOUNT_MODE_PASS_THROUGH:
+            mode = "pass_through";
+            break;
         default:
             PLOG(ERROR) << "Unknown mode " << std::to_string(mountMode);
             return -1;
@@ -464,6 +618,8 @@
     int nsFd;
     struct stat sb;
     pid_t child;
+    std::string userSource;
+    std::string storageSource;
 
     static bool apexUpdatable = android::sysprop::ApexProperties::updatable().value_or(false);
 
@@ -539,47 +695,28 @@
             goto next;
         }
 
+        if (mode == "default") {
+            storageSource = "/mnt/runtime/default";
+        } else if (mode == "read") {
+            storageSource = "/mnt/runtime/read";
+        } else if (mode == "write") {
+            storageSource = "/mnt/runtime/write";
+        } else if (mode == "full") {
+            storageSource = "/mnt/runtime/full";
+        } else {
+            // Sane default of no storage visible. No need to fork a child
+            // to remount uid.
+            goto next;
+        }
+
+        // Mount user-specific symlink helper into place
+        userSource = StringPrintf("/mnt/user/%d", multiuser_get_user_id(uid));
         if (!(child = fork())) {
-            if (setns(nsFd, CLONE_NEWNS) != 0) {
-                PLOG(ERROR) << "Failed to setns for " << de->d_name;
-                _exit(1);
-            }
-
-            android::vold::UnmountTree("/storage/");
-
-            std::string storageSource;
-            if (mode == "default") {
-                storageSource = "/mnt/runtime/default";
-            } else if (mode == "read") {
-                storageSource = "/mnt/runtime/read";
-            } else if (mode == "write") {
-                storageSource = "/mnt/runtime/write";
-            } else if (mode == "full") {
-                storageSource = "/mnt/runtime/full";
-            } else {
-                // Sane default of no storage visible
+            if (childProcess(storageSource.c_str(), userSource.c_str(), nsFd, de)) {
                 _exit(0);
-            }
-            if (TEMP_FAILURE_RETRY(
-                    mount(storageSource.c_str(), "/storage", NULL, MS_BIND | MS_REC, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to mount " << storageSource << " for " << de->d_name;
+            } else {
                 _exit(1);
             }
-            if (TEMP_FAILURE_RETRY(mount(NULL, "/storage", NULL, MS_REC | MS_SLAVE, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to set MS_SLAVE to /storage for " << de->d_name;
-                _exit(1);
-            }
-
-            // Mount user-specific symlink helper into place
-            userid_t user_id = multiuser_get_user_id(uid);
-            std::string userSource(StringPrintf("/mnt/user/%d", user_id));
-            if (TEMP_FAILURE_RETRY(
-                    mount(userSource.c_str(), "/storage/self", NULL, MS_BIND, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to mount " << userSource << " for " << de->d_name;
-                _exit(1);
-            }
-
-            _exit(0);
         }
 
         if (child == -1) {
@@ -600,10 +737,11 @@
 int VolumeManager::reset() {
     // Tear down all existing disks/volumes and start from a blank slate so
     // newly connected framework hears all events.
-    if (mInternalEmulated != nullptr) {
-        mInternalEmulated->destroy();
-        mInternalEmulated->create();
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
     }
+    mInternalEmulatedVolumes.clear();
+
     for (const auto& disk : mDisks) {
         disk->destroy();
         disk->create();
@@ -616,15 +754,18 @@
 
 // Can be called twice (sequentially) during shutdown. should be safe for that.
 int VolumeManager::shutdown() {
-    if (mInternalEmulated == nullptr) {
+    if (mInternalEmulatedVolumes.empty()) {
         return 0;  // already shutdown
     }
     android::vold::sSleepOnUnmount = false;
-    mInternalEmulated->destroy();
-    mInternalEmulated = nullptr;
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
+    }
     for (const auto& disk : mDisks) {
         disk->destroy();
     }
+
+    mInternalEmulatedVolumes.clear();
     mStubVolumes.clear();
     mDisks.clear();
     mPendingDisks.clear();
@@ -637,8 +778,8 @@
     ATRACE_NAME("VolumeManager::unmountAll()");
 
     // First, try gracefully unmounting all known devices
-    if (mInternalEmulated != nullptr) {
-        mInternalEmulated->unmount();
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->unmount();
     }
     for (const auto& stub : mStubVolumes) {
         stub->unmount();
@@ -665,7 +806,8 @@
 #ifdef __ANDROID_DEBUGGABLE__
              !StartsWith(test, "/mnt/scratch") &&
 #endif
-             !StartsWith(test, "/mnt/vendor") && !StartsWith(test, "/mnt/product")) ||
+             !StartsWith(test, "/mnt/vendor") && !StartsWith(test, "/mnt/product") &&
+             !StartsWith(test, "/mnt/installer")) ||
             StartsWith(test, "/storage/")) {
             toUnmount.push_front(test);
         }
@@ -680,15 +822,29 @@
     return 0;
 }
 
-int VolumeManager::mkdirs(const std::string& path) {
+int VolumeManager::setupAppDir(const std::string& path, const std::string& appDirRoot,
+                               int32_t appUid) {
     // Only offer to create directories for paths managed by vold
-    if (StartsWith(path, "/storage/")) {
-        // fs_mkdirs() does symlink checking and relative path enforcement
-        return fs_mkdirs(path.c_str(), 0700);
-    } else {
+    if (!StartsWith(path, "/storage/")) {
         LOG(ERROR) << "Failed to find mounted volume for " << path;
         return -EINVAL;
     }
+
+    // Convert paths to lower filesystem paths to avoid making FUSE requests for these reasons:
+    // 1. A FUSE request from vold puts vold at risk of hanging if the FUSE daemon is down
+    // 2. The FUSE daemon prevents requests on /mnt/user/0/emulated/<userid != 0> and a request
+    // on /storage/emulated/10 means /mnt/user/0/emulated/10
+    // TODO(b/146419093): Use lower filesystem paths that don't depend on sdcardfs
+    const std::string lowerPath = "/mnt/runtime/default/" + path.substr(9);
+    const std::string lowerAppDirRoot = "/mnt/runtime/default/" + appDirRoot.substr(9);
+
+    // First create the root which holds app dirs, if needed.
+    int ret = PrepareDirsFromRoot(lowerAppDirRoot, "/mnt/runtime/default/", 0771, AID_MEDIA_RW, AID_MEDIA_RW);
+    if (ret != 0) {
+        return ret;
+    }
+    // Then, create app-specific dirs with the correct UID/GID
+    return PrepareDirsFromRoot(lowerPath, lowerAppDirRoot, 0770, appUid, AID_MEDIA_RW);
 }
 
 int VolumeManager::createObb(const std::string& sourcePath, const std::string& sourceKey,
diff --git a/VolumeManager.h b/VolumeManager.h
index 9bf7599..cacab85 100644
--- a/VolumeManager.h
+++ b/VolumeManager.h
@@ -23,6 +23,7 @@
 
 #include <list>
 #include <mutex>
+#include <set>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -84,6 +85,8 @@
 
     void listVolumes(android::vold::VolumeBase::Type type, std::list<std::string>& list) const;
 
+    const std::set<userid_t>& getStartedUsers() const { return mStartedUsers; }
+
     int forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
     int onUserAdded(userid_t userId, int userSerialNumber);
@@ -91,6 +94,7 @@
     int onUserStarted(userid_t userId);
     int onUserStopped(userid_t userId);
 
+    void createPendingDisksIfNeeded();
     int onSecureKeyguardStateChanged(bool isShowing);
 
     int setPrimary(const std::shared_ptr<android::vold::VolumeBase>& vol);
@@ -110,13 +114,30 @@
     static VolumeManager* Instance();
 
     /*
-     * Ensure that all directories along given path exist, creating parent
-     * directories as needed.  Validates that given path is absolute and that
-     * it contains no relative "." or ".." paths or symlinks.  Last path segment
-     * is treated as filename and ignored, unless the path ends with "/".  Also
-     * ensures that path belongs to a volume managed by vold.
+     * Creates a directory 'path' for an application, automatically creating
+     * directories along the given path if they don't exist yet. 'appDirRoot'
+     * is the "root" directory for app-specific directories of this kind;
+     * 'path' must always start with 'appDirRoot'.
+     *
+     * Example:
+     *   path = /storage/emulated/0/Android/data/com.foo/files/
+     *   appDirRoot = /storage/emulated/0/Android/data/
+     *
+     * This function will set the UID of all app-specific directories below
+     * 'appDirRoot' to the 'appUid' argument. In the given example, the UID
+     * of /storage/emulated/0/Android/data/com.foo and
+     * /storage/emulated/0/Android/data/com.foo/files would be set to 'appUid'.
+     *
+     * The UID of the parent directories will be set according to the
+     * requirements of the underlying filesystem and are of no concern to the
+     * caller.
+     *
+     * Validates that given paths are absolute and that they contain no relative
+     * "." or ".." paths or symlinks.  Last path segment is treated as filename
+     * and ignored, unless the path ends with "/".  Also ensures that path
+     * belongs to a volume managed by vold.
      */
-    int mkdirs(const std::string& path);
+    int setupAppDir(const std::string& path, const std::string& appDirRoot, int32_t appUid);
 
     int createObb(const std::string& path, const std::string& key, int32_t ownerGid,
                   std::string* outVolId);
@@ -137,6 +158,9 @@
 
     int linkPrimary(userid_t userId);
 
+    void createEmulatedVolumesForUser(userid_t userId);
+    void destroyEmulatedVolumesForUser(userid_t userId);
+
     void handleDiskAdded(const std::shared_ptr<android::vold::Disk>& disk);
     void handleDiskChanged(dev_t device);
     void handleDiskRemoved(dev_t device);
@@ -151,13 +175,15 @@
     std::list<std::shared_ptr<android::vold::Disk>> mPendingDisks;
     std::list<std::shared_ptr<android::vold::VolumeBase>> mObbVolumes;
     std::list<std::shared_ptr<android::vold::VolumeBase>> mStubVolumes;
+    std::list<std::shared_ptr<android::vold::VolumeBase>> mInternalEmulatedVolumes;
 
     std::unordered_map<userid_t, int> mAddedUsers;
-    std::unordered_set<userid_t> mStartedUsers;
+    // This needs to be a regular set because we care about the ordering here;
+    // user 0 should always go first, because it is responsible for sdcardfs.
+    std::set<userid_t> mStartedUsers;
 
     std::string mVirtualDiskPath;
     std::shared_ptr<android::vold::Disk> mVirtualDisk;
-    std::shared_ptr<android::vold::VolumeBase> mInternalEmulated;
     std::shared_ptr<android::vold::VolumeBase> mPrimary;
 
     int mNextObbId;
diff --git a/binder/android/os/IVold.aidl b/binder/android/os/IVold.aidl
index 681e9dc..1819e09 100644
--- a/binder/android/os/IVold.aidl
+++ b/binder/android/os/IVold.aidl
@@ -18,6 +18,7 @@
 
 import android.os.incremental.IncrementalFileSystemControlParcel;
 import android.os.IVoldListener;
+import android.os.IVoldMountCallback;
 import android.os.IVoldTaskListener;
 
 /** {@hide} */
@@ -41,7 +42,8 @@
     void partition(@utf8InCpp String diskId, int partitionType, int ratio);
     void forgetPartition(@utf8InCpp String partGuid, @utf8InCpp String fsUuid);
 
-    void mount(@utf8InCpp String volId, int mountFlags, int mountUserId);
+    void mount(@utf8InCpp String volId, int mountFlags, int mountUserId,
+         @nullable IVoldMountCallback callback);
     void unmount(@utf8InCpp String volId);
     void format(@utf8InCpp String volId, @utf8InCpp String fsType);
     void benchmark(@utf8InCpp String volId, IVoldTaskListener listener);
@@ -52,7 +54,7 @@
 
     void remountUid(int uid, int remountMode);
 
-    void mkdirs(@utf8InCpp String path);
+    void setupAppDir(@utf8InCpp String path, @utf8InCpp String appDirRoot, int appUid);
 
     @utf8InCpp String createObb(@utf8InCpp String sourcePath, @utf8InCpp String sourceKey,
                                 int ownerGid);
@@ -90,6 +92,8 @@
 
     void addUserKeyAuth(int userId, int userSerial, @utf8InCpp String token,
                         @utf8InCpp String secret);
+    void clearUserKeyAuth(int userId, int userSerial, @utf8InCpp String token,
+                        @utf8InCpp String secret);
     void fixateNewestUserKeyAuth(int userId);
 
     void unlockUserKey(int userId, int userSerial, @utf8InCpp String token,
@@ -164,6 +168,8 @@
     const int REMOUNT_MODE_LEGACY = 4;
     const int REMOUNT_MODE_INSTALLER = 5;
     const int REMOUNT_MODE_FULL = 6;
+    const int REMOUNT_MODE_PASS_THROUGH = 7;
+    const int REMOUNT_MODE_ANDROID_WRITABLE = 8;
 
     const int VOLUME_STATE_UNMOUNTED = 0;
     const int VOLUME_STATE_CHECKING = 1;
diff --git a/binder/android/os/IVoldListener.aidl b/binder/android/os/IVoldListener.aidl
index 0dcfc04..b3e4ba5 100644
--- a/binder/android/os/IVoldListener.aidl
+++ b/binder/android/os/IVoldListener.aidl
@@ -25,7 +25,7 @@
     void onDiskDestroyed(@utf8InCpp String diskId);
 
     void onVolumeCreated(@utf8InCpp String volId,
-            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid);
+            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid, int userId);
     void onVolumeStateChanged(@utf8InCpp String volId, int state);
     void onVolumeMetadataChanged(@utf8InCpp String volId,
             @utf8InCpp String fsType, @utf8InCpp String fsUuid, @utf8InCpp String fsLabel);
diff --git a/binder/android/os/IVoldMountCallback.aidl b/binder/android/os/IVoldMountCallback.aidl
new file mode 100644
index 0000000..6bf46d7
--- /dev/null
+++ b/binder/android/os/IVoldMountCallback.aidl
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.os;
+
+/** {@hide} */
+interface IVoldMountCallback {
+    boolean onVolumeChecking(FileDescriptor fuseFd, @utf8InCpp String path,
+        @utf8InCpp String internalPath);
+}
diff --git a/model/EmulatedVolume.cpp b/model/EmulatedVolume.cpp
index 552fe2f..082dea5 100644
--- a/model/EmulatedVolume.cpp
+++ b/model/EmulatedVolume.cpp
@@ -15,10 +15,13 @@
  */
 
 #include "EmulatedVolume.h"
+
+#include "AppFuseUtil.h"
 #include "Utils.h"
 #include "VolumeManager.h"
 
 #include <android-base/logging.h>
+#include <android-base/properties.h>
 #include <android-base/stringprintf.h>
 #include <cutils/fs.h>
 #include <private/android_filesystem_config.h>
@@ -37,114 +40,279 @@
 namespace android {
 namespace vold {
 
-static const char* kFusePath = "/system/bin/sdcard";
+static const char* kSdcardFsPath = "/system/bin/sdcard";
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath)
-    : VolumeBase(Type::kEmulated), mFusePid(0) {
-    setId("emulated");
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, int userId)
+    : VolumeBase(Type::kEmulated) {
+    setId(StringPrintf("emulated;%u", userId));
     mRawPath = rawPath;
     mLabel = "emulated";
+    mFuseMounted = false;
+    mUseSdcardFs = IsFilesystemSupported("sdcardfs");
 }
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid)
-    : VolumeBase(Type::kEmulated), mFusePid(0) {
-    setId(StringPrintf("emulated:%u,%u", major(device), minor(device)));
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid,
+                               int userId)
+    : VolumeBase(Type::kEmulated) {
+    setId(StringPrintf("emulated:%u,%u;%u", major(device), minor(device), userId));
     mRawPath = rawPath;
     mLabel = fsUuid;
+    mFuseMounted = false;
+    mUseSdcardFs = IsFilesystemSupported("sdcardfs");
 }
 
 EmulatedVolume::~EmulatedVolume() {}
 
-status_t EmulatedVolume::doMount() {
+std::string EmulatedVolume::getLabel() {
     // We could have migrated storage to an adopted private volume, so always
     // call primary storage "emulated" to avoid media rescans.
-    std::string label = mLabel;
     if (getMountFlags() & MountFlags::kPrimary) {
-        label = "emulated";
+        return "emulated";
+    } else {
+        return mLabel;
+    }
+}
+
+// Creates a bind mount from source to target, creating the source (!) directory
+// if not yet present.
+static status_t doFuseBindMount(const std::string& source, const std::string& target) {
+    if (access(source.c_str(), F_OK) != 0) {
+        // Android path may not exist yet if users has just been created; create it on
+        // the lower fs.
+        if (fs_prepare_dir(source.c_str(), 0771, AID_MEDIA_RW, AID_MEDIA_RW) != 0) {
+            PLOG(ERROR) << "Failed to create " << source;
+            return -errno;
+        }
+    }
+    LOG(INFO) << "Bind mounting " << source << " on " << target;
+    auto status = BindMount(source, target);
+    if (status != OK) {
+        return status;
+    }
+    LOG(INFO) << "Bind mounted " << source << " on " << target;
+    return OK;
+}
+
+status_t EmulatedVolume::mountFuseBindMounts() {
+    std::string androidSource;
+    std::string label = getLabel();
+    int userId = getMountUserId();
+
+    if (mUseSdcardFs) {
+        androidSource = StringPrintf("/mnt/runtime/default/%s/%d/Android", label.c_str(), userId);
+    } else {
+        androidSource = StringPrintf("/%s/%d/Android", mRawPath.c_str(), userId);
+    }
+    std::string androidTarget(
+            StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
+
+    auto status = doFuseBindMount(androidSource, androidTarget);
+    if (status != OK) {
+        return status;
     }
 
-    mFuseDefault = StringPrintf("/mnt/runtime/default/%s", label.c_str());
-    mFuseRead = StringPrintf("/mnt/runtime/read/%s", label.c_str());
-    mFuseWrite = StringPrintf("/mnt/runtime/write/%s", label.c_str());
-    mFuseFull = StringPrintf("/mnt/runtime/full/%s", label.c_str());
+    // Installers get the same view as all other apps, with the sole exception that the
+    // OBB dirs (Android/obb) are writable to them. On sdcardfs devices, this requires
+    // a special bind mount, since app-private and OBB dirs share the same GID, but we
+    // only want to give access to the latter.
+    if (!mUseSdcardFs) {
+        return OK;
+    }
+    std::string installerSource(
+            StringPrintf("/mnt/runtime/write/%s/%d/Android/obb", label.c_str(), userId));
+    std::string installerTarget(
+            StringPrintf("/mnt/installer/%d/%s/%d/Android/obb", userId, label.c_str(), userId));
+
+    status = doFuseBindMount(installerSource, installerTarget);
+    if (status != OK) {
+        return status;
+    }
+    return OK;
+}
+
+status_t EmulatedVolume::unmountFuseBindMounts() {
+    std::string label = getLabel();
+    int userId = getMountUserId();
+
+    if (mUseSdcardFs) {
+        std::string installerTarget(
+                StringPrintf("/mnt/installer/%d/%s/%d/Android/obb", userId, label.c_str(), userId));
+        LOG(INFO) << "Unmounting " << installerTarget;
+        auto status = UnmountTree(installerTarget);
+        if (status != OK) {
+            LOG(ERROR) << "Failed to unmount " << installerTarget;
+            // Intentional continue to try to unmount the other bind mount
+        }
+    }
+
+    std::string androidTarget(
+            StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
+
+    LOG(INFO) << "Unmounting " << androidTarget;
+    auto status = UnmountTree(androidTarget);
+    if (status != OK) {
+        return status;
+    }
+    LOG(INFO) << "Unmounted " << androidTarget;
+
+    return OK;
+}
+
+status_t EmulatedVolume::doMount() {
+    std::string label = getLabel();
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
+
+    mSdcardFsDefault = StringPrintf("/mnt/runtime/default/%s", label.c_str());
+    mSdcardFsRead = StringPrintf("/mnt/runtime/read/%s", label.c_str());
+    mSdcardFsWrite = StringPrintf("/mnt/runtime/write/%s", label.c_str());
+    mSdcardFsFull = StringPrintf("/mnt/runtime/full/%s", label.c_str());
 
     setInternalPath(mRawPath);
     setPath(StringPrintf("/storage/%s", label.c_str()));
 
-    if (fs_prepare_dir(mFuseDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
+    if (fs_prepare_dir(mSdcardFsDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        fs_prepare_dir(mSdcardFsRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        fs_prepare_dir(mSdcardFsWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        fs_prepare_dir(mSdcardFsFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
         PLOG(ERROR) << getId() << " failed to create mount points";
         return -errno;
     }
 
-    dev_t before = GetDevice(mFuseFull);
+    dev_t before = GetDevice(mSdcardFsFull);
 
-    if (!(mFusePid = fork())) {
-        // clang-format off
-        if (execl(kFusePath, kFusePath,
-                "-u", "1023", // AID_MEDIA_RW
-                "-g", "1023", // AID_MEDIA_RW
-                "-m",
-                "-w",
-                "-G",
-                "-i",
-                "-o",
-                mRawPath.c_str(),
-                label.c_str(),
-                NULL)) {
-            // clang-format on
-            PLOG(ERROR) << "Failed to exec";
+    bool isFuse = base::GetBoolProperty(kPropFuse, false);
+
+    // Mount sdcardfs regardless of FUSE, since we need it to bind-mount on top of the
+    // FUSE volume for various reasons.
+    if (mUseSdcardFs && getMountUserId() == 0) {
+        LOG(INFO) << "Executing sdcardfs";
+        int sdcardFsPid;
+        if (!(sdcardFsPid = fork())) {
+            // clang-format off
+            if (execl(kSdcardFsPath, kSdcardFsPath,
+                    "-u", "1023", // AID_MEDIA_RW
+                    "-g", "1023", // AID_MEDIA_RW
+                    "-m",
+                    "-w",
+                    "-G",
+                    "-i",
+                    "-o",
+                    mRawPath.c_str(),
+                    label.c_str(),
+                    NULL)) {
+                // clang-format on
+                PLOG(ERROR) << "Failed to exec";
+            }
+
+            LOG(ERROR) << "sdcardfs exiting";
+            _exit(1);
         }
 
-        LOG(ERROR) << "FUSE exiting";
-        _exit(1);
-    }
-
-    if (mFusePid == -1) {
-        PLOG(ERROR) << getId() << " failed to fork";
-        return -errno;
-    }
-
-    nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
-    while (before == GetDevice(mFuseFull)) {
-        LOG(DEBUG) << "Waiting for FUSE to spin up...";
-        usleep(50000);  // 50ms
-
-        nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
-        if (nanoseconds_to_milliseconds(now - start) > 5000) {
-            LOG(WARNING) << "Timed out while waiting for FUSE to spin up";
-            return -ETIMEDOUT;
+        if (sdcardFsPid == -1) {
+            PLOG(ERROR) << getId() << " failed to fork";
+            return -errno;
         }
+
+        nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
+        while (before == GetDevice(mSdcardFsFull)) {
+            LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
+            usleep(50000);  // 50ms
+
+            nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
+            if (nanoseconds_to_milliseconds(now - start) > 5000) {
+                LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
+                return -ETIMEDOUT;
+            }
+        }
+        /* sdcardfs will have exited already. The filesystem will still be running */
+        TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
+        sdcardFsPid = 0;
     }
-    /* sdcardfs will have exited already. FUSE will still be running */
-    TEMP_FAILURE_RETRY(waitpid(mFusePid, nullptr, 0));
-    mFusePid = 0;
+    if (isFuse && isVisible) {
+        LOG(INFO) << "Mounting emulated fuse volume";
+        android::base::unique_fd fd;
+        int user_id = getMountUserId();
+        int result = MountUserFuse(user_id, getInternalPath(), label, &fd);
+
+        if (result != 0) {
+            PLOG(ERROR) << "Failed to mount emulated fuse volume";
+            return -result;
+        }
+
+        mFuseMounted = true;
+        auto callback = getMountCallback();
+        if (callback) {
+            bool is_ready = false;
+            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
+            if (!is_ready) {
+                fd.reset();
+                doUnmount();
+                return -EIO;
+            }
+        }
+
+        // Only do the bind-mounts when we know for sure the FUSE daemon can resolve the path.
+        status_t res = mountFuseBindMounts();
+        if (res != OK) {
+            fd.reset();
+            doUnmount();
+        }
+        return res;
+    }
 
     return OK;
 }
 
 status_t EmulatedVolume::doUnmount() {
-    // Unmount the storage before we kill the FUSE process. If we kill
-    // the FUSE process first, most file system operations will return
+    int userId = getMountUserId();
+
+    // Kill all processes using the filesystem before we unmount it. If we
+    // unmount the filesystem first, most file system operations will return
     // ENOTCONN until the unmount completes. This is an exotic and unusual
     // error code and might cause broken behaviour in applications.
-    KillProcessesUsingPath(getPath());
-    ForceUnmount(mFuseDefault);
-    ForceUnmount(mFuseRead);
-    ForceUnmount(mFuseWrite);
-    ForceUnmount(mFuseFull);
+    if (mFuseMounted) {
+        // For FUSE specifically, we have an emulated volume per user, so only kill
+        // processes using files from this particular user.
+        std::string user_path(StringPrintf("%s/%d", getPath().c_str(), getMountUserId()));
+        LOG(INFO) << "Killing all processes referencing " << user_path;
+        KillProcessesUsingPath(user_path);
+    } else {
+        KillProcessesUsingPath(getPath());
+    }
 
-    rmdir(mFuseDefault.c_str());
-    rmdir(mFuseRead.c_str());
-    rmdir(mFuseWrite.c_str());
-    rmdir(mFuseFull.c_str());
+    if (mFuseMounted) {
+        std::string label = getLabel();
+        // Ignoring unmount return status because we do want to try to unmount
+        // the rest cleanly.
 
-    mFuseDefault.clear();
-    mFuseRead.clear();
-    mFuseWrite.clear();
-    mFuseFull.clear();
+        unmountFuseBindMounts();
+        if (UnmountUserFuse(userId, getInternalPath(), label) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";
+            return -errno;
+        }
+
+        mFuseMounted = false;
+    }
+    if (getMountUserId() != 0 || !mUseSdcardFs) {
+        // For sdcardfs, only unmount for user 0, since user 0 will always be running
+        // and the paths don't change for different users.
+        return OK;
+    }
+
+    ForceUnmount(mSdcardFsDefault);
+    ForceUnmount(mSdcardFsRead);
+    ForceUnmount(mSdcardFsWrite);
+    ForceUnmount(mSdcardFsFull);
+
+    rmdir(mSdcardFsDefault.c_str());
+    rmdir(mSdcardFsRead.c_str());
+    rmdir(mSdcardFsWrite.c_str());
+    rmdir(mSdcardFsFull.c_str());
+
+    mSdcardFsDefault.clear();
+    mSdcardFsRead.clear();
+    mSdcardFsWrite.clear();
+    mSdcardFsFull.clear();
 
     return OK;
 }
diff --git a/model/EmulatedVolume.h b/model/EmulatedVolume.h
index fddfe4e..4f76a60 100644
--- a/model/EmulatedVolume.h
+++ b/model/EmulatedVolume.h
@@ -27,7 +27,7 @@
 /*
  * Shared storage emulated on top of private storage.
  *
- * Knows how to spawn a FUSE daemon to synthesize permissions.  ObbVolume
+ * Knows how to spawn a sdcardfs daemon to synthesize permissions.  ObbVolume
  * can be stacked above it.
  *
  * This volume is always multi-user aware, but is only binds itself to
@@ -37,8 +37,8 @@
  */
 class EmulatedVolume : public VolumeBase {
   public:
-    explicit EmulatedVolume(const std::string& rawPath);
-    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid);
+    explicit EmulatedVolume(const std::string& rawPath, int userId);
+    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid, int userId);
     virtual ~EmulatedVolume();
 
   protected:
@@ -46,16 +46,23 @@
     status_t doUnmount() override;
 
   private:
+    status_t mountFuseBindMounts();
+    status_t unmountFuseBindMounts();
+
+    std::string getLabel();
     std::string mRawPath;
     std::string mLabel;
 
-    std::string mFuseDefault;
-    std::string mFuseRead;
-    std::string mFuseWrite;
-    std::string mFuseFull;
+    std::string mSdcardFsDefault;
+    std::string mSdcardFsRead;
+    std::string mSdcardFsWrite;
+    std::string mSdcardFsFull;
 
-    /* PID of FUSE wrapper */
-    pid_t mFusePid;
+    /* Whether we mounted FUSE for this volume */
+    bool mFuseMounted;
+
+    /* Whether to use sdcardfs for this volume */
+    bool mUseSdcardFs;
 
     DISALLOW_COPY_AND_ASSIGN(EmulatedVolume);
 };
diff --git a/model/PrivateVolume.cpp b/model/PrivateVolume.cpp
index de2a09f..5098e5d 100644
--- a/model/PrivateVolume.cpp
+++ b/model/PrivateVolume.cpp
@@ -155,12 +155,18 @@
         return -EIO;
     }
 
-    // Create a new emulated volume stacked above us, it will automatically
-    // be destroyed during unmount
+    auto vol_manager = VolumeManager::Instance();
     std::string mediaPath(mPath + "/media");
-    auto vol = std::shared_ptr<VolumeBase>(new EmulatedVolume(mediaPath, mRawDevice, mFsUuid));
-    addVolume(vol);
-    vol->create();
+
+    // Create a new emulated volume stacked above us for all added users, they will automatically
+    // be destroyed during unmount
+    for (userid_t user : vol_manager->getStartedUsers()) {
+        auto vol = std::shared_ptr<VolumeBase>(
+                new EmulatedVolume(mediaPath, mRawDevice, mFsUuid, user));
+        vol->setMountUserId(user);
+        addVolume(vol);
+        vol->create();
+    }
 
     return OK;
 }
diff --git a/model/PrivateVolume.h b/model/PrivateVolume.h
index cb8e75d..656172f 100644
--- a/model/PrivateVolume.h
+++ b/model/PrivateVolume.h
@@ -42,6 +42,8 @@
     const std::string& getFsType() const { return mFsType; };
     const std::string& getRawDevPath() const { return mRawDevPath; };
     const std::string& getRawDmDevPath() const { return mDmDevPath; };
+    const std::string& getFsUuid() const { return mFsUuid; };
+    dev_t getRawDevice() const { return mRawDevice; };
 
   protected:
     status_t doCreate() override;
diff --git a/model/PublicVolume.cpp b/model/PublicVolume.cpp
index 0a6b351..b246c95 100644
--- a/model/PublicVolume.cpp
+++ b/model/PublicVolume.cpp
@@ -15,6 +15,8 @@
  */
 
 #include "PublicVolume.h"
+
+#include "AppFuseUtil.h"
 #include "Utils.h"
 #include "VolumeManager.h"
 #include "fs/Exfat.h"
@@ -41,13 +43,15 @@
 namespace android {
 namespace vold {
 
-static const char* kFusePath = "/system/bin/sdcard";
+static const char* kSdcardFsPath = "/system/bin/sdcard";
 
 static const char* kAsecPath = "/mnt/secure/asec";
 
-PublicVolume::PublicVolume(dev_t device) : VolumeBase(Type::kPublic), mDevice(device), mFusePid(0) {
+PublicVolume::PublicVolume(dev_t device) : VolumeBase(Type::kPublic), mDevice(device) {
     setId(StringPrintf("public:%u,%u", major(device), minor(device)));
     mDevPath = StringPrintf("/dev/block/vold/%s", getId().c_str());
+    mFuseMounted = false;
+    mUseSdcardFs = IsFilesystemSupported("sdcardfs");
 }
 
 PublicVolume::~PublicVolume() {}
@@ -93,6 +97,7 @@
 }
 
 status_t PublicVolume::doMount() {
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
     readMetadata();
 
     if (mFsType == "vfat" && vfat::IsSupported()) {
@@ -118,13 +123,13 @@
 
     mRawPath = StringPrintf("/mnt/media_rw/%s", stableName.c_str());
 
-    mFuseDefault = StringPrintf("/mnt/runtime/default/%s", stableName.c_str());
-    mFuseRead = StringPrintf("/mnt/runtime/read/%s", stableName.c_str());
-    mFuseWrite = StringPrintf("/mnt/runtime/write/%s", stableName.c_str());
-    mFuseFull = StringPrintf("/mnt/runtime/full/%s", stableName.c_str());
+    mSdcardFsDefault = StringPrintf("/mnt/runtime/default/%s", stableName.c_str());
+    mSdcardFsRead = StringPrintf("/mnt/runtime/read/%s", stableName.c_str());
+    mSdcardFsWrite = StringPrintf("/mnt/runtime/write/%s", stableName.c_str());
+    mSdcardFsFull = StringPrintf("/mnt/runtime/full/%s", stableName.c_str());
 
     setInternalPath(mRawPath);
-    if (getMountFlags() & MountFlags::kVisible) {
+    if (isVisible) {
         setPath(StringPrintf("/storage/%s", stableName.c_str()));
     } else {
         setPath(mRawPath);
@@ -152,72 +157,99 @@
         initAsecStage();
     }
 
-    if (!(getMountFlags() & MountFlags::kVisible)) {
-        // Not visible to apps, so no need to spin up FUSE
+    if (!isVisible) {
+        // Not visible to apps, so no need to spin up sdcardfs or FUSE
         return OK;
     }
 
-    if (fs_prepare_dir(mFuseDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
-        PLOG(ERROR) << getId() << " failed to create FUSE mount points";
-        return -errno;
-    }
-
-    dev_t before = GetDevice(mFuseFull);
-
-    if (!(mFusePid = fork())) {
-        if (getMountFlags() & MountFlags::kPrimary) {
-            // clang-format off
-            if (execl(kFusePath, kFusePath,
-                    "-u", "1023", // AID_MEDIA_RW
-                    "-g", "1023", // AID_MEDIA_RW
-                    "-U", std::to_string(getMountUserId()).c_str(),
-                    "-w",
-                    mRawPath.c_str(),
-                    stableName.c_str(),
-                    NULL)) {
-                // clang-format on
-                PLOG(ERROR) << "Failed to exec";
-            }
-        } else {
-            // clang-format off
-            if (execl(kFusePath, kFusePath,
-                    "-u", "1023", // AID_MEDIA_RW
-                    "-g", "1023", // AID_MEDIA_RW
-                    "-U", std::to_string(getMountUserId()).c_str(),
-                    mRawPath.c_str(),
-                    stableName.c_str(),
-                    NULL)) {
-                // clang-format on
-                PLOG(ERROR) << "Failed to exec";
-            }
+    if (mUseSdcardFs) {
+        if (fs_prepare_dir(mSdcardFsDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+            fs_prepare_dir(mSdcardFsRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+            fs_prepare_dir(mSdcardFsWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+            fs_prepare_dir(mSdcardFsFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
+            PLOG(ERROR) << getId() << " failed to create sdcardfs mount points";
+            return -errno;
         }
 
-        LOG(ERROR) << "FUSE exiting";
-        _exit(1);
+        dev_t before = GetDevice(mSdcardFsFull);
+
+        int sdcardFsPid;
+        if (!(sdcardFsPid = fork())) {
+            if (getMountFlags() & MountFlags::kPrimary) {
+                // clang-format off
+                if (execl(kSdcardFsPath, kSdcardFsPath,
+                        "-u", "1023", // AID_MEDIA_RW
+                        "-g", "1023", // AID_MEDIA_RW
+                        "-U", std::to_string(getMountUserId()).c_str(),
+                        "-w",
+                        mRawPath.c_str(),
+                        stableName.c_str(),
+                        NULL)) {
+                    // clang-format on
+                    PLOG(ERROR) << "Failed to exec";
+                }
+            } else {
+                // clang-format off
+                if (execl(kSdcardFsPath, kSdcardFsPath,
+                        "-u", "1023", // AID_MEDIA_RW
+                        "-g", "1023", // AID_MEDIA_RW
+                        "-U", std::to_string(getMountUserId()).c_str(),
+                        mRawPath.c_str(),
+                        stableName.c_str(),
+                        NULL)) {
+                    // clang-format on
+                    PLOG(ERROR) << "Failed to exec";
+                }
+            }
+
+            LOG(ERROR) << "sdcardfs exiting";
+            _exit(1);
+        }
+
+        if (sdcardFsPid == -1) {
+            PLOG(ERROR) << getId() << " failed to fork";
+            return -errno;
+        }
+
+        nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
+        while (before == GetDevice(mSdcardFsFull)) {
+            LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
+            usleep(50000);  // 50ms
+
+            nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
+            if (nanoseconds_to_milliseconds(now - start) > 5000) {
+                LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
+                return -ETIMEDOUT;
+            }
+        }
+        /* sdcardfs will have exited already. The filesystem will still be running */
+        TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
     }
 
-    if (mFusePid == -1) {
-        PLOG(ERROR) << getId() << " failed to fork";
-        return -errno;
-    }
+    bool isFuse = base::GetBoolProperty(kPropFuse, false);
+    if (isFuse) {
+        // We need to mount FUSE *after* sdcardfs, since the FUSE daemon may depend
+        // on sdcardfs being up.
+        LOG(INFO) << "Mounting public fuse volume";
+        android::base::unique_fd fd;
+        int user_id = getMountUserId();
+        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
 
-    nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
-    while (before == GetDevice(mFuseFull)) {
-        LOG(DEBUG) << "Waiting for FUSE to spin up...";
-        usleep(50000);  // 50ms
+        if (result != 0) {
+            LOG(ERROR) << "Failed to mount public fuse volume";
+            return -result;
+        }
 
-        nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
-        if (nanoseconds_to_milliseconds(now - start) > 5000) {
-            LOG(WARNING) << "Timed out while waiting for FUSE to spin up";
-            return -ETIMEDOUT;
+        mFuseMounted = true;
+        auto callback = getMountCallback();
+        if (callback) {
+            bool is_ready = false;
+            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
+            if (!is_ready) {
+                return -EIO;
+            }
         }
     }
-    /* sdcardfs will have exited already. FUSE will still be running */
-    TEMP_FAILURE_RETRY(waitpid(mFusePid, nullptr, 0));
-    mFusePid = 0;
 
     return OK;
 }
@@ -229,24 +261,41 @@
     // error code and might cause broken behaviour in applications.
     KillProcessesUsingPath(getPath());
 
+    if (mFuseMounted) {
+        // Use UUID as stable name, if available
+        std::string stableName = getId();
+        if (!mFsUuid.empty()) {
+            stableName = mFsUuid;
+        }
+
+        if (UnmountUserFuse(getMountUserId(), getInternalPath(), stableName) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on public fuse volume";
+            return -errno;
+        }
+
+        mFuseMounted = false;
+    }
+
     ForceUnmount(kAsecPath);
 
-    ForceUnmount(mFuseDefault);
-    ForceUnmount(mFuseRead);
-    ForceUnmount(mFuseWrite);
-    ForceUnmount(mFuseFull);
+    if (mUseSdcardFs) {
+        ForceUnmount(mSdcardFsDefault);
+        ForceUnmount(mSdcardFsRead);
+        ForceUnmount(mSdcardFsWrite);
+        ForceUnmount(mSdcardFsFull);
+
+        rmdir(mSdcardFsDefault.c_str());
+        rmdir(mSdcardFsRead.c_str());
+        rmdir(mSdcardFsWrite.c_str());
+        rmdir(mSdcardFsFull.c_str());
+
+        mSdcardFsDefault.clear();
+        mSdcardFsRead.clear();
+        mSdcardFsWrite.clear();
+        mSdcardFsFull.clear();
+    }
     ForceUnmount(mRawPath);
-
-    rmdir(mFuseDefault.c_str());
-    rmdir(mFuseRead.c_str());
-    rmdir(mFuseWrite.c_str());
-    rmdir(mFuseFull.c_str());
     rmdir(mRawPath.c_str());
-
-    mFuseDefault.clear();
-    mFuseRead.clear();
-    mFuseWrite.clear();
-    mFuseFull.clear();
     mRawPath.clear();
 
     return OK;
diff --git a/model/PublicVolume.h b/model/PublicVolume.h
index 2feccca..3156b53 100644
--- a/model/PublicVolume.h
+++ b/model/PublicVolume.h
@@ -27,7 +27,7 @@
 /*
  * Shared storage provided by public (vfat) partition.
  *
- * Knows how to mount itself and then spawn a FUSE daemon to synthesize
+ * Knows how to mount itself and then spawn a sdcardfs daemon to synthesize
  * permissions.  AsecVolume and ObbVolume can be stacked above it.
  *
  * This volume is not inherently multi-user aware, so it has two possible
@@ -60,13 +60,16 @@
     /* Mount point of raw partition */
     std::string mRawPath;
 
-    std::string mFuseDefault;
-    std::string mFuseRead;
-    std::string mFuseWrite;
-    std::string mFuseFull;
+    std::string mSdcardFsDefault;
+    std::string mSdcardFsRead;
+    std::string mSdcardFsWrite;
+    std::string mSdcardFsFull;
 
-    /* PID of FUSE wrapper */
-    pid_t mFusePid;
+    /* Whether we mounted FUSE for this volume */
+    bool mFuseMounted;
+
+    /* Whether to use sdcardfs for this volume */
+    bool mUseSdcardFs;
 
     /* Filesystem type */
     std::string mFsType;
diff --git a/model/VolumeBase.cpp b/model/VolumeBase.cpp
index ffc7900..636c065 100644
--- a/model/VolumeBase.cpp
+++ b/model/VolumeBase.cpp
@@ -143,6 +143,16 @@
     return OK;
 }
 
+status_t VolumeBase::setMountCallback(
+        const android::sp<android::os::IVoldMountCallback>& callback) {
+    mMountCallback = callback;
+    return OK;
+}
+
+sp<android::os::IVoldMountCallback> VolumeBase::getMountCallback() const {
+    return mMountCallback;
+}
+
 android::sp<android::os::IVoldListener> VolumeBase::getListener() const {
     if (mSilent) {
         return nullptr;
@@ -176,7 +186,8 @@
 
     auto listener = getListener();
     if (listener) {
-        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid);
+        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid,
+                                  mMountUserId);
     }
 
     setState(State::kUnmounted);
diff --git a/model/VolumeBase.h b/model/VolumeBase.h
index 53eeb6f..1d88d1b 100644
--- a/model/VolumeBase.h
+++ b/model/VolumeBase.h
@@ -19,6 +19,7 @@
 
 #include "Utils.h"
 #include "android/os/IVoldListener.h"
+#include "android/os/IVoldMountCallback.h"
 
 #include <cutils/multiuser.h>
 #include <utils/Errors.h>
@@ -87,11 +88,13 @@
     State getState() const { return mState; }
     const std::string& getPath() const { return mPath; }
     const std::string& getInternalPath() const { return mInternalPath; }
+    const std::list<std::shared_ptr<VolumeBase>>& getVolumes() const { return mVolumes; }
 
     status_t setDiskId(const std::string& diskId);
     status_t setPartGuid(const std::string& partGuid);
     status_t setMountFlags(int mountFlags);
     status_t setMountUserId(userid_t mountUserId);
+    status_t setMountCallback(const android::sp<android::os::IVoldMountCallback>& callback);
     status_t setSilent(bool silent);
 
     void addVolume(const std::shared_ptr<VolumeBase>& volume);
@@ -123,6 +126,7 @@
     status_t setInternalPath(const std::string& internalPath);
 
     android::sp<android::os::IVoldListener> getListener() const;
+    android::sp<android::os::IVoldMountCallback> getMountCallback() const;
 
   private:
     /* ID that uniquely references volume while alive */
@@ -147,6 +151,7 @@
     std::string mInternalPath;
     /* Flag indicating that volume should emit no events */
     bool mSilent;
+    android::sp<android::os::IVoldMountCallback> mMountCallback;
 
     /* Volumes stacked on top of this volume */
     std::list<std::shared_ptr<VolumeBase>> mVolumes;