Allow callers to prepare CE/DE user storage.

Give callers the option of preparing CE and/or DE storage.  The
framework will only prepare CE storage after the CE keys have been
unlocked for that user.

When init is calling enablecrypto, kick off the work in a thread so
that we can make other calls back into vold without causing
deadlock.  Leaves blocking call intact for framework callers.

Clean up 'vdc' tool to send useful transaction numbers, and
actually watch for the matching result to come back.  This fixes
race conditions when there are multiple 'vdc' callers.

Also add other system and misc directories to match spec.

Bug: 25796509
Change-Id: Ie4f853db6e387916b845d2b5fb92925d743b063d
diff --git a/Android.mk b/Android.mk
index bea18b2..99a1739 100644
--- a/Android.mk
+++ b/Android.mk
@@ -88,7 +88,7 @@
 include $(CLEAR_VARS)
 
 LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk
-LOCAL_MODULE:= vold
+LOCAL_MODULE := vold
 LOCAL_CLANG := true
 LOCAL_SRC_FILES := \
 	main.cpp \
@@ -115,9 +115,9 @@
 
 LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk
 LOCAL_CLANG := true
-LOCAL_SRC_FILES:= vdc.cpp
-LOCAL_MODULE:= vdc
-LOCAL_SHARED_LIBRARIES := libcutils
+LOCAL_SRC_FILES := vdc.cpp
+LOCAL_MODULE := vdc
+LOCAL_SHARED_LIBRARIES := libcutils libbase
 LOCAL_CFLAGS := $(vold_cflags)
 LOCAL_CONLYFLAGS := $(vold_conlyflags)
 LOCAL_INIT_RC := vdc.rc
diff --git a/CryptCommandListener.cpp b/CryptCommandListener.cpp
index 2eac60e..75c840f 100644
--- a/CryptCommandListener.cpp
+++ b/CryptCommandListener.cpp
@@ -30,6 +30,7 @@
 #include <inttypes.h>
 
 #include <algorithm>
+#include <thread>
 
 #define LOG_TAG "VoldCryptCmdListener"
 
@@ -144,6 +145,25 @@
     return false;
 }
 
+static int do_enablecrypto(char** argv, int type, bool no_ui) {
+    int rc;
+    int tries;
+    for (tries = 0; tries < 2; ++tries) {
+        if (type == CRYPT_TYPE_DEFAULT) {
+            rc = cryptfs_enable_default(argv[2], no_ui);
+        } else {
+            rc = cryptfs_enable(argv[2], type, argv[4], no_ui);
+        }
+
+        if (rc == 0) {
+            return 0;
+        } else if (tries == 0) {
+            Process::killProcessesWithOpenFiles(DATA_MNT_POINT, SIGKILL);
+        }
+    }
+    return -1;
+}
+
 int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
                                                  int argc, char **argv) {
     if ((cli->getUid() != 0) && (cli->getUid() != AID_SYSTEM)) {
@@ -166,7 +186,10 @@
     } else if (subcommand == "restart") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
-        rc = cryptfs_restart();
+
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&cryptfs_restart).detach();
     } else if (subcommand == "cryptocomplete") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
@@ -216,31 +239,16 @@
             }
         }
 
-        if (!valid ) {
+        if (!valid) {
             cli->sendMsg(ResponseCode::CommandSyntaxError, syntax, false);
             return 0;
         }
 
         dumpArgs(argc, argv, 4);
 
-        int tries;
-        for (tries = 0; tries < 2; ++tries) {
-            if (type == -1) {
-                cli->sendMsg(ResponseCode::CommandSyntaxError, syntax,
-                             false);
-                return 0;
-            } else if (type == CRYPT_TYPE_DEFAULT) {
-              rc = cryptfs_enable_default(argv[2], no_ui);
-            } else {
-                rc = cryptfs_enable(argv[2], type, argv[4], no_ui);
-            }
-
-            if (rc == 0) {
-                break;
-            } else if (tries == 0) {
-                Process::killProcessesWithOpenFiles(DATA_MNT_POINT, SIGKILL);
-            }
-        }
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&do_enablecrypto, argv, type, no_ui).detach();
     } else if (subcommand == "enablefilecrypto") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
@@ -301,7 +309,10 @@
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         SLOGD("cryptfs mountdefaultencrypted");
         dumpArgs(argc, argv, -1);
-        rc = cryptfs_mount_default_encrypted();
+
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&cryptfs_mount_default_encrypted).detach();
     } else if (subcommand == "getpwtype") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         SLOGD("cryptfs getpwtype");
@@ -379,12 +390,12 @@
         return sendGenericOkFail(cli, e4crypt_lock_user_key(atoi(argv[2])));
 
     } else if (subcommand == "prepare_user_storage") {
-        if (!check_argc(cli, subcommand, argc, 6, "<uuid> <user> <serial> <ephemeral>")) return 0;
+        if (!check_argc(cli, subcommand, argc, 6, "<uuid> <user> <serial> <flags>")) return 0;
         return sendGenericOkFail(cli,
                                  e4crypt_prepare_user_storage(parseNull(argv[2]),
                                                               atoi(argv[3]),
                                                               atoi(argv[4]),
-                                                              atoi(argv[5]) != 0));
+                                                              atoi(argv[5])));
 
     } else {
         dumpArgs(argc, argv, -1);
diff --git a/Ext4Crypt.cpp b/Ext4Crypt.cpp
index 5ae906a..d66fdc6 100644
--- a/Ext4Crypt.cpp
+++ b/Ext4Crypt.cpp
@@ -40,7 +40,7 @@
 
 #include "key_control.h"
 #include "cryptfs.h"
-#include "ext4_crypt_init_extensions.h"
+#include "ext4_crypt.h"
 
 #define LOG_TAG "Ext4Crypt"
 
@@ -56,6 +56,10 @@
 
 using android::base::StringPrintf;
 
+// NOTE: keep in sync with StorageManager
+static constexpr int FLAG_STORAGE_DE = 1 << 0;
+static constexpr int FLAG_STORAGE_CE = 1 << 1;
+
 static bool e4crypt_is_native() {
     char value[PROPERTY_VALUE_MAX];
     property_get("ro.crypto.type", value, "none");
@@ -66,6 +70,10 @@
     return property_get_bool("persist.sys.emulate_fbe", false);
 }
 
+static const char* escape_null(const char* value) {
+    return (value == nullptr) ? "null" : value;
+}
+
 namespace {
     // Key length in bits
     const int key_length = 128;
@@ -282,8 +290,8 @@
     return true;
 }
 
-static bool set_policy(const std::string &raw_ref, const std::string& path) {
-    if (do_policy_set(path.c_str(), raw_ref.data(), raw_ref.size()) != 0) {
+static bool ensure_policy(const std::string &raw_ref, const std::string& path) {
+    if (e4crypt_policy_ensure(path.c_str(), raw_ref.data(), raw_ref.size()) != 0) {
         LOG(ERROR) << "Failed to set policy on: " << path;
         return false;
     }
@@ -395,13 +403,17 @@
             }
             if (!create_and_install_user_keys(0, false)) return -1;
         }
+        // TODO: switch to loading only DE_0 here once framework makes
+        // explicit calls to install DE keys for secondary users
         if (!load_all_de_keys()) return -1;
     }
-    // Ignore failures. FIXME this is horrid
-    // FIXME: we need an idempotent policy-setting call, which simply verifies the
-    // policy is already set on a second run, even if the directory is nonempty.
-    // Then we need to call it all the time.
-    e4crypt_prepare_user_storage(nullptr, 0, 0, false);
+    // We can only safely prepare DE storage here, since CE keys are probably
+    // entangled with user credentials.  The framework will always prepare CE
+    // storage once CE keys are installed.
+    if (e4crypt_prepare_user_storage(nullptr, 0, 0, FLAG_STORAGE_DE) != 0) {
+        LOG(ERROR) << "Failed to prepare user 0 storage";
+        return -1;
+    }
     return 0;
 }
 
@@ -484,6 +496,7 @@
     return 0;
 }
 
+// TODO: rename to 'install' for consistency, and take flags to know which keys to install
 int e4crypt_unlock_user_key(userid_t user_id, int serial, const char* token) {
     LOG(DEBUG) << "e4crypt_unlock_user_key " << user_id << " " << (token != nullptr);
     if (e4crypt_is_native()) {
@@ -505,6 +518,7 @@
     return 0;
 }
 
+// TODO: rename to 'evict' for consistency
 int e4crypt_lock_user_key(userid_t user_id) {
     if (e4crypt_is_native()) {
         // TODO: remove from kernel keyring
@@ -521,35 +535,48 @@
     return 0;
 }
 
-int e4crypt_prepare_user_storage(const char* volume_uuid,
-                                 userid_t user_id,
-                                 int serial,
-                                 bool ephemeral) {
-    if (volume_uuid) {
-        LOG(DEBUG) << "e4crypt_prepare_user_storage " << volume_uuid << " " << user_id;
-    } else {
-        LOG(DEBUG) << "e4crypt_prepare_user_storage, null volume " << user_id;
+int e4crypt_prepare_user_storage(const char* volume_uuid, userid_t user_id,
+        int serial, int flags) {
+    LOG(DEBUG) << "e4crypt_prepare_user_storage for volume " << escape_null(volume_uuid)
+            << ", user " << user_id << ", serial " << serial << ", flags " << flags;
+
+    if (flags & FLAG_STORAGE_DE) {
+        auto system_de_path = android::vold::BuildDataSystemDePath(user_id);
+        auto misc_de_path = android::vold::BuildDataMiscDePath(user_id);
+        auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
+
+        if (!prepare_dir(system_de_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
+        if (!prepare_dir(misc_de_path, 01771, AID_SYSTEM, AID_MISC)) return -1;
+        if (!prepare_dir(user_de_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
+
+        if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
+            std::string de_raw_ref;
+            if (!lookup_key_ref(s_de_key_raw_refs, user_id, de_raw_ref)) return -1;
+            if (!ensure_policy(de_raw_ref, system_de_path)) return -1;
+            if (!ensure_policy(de_raw_ref, misc_de_path)) return -1;
+            if (!ensure_policy(de_raw_ref, user_de_path)) return -1;
+        }
     }
-    auto system_ce_path = android::vold::BuildDataSystemCePath(user_id);
-    auto media_ce_path = android::vold::BuildDataMediaPath(volume_uuid, user_id);
-    auto user_ce_path = android::vold::BuildDataUserPath(volume_uuid, user_id);
-    auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
 
-    // FIXME: should this be 0770 or 0700?
-    if (!prepare_dir(system_ce_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
-    if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return -1;
-    if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
-    if (!prepare_dir(user_de_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
+    if (flags & FLAG_STORAGE_CE) {
+        auto system_ce_path = android::vold::BuildDataSystemCePath(user_id);
+        auto misc_ce_path = android::vold::BuildDataMiscCePath(user_id);
+        auto media_ce_path = android::vold::BuildDataMediaPath(volume_uuid, user_id);
+        auto user_ce_path = android::vold::BuildDataUserPath(volume_uuid, user_id);
 
-    if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
-        std::string ce_raw_ref, de_raw_ref;
-        if (!lookup_key_ref(s_ce_key_raw_refs, user_id, ce_raw_ref)) return -1;
-        if (!lookup_key_ref(s_de_key_raw_refs, user_id, de_raw_ref)) return -1;
-        if (!set_policy(ce_raw_ref, system_ce_path)) return -1;
-        if (!set_policy(ce_raw_ref, media_ce_path)) return -1;
-        if (!set_policy(ce_raw_ref, user_ce_path)) return -1;
-        if (!set_policy(de_raw_ref, user_de_path)) return -1;
-        // FIXME I thought there were more DE directories than this
+        if (!prepare_dir(system_ce_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
+        if (!prepare_dir(misc_ce_path, 01771, AID_SYSTEM, AID_MISC)) return -1;
+        if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return -1;
+        if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
+
+        if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
+            std::string ce_raw_ref;
+            if (!lookup_key_ref(s_ce_key_raw_refs, user_id, ce_raw_ref)) return -1;
+            if (!ensure_policy(ce_raw_ref, system_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, misc_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, media_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, user_ce_path)) return -1;
+        }
     }
 
     return 0;
diff --git a/Ext4Crypt.h b/Ext4Crypt.h
index d60af6e..d732e86 100644
--- a/Ext4Crypt.h
+++ b/Ext4Crypt.h
@@ -32,9 +32,7 @@
 int e4crypt_unlock_user_key(userid_t user_id, int serial, const char* token);
 int e4crypt_lock_user_key(userid_t user_id);
 
-int e4crypt_prepare_user_storage(const char* volume_uuid,
-                                 userid_t user_id,
-                                 int serial,
-                                 bool ephemeral);
+int e4crypt_prepare_user_storage(const char* volume_uuid, userid_t user_id,
+        int serial, int flags);
 
 __END_DECLS
diff --git a/Utils.cpp b/Utils.cpp
index 9f0e0f3..1d1b236 100644
--- a/Utils.cpp
+++ b/Utils.cpp
@@ -561,9 +561,19 @@
 }
 
 std::string BuildDataSystemCePath(userid_t userId) {
-    // TODO: unify with installd path generation logic
-    std::string data(BuildDataPath(nullptr));
-    return StringPrintf("%s/system_ce/%u", data.c_str(), userId);
+    return StringPrintf("%s/system_ce/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataSystemDePath(userid_t userId) {
+    return StringPrintf("%s/system_de/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataMiscCePath(userid_t userId) {
+    return StringPrintf("%s/misc_ce/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataMiscDePath(userid_t userId) {
+    return StringPrintf("%s/misc_de/%u", BuildDataPath(nullptr).c_str(), userId);
 }
 
 std::string BuildDataPath(const char* volumeUuid) {
diff --git a/Utils.h b/Utils.h
index f717da5..7ff92c8 100644
--- a/Utils.h
+++ b/Utils.h
@@ -97,6 +97,9 @@
 std::string BuildKeyPath(const std::string& partGuid);
 
 std::string BuildDataSystemCePath(userid_t userid);
+std::string BuildDataSystemDePath(userid_t userid);
+std::string BuildDataMiscCePath(userid_t userid);
+std::string BuildDataMiscDePath(userid_t userid);
 
 std::string BuildDataPath(const char* volumeUuid);
 std::string BuildDataMediaPath(const char* volumeUuid, userid_t userid);
diff --git a/cryptfs.c b/cryptfs.c
index 3b49a73..bd5807f 100644
--- a/cryptfs.c
+++ b/cryptfs.c
@@ -1582,6 +1582,9 @@
 {
     int i;
 
+    // NOTE: post_fs_data results in init calling back around to vold, so all
+    // callers to this method must be async
+
     /* Do the prep of the /data filesystem */
     property_set("vold.post_fs_data_done", "0");
     property_set("vold.decrypt", "trigger_post_fs_data");
diff --git a/vdc.cpp b/vdc.cpp
index d8476b7..4eb26cd 100644
--- a/vdc.cpp
+++ b/vdc.cpp
@@ -22,6 +22,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
+#include <poll.h>
 
 #include <sys/socket.h>
 #include <sys/select.h>
@@ -29,6 +30,8 @@
 #include <sys/types.h>
 #include <sys/un.h>
 
+#include <android-base/stringprintf.h>
+
 #include <cutils/sockets.h>
 #include <private/android_filesystem_config.h>
 
@@ -36,6 +39,8 @@
 static int do_monitor(int sock, int stop_after_cmd);
 static int do_cmd(int sock, int argc, char **argv);
 
+static constexpr int kCommandTimeoutMs = 20 * 1000;
+
 int main(int argc, char **argv) {
     int sock;
     int wait_for_socket;
@@ -44,12 +49,12 @@
     progname = argv[0];
 
     wait_for_socket = argc > 1 && strcmp(argv[1], "--wait") == 0;
-    if(wait_for_socket) {
+    if (wait_for_socket) {
         argv++;
         argc--;
     }
 
-    if(argc < 2) {
+    if (argc < 2) {
         usage(progname);
         exit(5);
     }
@@ -62,8 +67,8 @@
     while ((sock = socket_local_client(sockname,
                                  ANDROID_SOCKET_NAMESPACE_RESERVED,
                                  SOCK_STREAM)) < 0) {
-        if(!wait_for_socket) {
-            fprintf(stderr, "Error connecting (%s)\n", strerror(errno));
+        if (!wait_for_socket) {
+            fprintf(stdout, "Error connecting to %s: %s\n", sockname, strerror(errno));
             exit(4);
         } else {
             usleep(10000);
@@ -78,97 +83,92 @@
 }
 
 static int do_cmd(int sock, int argc, char **argv) {
-    char final_cmd[255] = "0 "; /* 0 is a (now required) sequence number */
+    int seq = getpid();
 
-    int i;
-    size_t ret;
+    std::string cmd(android::base::StringPrintf("%d ", seq));
+    for (int i = 1; i < argc; i++) {
+        if (!strchr(argv[i], ' ')) {
+            cmd.append(argv[i]);
+        } else {
+            cmd.push_back('\"');
+            cmd.append(argv[i]);
+            cmd.push_back('\"');
+        }
 
-    for (i = 1; i < argc; i++) {
-        char *cmp;
-
-        if (!strchr(argv[i], ' '))
-            asprintf(&cmp, "%s%s", argv[i], (i == (argc -1)) ? "" : " ");
-        else
-            asprintf(&cmp, "\"%s\"%s", argv[i], (i == (argc -1)) ? "" : " ");
-
-        ret = strlcat(final_cmd, cmp, sizeof(final_cmd));
-        if (ret >= sizeof(final_cmd))
-            abort();
-        free(cmp);
+        if (i < argc - 1) {
+            cmd.push_back(' ');
+        }
     }
 
-    if (write(sock, final_cmd, strlen(final_cmd) + 1) < 0) {
-        perror("write");
+    if (TEMP_FAILURE_RETRY(write(sock, cmd.c_str(), cmd.length() + 1)) < 0) {
+        fprintf(stderr, "Failed to write command: %s\n", strerror(errno));
         return errno;
     }
 
-    return do_monitor(sock, 1);
+    return do_monitor(sock, seq);
 }
 
-static int do_monitor(int sock, int stop_after_cmd) {
-    char *buffer = (char *) malloc(4096);
+static int do_monitor(int sock, int stop_after_seq) {
+    char buffer[4096];
+    int timeout = kCommandTimeoutMs;
 
-    if (!stop_after_cmd)
-        printf("[Connected to Vold]\n");
+    if (stop_after_seq == 0) {
+        fprintf(stderr, "Connected to vold\n");
+        timeout = -1;
+    }
 
-    while(1) {
-        fd_set read_fds;
-        struct timeval to;
-        int rc = 0;
-
-        to.tv_sec = 10;
-        to.tv_usec = 0;
-
-        FD_ZERO(&read_fds);
-        FD_SET(sock, &read_fds);
-
-        if ((rc = select(sock +1, &read_fds, NULL, NULL, &to)) < 0) {
-            fprintf(stderr, "Error in select (%s)\n", strerror(errno));
-            free(buffer);
-            return errno;
-        } else if (!rc) {
-            continue;
-            fprintf(stderr, "[TIMEOUT]\n");
+    while (1) {
+        struct pollfd poll_sock = { sock, POLLIN, 0 };
+        int rc = TEMP_FAILURE_RETRY(poll(&poll_sock, 1, timeout));
+        if (rc == 0) {
+            fprintf(stderr, "Timeout waiting for %d\n", stop_after_seq);
             return ETIMEDOUT;
-        } else if (FD_ISSET(sock, &read_fds)) {
-            memset(buffer, 0, 4096);
-            if ((rc = read(sock, buffer, 4096)) <= 0) {
-                if (rc == 0)
-                    fprintf(stderr, "Lost connection to Vold - did it crash?\n");
-                else
-                    fprintf(stderr, "Error reading data (%s)\n", strerror(errno));
-                free(buffer);
-                if (rc == 0)
-                    return ECONNRESET;
-                return errno;
-            }
+        } else if (rc < 0) {
+            fprintf(stderr, "Failed during poll: %s\n", strerror(errno));
+            return errno;
+        }
 
-            int offset = 0;
-            int i = 0;
+        if (!(poll_sock.revents & POLLIN)) {
+            fprintf(stderr, "No data; trying again\n");
+            continue;
+        }
 
-            for (i = 0; i < rc; i++) {
-                if (buffer[i] == '\0') {
-                    int code;
-                    char tmp[4];
+        memset(buffer, 0, sizeof(buffer));
+        rc = TEMP_FAILURE_RETRY(read(sock, buffer, sizeof(buffer)));
+        if (rc == 0) {
+            fprintf(stderr, "Lost connection, did vold crash?\n");
+            return ECONNRESET;
+        } else if (rc < 0) {
+            fprintf(stderr, "Error reading data: %s\n", strerror(errno));
+            return errno;
+        }
 
-                    strlcpy(tmp, buffer + offset, sizeof(tmp));
-                    code = atoi(tmp);
+        int offset = 0;
+        for (int i = 0; i < rc; i++) {
+            if (buffer[i] == '\0') {
+                char* res = buffer + offset;
+                fprintf(stdout, "%s\n", res);
 
-                    printf("%s\n", buffer + offset);
-                    if (stop_after_cmd) {
-                        if (code >= 200 && code < 600)
+                int code = atoi(strtok(res, " "));
+                if (code >= 200 && code < 600) {
+                    int seq = atoi(strtok(nullptr, " "));
+                    if (seq == stop_after_seq) {
+                        if (code == 200) {
                             return 0;
+                        } else {
+                            return code;
+                        }
                     }
-                    offset = i + 1;
                 }
+
+                offset = i + 1;
             }
         }
     }
-    free(buffer);
-    return 0;
+    return EIO;
 }
 
 static void usage(char *progname) {
     fprintf(stderr,
             "Usage: %s [--wait] <monitor>|<cmd> [arg1] [arg2...]\n", progname);
- }
+}