[CLATJ#21] ClatCoordinator: start clatd

launch clatd with the given cli arguments.

Bug: 212345928
Test: flash and boot
- run "atest ClatCoordinatorTest" in a follow commit.
- manual test
  1. Connect to ipv6-only wifi.
  2. Try IPv4 traffic.
     $ ping 8.8.8.8

Change-Id: Ie57a7c7a9d3d77396e91fa9f94c02f1ad05487c1

service/jni/com_android_server_connectivity_ClatCoordinator.cpp

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#define LOG_TAG "jniClatCoordinator"
 
 #include <arpa/inet.h>
 #include <errno.h>
@@ -22,6 +23,7 @@
 #include <log/log.h>
 #include <nativehelper/JNIHelp.h>
 #include <net/if.h>
+#include <spawn.h>
 #include <string>
 
 #include <netjniutils/netjniutils.h>
@@ -33,9 +35,16 @@
 // Sync from system/netd/include/netid_client.h
 #define MARK_UNSET 0u
 
+// Sync from system/netd/server/NetdConstants.h
+#define __INT_STRLEN(i) sizeof(#i)
+#define _INT_STRLEN(i) __INT_STRLEN(i)
+#define INT32_STRLEN _INT_STRLEN(INT32_MIN)
+
 #define DEVICEPREFIX "v4-"
 
 namespace android {
+static const char* kClatdPath = "/apex/com.android.tethering/bin/for-system/clatd";
+
 static void throwIOException(JNIEnv* env, const char* msg, int error) {
     jniThrowExceptionFmt(env, "java/io/IOException", "%s: %s", msg, strerror(error));
 }
@@ -282,8 +291,7 @@ int initTracker(const std::string& iface, const std::string& pfx96, const std::s
     return 0;
 }
 
-// TODO: fork clatd and rename to .._startClatd.
+static jint com_android_server_connectivity_ClatCoordinator_startClatd(
-static jint com_android_server_connectivity_ClatCoordinator_maybeStartBpf(
         JNIEnv* env, jobject clazz, jobject tunJavaFd, jobject readSockJavaFd,
         jobject writeSockJavaFd, jstring iface, jstring pfx96, jstring v4, jstring v6) {
     ScopedUtfChars ifaceStr(env, iface);
@@ -291,7 +299,121 @@ static jint com_android_server_connectivity_ClatCoordinator_maybeStartBpf(
     ScopedUtfChars v4Str(env, v4);
     ScopedUtfChars v6Str(env, v6);
 
-    // Start BPF if any
+    int tunFd = netjniutils::GetNativeFileDescriptor(env, tunJavaFd);
+    if (tunFd < 0) {
+        jniThrowExceptionFmt(env, "java/io/IOException", "Invalid tun file descriptor");
+        return -1;
+    }
+
+    int readSock = netjniutils::GetNativeFileDescriptor(env, readSockJavaFd);
+    if (readSock < 0) {
+        jniThrowExceptionFmt(env, "java/io/IOException", "Invalid read socket");
+        return -1;
+    }
+
+    int writeSock = netjniutils::GetNativeFileDescriptor(env, writeSockJavaFd);
+    if (writeSock < 0) {
+        jniThrowExceptionFmt(env, "java/io/IOException", "Invalid write socket");
+        return -1;
+    }
+
+    // 1. create a throwaway socket to reserve a file descriptor number
+    int passedTunFd = socket(AF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
+    if (passedTunFd == -1) {
+        throwIOException(env, "socket(ipv6/udp) for tun fd failed", errno);
+        return -1;
+    }
+    int passedSockRead = socket(AF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
+    if (passedSockRead == -1) {
+        throwIOException(env, "socket(ipv6/udp) for read socket failed", errno);
+        return -1;
+    }
+    int passedSockWrite = socket(AF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
+    if (passedSockWrite == -1) {
+        throwIOException(env, "socket(ipv6/udp) for write socket failed", errno);
+        return -1;
+    }
+
+    // these are the FD we'll pass to clatd on the cli, so need it as a string
+    char passedTunFdStr[INT32_STRLEN];
+    char passedSockReadStr[INT32_STRLEN];
+    char passedSockWriteStr[INT32_STRLEN];
+    snprintf(passedTunFdStr, sizeof(passedTunFdStr), "%d", passedTunFd);
+    snprintf(passedSockReadStr, sizeof(passedSockReadStr), "%d", passedSockRead);
+    snprintf(passedSockWriteStr, sizeof(passedSockWriteStr), "%d", passedSockWrite);
+
+    // 2. we're going to use this as argv[0] to clatd to make ps output more useful
+    std::string progname("clatd-");
+    progname += ifaceStr.c_str();
+
+    // clang-format off
+    const char* args[] = {progname.c_str(),
+                          "-i", ifaceStr.c_str(),
+                          "-p", pfx96Str.c_str(),
+                          "-4", v4Str.c_str(),
+                          "-6", v6Str.c_str(),
+                          "-t", passedTunFdStr,
+                          "-r", passedSockReadStr,
+                          "-w", passedSockWriteStr,
+                          nullptr};
+    // clang-format on
+
+    // 3. register vfork requirement
+    posix_spawnattr_t attr;
+    if (int ret = posix_spawnattr_init(&attr)) {
+        throwIOException(env, "posix_spawnattr_init failed", ret);
+        return -1;
+    }
+
+    // TODO: use android::base::ScopeGuard.
+    if (int ret = posix_spawnattr_setflags(&attr, POSIX_SPAWN_USEVFORK)) {
+        posix_spawnattr_destroy(&attr);
+        throwIOException(env, "posix_spawnattr_setflags failed", ret);
+        return -1;
+    }
+
+    // 4. register dup2() action: this is what 'clears' the CLOEXEC flag
+    // on the tun fd that we want the child clatd process to inherit
+    // (this will happen after the vfork, and before the execve)
+    posix_spawn_file_actions_t fa;
+    if (int ret = posix_spawn_file_actions_init(&fa)) {
+        posix_spawnattr_destroy(&attr);
+        throwIOException(env, "posix_spawn_file_actions_init failed", ret);
+        return -1;
+    }
+
+    if (int ret = posix_spawn_file_actions_adddup2(&fa, tunFd, passedTunFd)) {
+        posix_spawnattr_destroy(&attr);
+        posix_spawn_file_actions_destroy(&fa);
+        throwIOException(env, "posix_spawn_file_actions_adddup2 for tun fd failed", ret);
+        return -1;
+    }
+    if (int ret = posix_spawn_file_actions_adddup2(&fa, readSock, passedSockRead)) {
+        posix_spawnattr_destroy(&attr);
+        posix_spawn_file_actions_destroy(&fa);
+        throwIOException(env, "posix_spawn_file_actions_adddup2 for read socket failed", ret);
+        return -1;
+    }
+    if (int ret = posix_spawn_file_actions_adddup2(&fa, writeSock, passedSockWrite)) {
+        posix_spawnattr_destroy(&attr);
+        posix_spawn_file_actions_destroy(&fa);
+        throwIOException(env, "posix_spawn_file_actions_adddup2 for write socket failed", ret);
+        return -1;
+    }
+
+    // 5. actually perform vfork/dup2/execve
+    pid_t pid;
+    if (int ret = posix_spawn(&pid, kClatdPath, &fa, &attr, (char* const*)args, nullptr)) {
+        posix_spawnattr_destroy(&attr);
+        posix_spawn_file_actions_destroy(&fa);
+        throwIOException(env, "posix_spawn failed", ret);
+        return -1;
+    }
+
+    posix_spawnattr_destroy(&attr);
+    posix_spawn_file_actions_destroy(&fa);
+
+    // 5. Start BPF if any
     if (!net::clat::initMaps()) {
         net::clat::ClatdTracker tracker = {};
         if (!initTracker(ifaceStr.c_str(), pfx96Str.c_str(), v4Str.c_str(), v6Str.c_str(),
@@ -300,7 +422,7 @@ static jint com_android_server_connectivity_ClatCoordinator_maybeStartBpf(
         }
     }
 
-    return 0; // TODO: return forked clatd pid.
+    return pid;
 }
 
 // TODO: stop clatd and rename to .._stopClatd.
@@ -344,10 +466,10 @@ static const JNINativeMethod gMethods[] = {
          (void*)com_android_server_connectivity_ClatCoordinator_addAnycastSetsockopt},
         {"native_configurePacketSocket", "(Ljava/io/FileDescriptor;Ljava/lang/String;I)V",
          (void*)com_android_server_connectivity_ClatCoordinator_configurePacketSocket},
-        {"native_maybeStartBpf",
+        {"native_startClatd",
          "(Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;Ljava/lang/"
          "String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)I",
-         (void*)com_android_server_connectivity_ClatCoordinator_maybeStartBpf},
+         (void*)com_android_server_connectivity_ClatCoordinator_startClatd},
         {"native_maybeStopBpf",
          "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;I)V",
           (void*)com_android_server_connectivity_ClatCoordinator_maybeStopBpf},

service/src/com/android/server/connectivity/ClatCoordinator.java

@@ -170,12 +170,12 @@ public class ClatCoordinator {
         }
 
         /**
-         * Maybe start bpf.
+         * Start clatd.
          */
-        public int maybeStartBpf(@NonNull FileDescriptor tunfd, @NonNull FileDescriptor readsock6,
+        public int startClatd(@NonNull FileDescriptor tunfd, @NonNull FileDescriptor readsock6,
                 @NonNull FileDescriptor writesock6, @NonNull String iface, @NonNull String pfx96,
                 @NonNull String v4, @NonNull String v6) throws IOException {
-            return native_maybeStartBpf(tunfd, readsock6, writesock6, iface, pfx96, v4, v6);
+            return native_startClatd(tunfd, readsock6, writesock6, iface, pfx96, v4, v6);
         }
 
         /**
@@ -219,6 +219,9 @@ public class ClatCoordinator {
     public String clatStart(final String iface, final int netId,
             @NonNull final IpPrefix nat64Prefix)
             throws IOException {
+        if (mIface != null || mPid != INVALID_PID) {
+            throw new IOException("Clatd has started on " + mIface + " (pid " + mPid + ")");
+        }
         if (nat64Prefix.getPrefixLength() != 96) {
             throw new IOException("Prefix must be 96 bits long: " + nat64Prefix);
         }
@@ -327,20 +330,19 @@ public class ClatCoordinator {
             throw new IOException("configure packet socket failed: " + e);
         }
 
-        // [5] Maybe start bpf.
+        // [5] Start clatd.
         try {
-            mDeps.maybeStartBpf(tunFd.getFileDescriptor(), readSock6.getFileDescriptor(),
+            mPid = mDeps.startClatd(tunFd.getFileDescriptor(), readSock6.getFileDescriptor(),
                     writeSock6.getFileDescriptor(), iface, pfx96, v4, v6);
             mIface = iface;
             mNat64Prefix = pfx96;
             mXlatLocalAddress4 = v4;
             mXlatLocalAddress6 = v6;
         } catch (IOException e) {
-            throw new IOException("Error start bpf on " + iface + ": " + e);
+            throw new IOException("Error start clatd on " + iface + ": " + e);
         }
 
-        // TODO: start clatd and returns local xlat464 v6 address.
+        return v6;
-        return null;
     }
 
     /**
@@ -372,7 +374,7 @@ public class ClatCoordinator {
             int ifindex) throws IOException;
     private static native void native_configurePacketSocket(FileDescriptor sock, String v6,
             int ifindex) throws IOException;
-    private static native int native_maybeStartBpf(FileDescriptor tunfd, FileDescriptor readsock6,
+    private static native int native_startClatd(FileDescriptor tunfd, FileDescriptor readsock6,
             FileDescriptor writesock6, String iface, String pfx96, String v4, String v6)
             throws IOException;
     private static native void native_maybeStopBpf(String iface, String pfx96, String v4,