diff --git a/service/jni/com_android_server_connectivity_ClatCoordinator.cpp b/service/jni/com_android_server_connectivity_ClatCoordinator.cpp index e2c5a63f2b..de0e20a252 100644 --- a/service/jni/com_android_server_connectivity_ClatCoordinator.cpp +++ b/service/jni/com_android_server_connectivity_ClatCoordinator.cpp @@ -421,7 +421,7 @@ static void com_android_server_connectivity_ClatCoordinator_stopClatd(JNIEnv* en stopClatdProcess(pid); } -static jlong com_android_server_connectivity_ClatCoordinator_tagSocketAsClat( +static jlong com_android_server_connectivity_ClatCoordinator_getSocketCookie( JNIEnv* env, jobject clazz, jobject sockJavaFd) { int sockFd = netjniutils::GetNativeFileDescriptor(env, sockJavaFd); if (sockFd < 0) { @@ -435,58 +435,10 @@ static jlong com_android_server_connectivity_ClatCoordinator_tagSocketAsClat( return -1; } - bpf::BpfMap cookieTagMap; - auto res = cookieTagMap.init(COOKIE_TAG_MAP_PATH); - if (!res.ok()) { - throwIOException(env, "failed to init the cookieTagMap", res.error().code()); - return -1; - } - - // Tag raw socket with uid AID_CLAT and set tag as zero because tag is unused in bpf - // program for counting data usage in netd.c. Tagging socket is used to avoid counting - // duplicated clat traffic in bpf stat. - UidTagValue newKey = {.uid = (uint32_t)AID_CLAT, .tag = 0 /* unused */}; - res = cookieTagMap.writeValue(sock_cookie, newKey, BPF_ANY); - if (!res.ok()) { - jniThrowExceptionFmt(env, "java/io/IOException", "Failed to tag the socket: %s, fd: %d", - strerror(res.error().code()), cookieTagMap.getMap().get()); - return -1; - } - - ALOGI("tag uid AID_CLAT to socket fd %d, cookie %" PRIu64 "", sockFd, sock_cookie); + ALOGI("Get cookie %" PRIu64 " for socket fd %d", sock_cookie, sockFd); return static_cast(sock_cookie); } -static void com_android_server_connectivity_ClatCoordinator_untagSocket(JNIEnv* env, jobject clazz, - jlong cookie) { - uint64_t sock_cookie = static_cast(cookie); - if (sock_cookie == bpf::NONEXISTENT_COOKIE) { - jniThrowExceptionFmt(env, "java/io/IOException", "Invalid socket cookie"); - return; - } - - // The reason that deleting entry from cookie tag map directly is that the tag socket destroy - // listener only monitors on group INET_TCP, INET_UDP, INET6_TCP, INET6_UDP. The other socket - // types, ex: raw, are not able to be removed automatically by the listener. - // See TrafficController::makeSkDestroyListener. - bpf::BpfMap cookieTagMap; - auto res = cookieTagMap.init(COOKIE_TAG_MAP_PATH); - if (!res.ok()) { - throwIOException(env, "failed to init the cookieTagMap", res.error().code()); - return; - } - - res = cookieTagMap.deleteValue(sock_cookie); - if (!res.ok()) { - jniThrowExceptionFmt(env, "java/io/IOException", "Failed to untag the socket: %s", - strerror(res.error().code())); - return; - } - - ALOGI("untag socket cookie %" PRIu64 "", sock_cookie); - return; -} - /* * JNI registration. */ @@ -516,10 +468,8 @@ static const JNINativeMethod gMethods[] = { {"native_stopClatd", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;I)V", (void*)com_android_server_connectivity_ClatCoordinator_stopClatd}, - {"native_tagSocketAsClat", "(Ljava/io/FileDescriptor;)J", - (void*)com_android_server_connectivity_ClatCoordinator_tagSocketAsClat}, - {"native_untagSocket", "(J)V", - (void*)com_android_server_connectivity_ClatCoordinator_untagSocket}, + {"native_getSocketCookie", "(Ljava/io/FileDescriptor;)J", + (void*)com_android_server_connectivity_ClatCoordinator_getSocketCookie}, }; int register_com_android_server_connectivity_ClatCoordinator(JNIEnv* env) { diff --git a/service/src/com/android/server/connectivity/ClatCoordinator.java b/service/src/com/android/server/connectivity/ClatCoordinator.java index 5ea586af2e..6c4a021b53 100644 --- a/service/src/com/android/server/connectivity/ClatCoordinator.java +++ b/service/src/com/android/server/connectivity/ClatCoordinator.java @@ -46,6 +46,8 @@ import com.android.net.module.util.bpf.ClatEgress4Key; import com.android.net.module.util.bpf.ClatEgress4Value; import com.android.net.module.util.bpf.ClatIngress6Key; import com.android.net.module.util.bpf.ClatIngress6Value; +import com.android.net.module.util.bpf.CookieTagMapKey; +import com.android.net.module.util.bpf.CookieTagMapValue; import java.io.FileDescriptor; import java.io.IOException; @@ -63,6 +65,10 @@ import java.util.Objects; public class ClatCoordinator { private static final String TAG = ClatCoordinator.class.getSimpleName(); + // Sync from system/core/libcutils/include/private/android_filesystem_config.h + @VisibleForTesting + static final int AID_CLAT = 1029; + // Sync from external/android-clat/clatd.c // 40 bytes IPv6 header - 20 bytes IPv4 header + 8 bytes fragment header. @VisibleForTesting @@ -97,6 +103,8 @@ public class ClatCoordinator { @VisibleForTesting static final int PRIO_CLAT = 4; + private static final String COOKIE_TAG_MAP_PATH = + "/sys/fs/bpf/netd_shared/map_netd_cookie_tag_map"; private static final String CLAT_EGRESS4_MAP_PATH = makeMapPath("egress4"); private static final String CLAT_INGRESS6_MAP_PATH = makeMapPath("ingress6"); @@ -121,6 +129,8 @@ public class ClatCoordinator { @Nullable private final IBpfMap mEgressMap; @Nullable + private final IBpfMap mCookieTagMap; + @Nullable private ClatdTracker mClatdTracker = null; /** @@ -232,17 +242,10 @@ public class ClatCoordinator { } /** - * Tag socket as clat. + * Get socket cookie. */ - public long tagSocketAsClat(@NonNull FileDescriptor sock) throws IOException { - return native_tagSocketAsClat(sock); - } - - /** - * Untag socket. - */ - public void untagSocket(long cookie) throws IOException { - native_untagSocket(cookie); + public long getSocketCookie(@NonNull FileDescriptor sock) throws IOException { + return native_getSocketCookie(sock); } /** Get ingress6 BPF map. */ @@ -279,6 +282,23 @@ public class ClatCoordinator { } } + /** Get cookie tag map */ + @Nullable + public IBpfMap getBpfCookieTagMap() { + // Pre-T devices don't use ClatCoordinator to access clat map. Since Nat464Xlat + // initializes a ClatCoordinator object to avoid redundant null pointer check + // while using, ignore the BPF map initialization on pre-T devices. + // TODO: probably don't initialize ClatCoordinator object on pre-T devices. + if (!SdkLevel.isAtLeastT()) return null; + try { + return new BpfMap<>(COOKIE_TAG_MAP_PATH, + BpfMap.BPF_F_RDWR, CookieTagMapKey.class, CookieTagMapValue.class); + } catch (ErrnoException e) { + Log.wtf(TAG, "Cannot open cookie tag map: " + e); + return null; + } + } + /** Checks if the network interface uses an ethernet L2 header. */ public boolean isEthernet(String iface) throws IOException { return TcUtils.isEthernet(iface); @@ -388,6 +408,7 @@ public class ClatCoordinator { mNetd = mDeps.getNetd(); mIngressMap = mDeps.getBpfIngress6Map(); mEgressMap = mDeps.getBpfEgress4Map(); + mCookieTagMap = mDeps.getBpfCookieTagMap(); } private void maybeStartBpf(final ClatdTracker tracker) { @@ -536,6 +557,43 @@ public class ClatCoordinator { } } + private void tagSocketAsClat(long cookie) throws IOException { + if (mCookieTagMap == null) { + throw new IOException("Cookie tag map is not initialized"); + } + + // Tag raw socket with uid AID_CLAT and set tag as zero because tag is unused in bpf + // program for counting data usage in netd.c. Tagging socket is used to avoid counting + // duplicated clat traffic in bpf stat. + final CookieTagMapKey key = new CookieTagMapKey(cookie); + final CookieTagMapValue value = new CookieTagMapValue(AID_CLAT, 0 /* tag, unused */); + try { + mCookieTagMap.insertEntry(key, value); + } catch (ErrnoException | IllegalStateException e) { + throw new IOException("Could not insert entry (" + key + ", " + value + + ") on cookie tag map: " + e); + } + Log.i(TAG, "tag socket cookie " + cookie); + } + + private void untagSocket(long cookie) throws IOException { + if (mCookieTagMap == null) { + throw new IOException("Cookie tag map is not initialized"); + } + + // The reason that deleting entry from cookie tag map directly is that the tag socket + // destroy listener only monitors on group INET_TCP, INET_UDP, INET6_TCP, INET6_UDP. + // The other socket types, ex: raw, are not able to be removed automatically by the + // listener. See TrafficController::makeSkDestroyListener. + final CookieTagMapKey key = new CookieTagMapKey(cookie); + try { + mCookieTagMap.deleteEntry(key); + } catch (ErrnoException | IllegalStateException e) { + throw new IOException("Could not delete entry (" + key + ") on cookie tag map: " + e); + } + Log.i(TAG, "untag socket cookie " + cookie); + } + /** * Start clatd for a given interface and NAT64 prefix. */ @@ -686,7 +744,8 @@ public class ClatCoordinator { // Tag socket as AID_CLAT to avoid duplicated CLAT data usage accounting. final long cookie; try { - cookie = mDeps.tagSocketAsClat(writeSock6.getFileDescriptor()); + cookie = mDeps.getSocketCookie(writeSock6.getFileDescriptor()); + tagSocketAsClat(cookie); } catch (IOException e) { maybeCleanUp(tunFd, readSock6, writeSock6); throw new IOException("tag raw socket failed: " + e); @@ -696,6 +755,11 @@ public class ClatCoordinator { try { mDeps.configurePacketSocket(readSock6.getFileDescriptor(), v6Str, ifIndex); } catch (IOException e) { + try { + untagSocket(cookie); + } catch (IOException e2) { + Log.e(TAG, "untagSocket cookie " + cookie + " failed: " + e2); + } maybeCleanUp(tunFd, readSock6, writeSock6); throw new IOException("configure packet socket failed: " + e); } @@ -706,8 +770,11 @@ public class ClatCoordinator { pid = mDeps.startClatd(tunFd.getFileDescriptor(), readSock6.getFileDescriptor(), writeSock6.getFileDescriptor(), iface, pfx96Str, v4Str, v6Str); } catch (IOException e) { - // TODO: probably refactor to handle the exception of #untagSocket if any. - mDeps.untagSocket(cookie); + try { + untagSocket(cookie); + } catch (IOException e2) { + Log.e(TAG, "untagSocket cookie " + cookie + " failed: " + e2); + } throw new IOException("Error start clatd on " + iface + ": " + e); } finally { // The file descriptors have been duplicated (dup2) to clatd in native_startClatd(). @@ -774,7 +841,7 @@ public class ClatCoordinator { mDeps.stopClatd(mClatdTracker.iface, mClatdTracker.pfx96.getHostAddress(), mClatdTracker.v4.getHostAddress(), mClatdTracker.v6.getHostAddress(), mClatdTracker.pid); - mDeps.untagSocket(mClatdTracker.cookie); + untagSocket(mClatdTracker.cookie); Log.i(TAG, "clatd on " + mClatdTracker.iface + " stopped"); mClatdTracker = null; @@ -870,6 +937,5 @@ public class ClatCoordinator { throws IOException; private static native void native_stopClatd(String iface, String pfx96, String v4, String v6, int pid) throws IOException; - private static native long native_tagSocketAsClat(FileDescriptor sock) throws IOException; - private static native void native_untagSocket(long cookie) throws IOException; + private static native long native_getSocketCookie(FileDescriptor sock) throws IOException; } diff --git a/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java b/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java index feee2935db..bbb61cd3b9 100644 --- a/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java +++ b/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java @@ -21,6 +21,7 @@ import static android.system.OsConstants.ETH_P_IP; import static android.system.OsConstants.ETH_P_IPV6; import static com.android.net.module.util.NetworkStackConstants.ETHER_MTU; +import static com.android.server.connectivity.ClatCoordinator.AID_CLAT; import static com.android.server.connectivity.ClatCoordinator.CLAT_MAX_MTU; import static com.android.server.connectivity.ClatCoordinator.EGRESS; import static com.android.server.connectivity.ClatCoordinator.INGRESS; @@ -56,6 +57,8 @@ import com.android.net.module.util.bpf.ClatEgress4Key; import com.android.net.module.util.bpf.ClatEgress4Value; import com.android.net.module.util.bpf.ClatIngress6Key; import com.android.net.module.util.bpf.ClatIngress6Value; +import com.android.net.module.util.bpf.CookieTagMapKey; +import com.android.net.module.util.bpf.CookieTagMapValue; import com.android.testutils.DevSdkIgnoreRule; import com.android.testutils.DevSdkIgnoreRunner; import com.android.testutils.TestBpfMap; @@ -127,11 +130,16 @@ public class ClatCoordinatorTest { INET6_PFX96, INET6_LOCAL6); private static final ClatIngress6Value INGRESS_VALUE = new ClatIngress6Value(STACKED_IFINDEX, INET4_LOCAL4); + private static final CookieTagMapKey COOKIE_TAG_KEY = new CookieTagMapKey(RAW_SOCK_COOKIE); + private static final CookieTagMapValue COOKIE_TAG_VALUE = new CookieTagMapValue(AID_CLAT, + 0 /* tag, unused */); private final TestBpfMap mIngressMap = spy(new TestBpfMap<>(ClatIngress6Key.class, ClatIngress6Value.class)); private final TestBpfMap mEgressMap = spy(new TestBpfMap<>(ClatEgress4Key.class, ClatEgress4Value.class)); + private final TestBpfMap mCookieTagMap = + spy(new TestBpfMap<>(CookieTagMapKey.class, CookieTagMapValue.class)); @Mock private INetd mNetd; @Spy private TestDependencies mDeps = new TestDependencies(); @@ -313,25 +321,10 @@ public class ClatCoordinatorTest { } /** - * Tag socket as clat. + * Get socket cookie. */ - @Override - public long tagSocketAsClat(@NonNull FileDescriptor sock) throws IOException { - if (Objects.equals(RAW_SOCK_PFD.getFileDescriptor(), sock)) { - return RAW_SOCK_COOKIE; - } - fail("unsupported arg: " + sock); - return 0; - } - - /** - * Untag socket. - */ - @Override - public void untagSocket(long cookie) throws IOException { - if (cookie != RAW_SOCK_COOKIE) { - fail("unsupported arg: " + cookie); - } + public long getSocketCookie(@NonNull FileDescriptor sock) throws IOException { + return RAW_SOCK_COOKIE; } /** Get ingress6 BPF map. */ @@ -346,6 +339,12 @@ public class ClatCoordinatorTest { return mEgressMap; } + /** Get cookie tag map */ + @Override + public IBpfMap getBpfCookieTagMap() { + return mCookieTagMap; + } + /** Checks if the network interface uses an ethernet L2 header. */ public boolean isEthernet(String iface) throws IOException { if (BASE_IFACE.equals(iface)) return true; @@ -400,8 +399,8 @@ public class ClatCoordinatorTest { @Test public void testStartStopClatd() throws Exception { final ClatCoordinator coordinator = makeClatCoordinator(); - final InOrder inOrder = inOrder(mNetd, mDeps, mIngressMap, mEgressMap); - clearInvocations(mNetd, mDeps, mIngressMap, mEgressMap); + final InOrder inOrder = inOrder(mNetd, mDeps, mIngressMap, mEgressMap, mCookieTagMap); + clearInvocations(mNetd, mDeps, mIngressMap, mEgressMap, mCookieTagMap); // [1] Start clatd. final String addr6For464xlat = coordinator.clatStart(BASE_IFACE, NETID, NAT64_IP_PREFIX); @@ -444,8 +443,9 @@ public class ClatCoordinatorTest { inOrder.verify(mDeps).addAnycastSetsockopt( argThat(fd -> Objects.equals(RAW_SOCK_PFD.getFileDescriptor(), fd)), eq(XLAT_LOCAL_IPV6ADDR_STRING), eq(BASE_IFINDEX)); - inOrder.verify(mDeps).tagSocketAsClat( + inOrder.verify(mDeps).getSocketCookie( argThat(fd -> Objects.equals(RAW_SOCK_PFD.getFileDescriptor(), fd))); + inOrder.verify(mCookieTagMap).insertEntry(eq(COOKIE_TAG_KEY), eq(COOKIE_TAG_VALUE)); inOrder.verify(mDeps).configurePacketSocket( argThat(fd -> Objects.equals(PACKET_SOCK_PFD.getFileDescriptor(), fd)), eq(XLAT_LOCAL_IPV6ADDR_STRING), eq(BASE_IFINDEX)); @@ -481,7 +481,7 @@ public class ClatCoordinatorTest { inOrder.verify(mIngressMap).deleteEntry(eq(INGRESS_KEY)); inOrder.verify(mDeps).stopClatd(eq(BASE_IFACE), eq(NAT64_PREFIX_STRING), eq(XLAT_LOCAL_IPV4ADDR_STRING), eq(XLAT_LOCAL_IPV6ADDR_STRING), eq(CLATD_PID)); - inOrder.verify(mDeps).untagSocket(eq(RAW_SOCK_COOKIE)); + inOrder.verify(mCookieTagMap).deleteEntry(eq(COOKIE_TAG_KEY)); assertNull(coordinator.getClatdTrackerForTesting()); inOrder.verifyNoMoreInteractions(); @@ -679,18 +679,6 @@ public class ClatCoordinatorTest { true /* needToClosePacketSockFd */, true /* needToCloseRawSockFd */); } - @Test - public void testNotStartClatWithNativeFailureTagSocketAsClat() throws Exception { - class FailureDependencies extends TestDependencies { - @Override - public long tagSocketAsClat(@NonNull FileDescriptor sock) throws IOException { - throw new IOException(); - } - } - checkNotStartClat(new FailureDependencies(), true /* needToCloseTunFd */, - true /* needToClosePacketSockFd */, true /* needToCloseRawSockFd */); - } - @Test public void testNotStartClatWithNativeFailureConfigurePacketSocket() throws Exception { class FailureDependencies extends TestDependencies { @@ -718,4 +706,28 @@ public class ClatCoordinatorTest { checkNotStartClat(new FailureDependencies(), true /* needToCloseTunFd */, true /* needToClosePacketSockFd */, true /* needToCloseRawSockFd */); } + + @Test + public void testNotStartClatWithNativeFailureGetSocketCookie() throws Exception { + class FailureDependencies extends TestDependencies { + @Override + public long getSocketCookie(@NonNull FileDescriptor sock) throws IOException { + throw new IOException(); + } + } + checkNotStartClat(new FailureDependencies(), true /* needToCloseTunFd */, + true /* needToClosePacketSockFd */, true /* needToCloseRawSockFd */); + } + + @Test + public void testNotStartClatWithNullCookieTagMap() throws Exception { + class FailureDependencies extends TestDependencies { + @Override + public IBpfMap getBpfCookieTagMap() { + return null; + } + } + checkNotStartClat(new FailureDependencies(), true /* needToCloseTunFd */, + true /* needToClosePacketSockFd */, true /* needToCloseRawSockFd */); + } }