diff --git a/service/native/libs/libclat/bpfhelper.cpp b/service/native/libs/libclat/bpfhelper.cpp index 00785ad817..475675fa56 100644 --- a/service/native/libs/libclat/bpfhelper.cpp +++ b/service/native/libs/libclat/bpfhelper.cpp @@ -82,43 +82,6 @@ void maybeStartBpf(const ClatdTracker& tracker) { } unique_fd rxProgFd(rv); - ClatEgress4Key txKey = { - .iif = tracker.v4ifIndex, - .local4 = tracker.v4, - }; - ClatEgress4Value txValue = { - .oif = tracker.ifIndex, - .local6 = tracker.v6, - .pfx96 = tracker.pfx96, - .oifIsEthernet = isEthernet.value(), - }; - - auto ret = mClatEgress4Map.writeValue(txKey, txValue, BPF_ANY); - if (!ret.ok()) { - ALOGE("mClatEgress4Map.writeValue failure: %s", strerror(ret.error().code())); - return; - } - - ClatIngress6Key rxKey = { - .iif = tracker.ifIndex, - .pfx96 = tracker.pfx96, - .local6 = tracker.v6, - }; - ClatIngress6Value rxValue = { - // TODO: move all the clat code to eBPF and remove the tun interface entirely. - .oif = tracker.v4ifIndex, - .local4 = tracker.v4, - }; - - ret = mClatIngress6Map.writeValue(rxKey, rxValue, BPF_ANY); - if (!ret.ok()) { - ALOGE("mClatIngress6Map.writeValue failure: %s", strerror(ret.error().code())); - ret = mClatEgress4Map.deleteValue(txKey); - if (!ret.ok()) - ALOGE("mClatEgress4Map.deleteValue failure: %s", strerror(ret.error().code())); - return; - } - // We do tc setup *after* populating the maps, so scanning through them // can always be used to tell us what needs cleanup. @@ -130,12 +93,6 @@ void maybeStartBpf(const ClatdTracker& tracker) { if (rv) { ALOGE("tcQdiscAddDevClsact(%d[%s]) failure: %s", tracker.v4ifIndex, tracker.v4iface, strerror(-rv)); - ret = mClatEgress4Map.deleteValue(txKey); - if (!ret.ok()) - ALOGE("mClatEgress4Map.deleteValue failure: %s", strerror(ret.error().code())); - ret = mClatIngress6Map.deleteValue(rxKey); - if (!ret.ok()) - ALOGE("mClatIngress6Map.deleteValue failure: %s", strerror(ret.error().code())); return; } @@ -148,12 +105,6 @@ void maybeStartBpf(const ClatdTracker& tracker) { // with interface addition, the lifetime is till interface deletion. Moreover, the clsact // has no clat filter now. It should not break anything. - ret = mClatEgress4Map.deleteValue(txKey); - if (!ret.ok()) - ALOGE("mClatEgress4Map.deleteValue failure: %s", strerror(ret.error().code())); - ret = mClatIngress6Map.deleteValue(rxKey); - if (!ret.ok()) - ALOGE("mClatIngress6Map.deleteValue failure: %s", strerror(ret.error().code())); return; } @@ -170,12 +121,6 @@ void maybeStartBpf(const ClatdTracker& tracker) { // The v4- interface clsact is not deleted. See the reason in the error unwinding code of // the egress filter attaching of v4- tun interface. - ret = mClatEgress4Map.deleteValue(txKey); - if (!ret.ok()) - ALOGE("mClatEgress4Map.deleteValue failure: %s", strerror(ret.error().code())); - ret = mClatIngress6Map.deleteValue(rxKey); - if (!ret.ok()) - ALOGE("mClatIngress6Map.deleteValue failure: %s", strerror(ret.error().code())); return; } @@ -194,26 +139,6 @@ void maybeStopBpf(const ClatdTracker& tracker) { ALOGE("tcFilterDelDevEgressClatIpv4(%d[%s]) failure: %s", tracker.v4ifIndex, tracker.v4iface, strerror(-rv)); } - - // We cleanup the maps last, so scanning through them can be used to - // determine what still needs cleanup. - - ClatEgress4Key txKey = { - .iif = tracker.v4ifIndex, - .local4 = tracker.v4, - }; - - auto ret = mClatEgress4Map.deleteValue(txKey); - if (!ret.ok()) ALOGE("mClatEgress4Map.deleteValue failure: %s", strerror(ret.error().code())); - - ClatIngress6Key rxKey = { - .iif = tracker.ifIndex, - .pfx96 = tracker.pfx96, - .local6 = tracker.v6, - }; - - ret = mClatIngress6Map.deleteValue(rxKey); - if (!ret.ok()) ALOGE("mClatIngress6Map.deleteValue failure: %s", strerror(ret.error().code())); } } // namespace clat diff --git a/service/src/com/android/server/connectivity/ClatCoordinator.java b/service/src/com/android/server/connectivity/ClatCoordinator.java index 2e26ae4f18..35b8f17c82 100644 --- a/service/src/com/android/server/connectivity/ClatCoordinator.java +++ b/service/src/com/android/server/connectivity/ClatCoordinator.java @@ -30,10 +30,19 @@ import android.net.IpPrefix; import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.os.ServiceSpecificException; +import android.system.ErrnoException; import android.util.Log; import com.android.internal.annotations.VisibleForTesting; +import com.android.modules.utils.build.SdkLevel; +import com.android.net.module.util.BpfMap; +import com.android.net.module.util.IBpfMap; import com.android.net.module.util.InterfaceParams; +import com.android.net.module.util.TcUtils; +import com.android.net.module.util.bpf.ClatEgress4Key; +import com.android.net.module.util.bpf.ClatEgress4Value; +import com.android.net.module.util.bpf.ClatIngress6Key; +import com.android.net.module.util.bpf.ClatIngress6Value; import java.io.FileDescriptor; import java.io.IOException; @@ -71,11 +80,22 @@ public class ClatCoordinator { private static final int INVALID_IFINDEX = 0; + private static final String CLAT_EGRESS4_MAP_PATH = makeMapPath("egress4"); + private static final String CLAT_INGRESS6_MAP_PATH = makeMapPath("ingress6"); + + private static String makeMapPath(String which) { + return "/sys/fs/bpf/map_clatd_clat_" + which + "_map"; + } + @NonNull private final INetd mNetd; @NonNull private final Dependencies mDeps; @Nullable + private final IBpfMap mIngressMap; + @Nullable + private final IBpfMap mEgressMap; + @Nullable private ClatdTracker mClatdTracker = null; @VisibleForTesting @@ -195,6 +215,45 @@ public class ClatCoordinator { public void untagSocket(long cookie) throws IOException { native_untagSocket(cookie); } + + /** Get ingress6 BPF map. */ + @Nullable + public IBpfMap getBpfIngress6Map() { + // Pre-T devices don't use ClatCoordinator to access clat map. Since Nat464Xlat + // initializes a ClatCoordinator object to avoid redundant null pointer check + // while using, ignore the BPF map initialization on pre-T devices. + // TODO: probably don't initialize ClatCoordinator object on pre-T devices. + if (!SdkLevel.isAtLeastT()) return null; + try { + return new BpfMap<>(CLAT_INGRESS6_MAP_PATH, + BpfMap.BPF_F_RDWR, ClatIngress6Key.class, ClatIngress6Value.class); + } catch (ErrnoException e) { + Log.e(TAG, "Cannot create ingress6 map: " + e); + return null; + } + } + + /** Get egress4 BPF map. */ + @Nullable + public IBpfMap getBpfEgress4Map() { + // Pre-T devices don't use ClatCoordinator to access clat map. Since Nat464Xlat + // initializes a ClatCoordinator object to avoid redundant null pointer check + // while using, ignore the BPF map initialization on pre-T devices. + // TODO: probably don't initialize ClatCoordinator object on pre-T devices. + if (!SdkLevel.isAtLeastT()) return null; + try { + return new BpfMap<>(CLAT_EGRESS4_MAP_PATH, + BpfMap.BPF_F_RDWR, ClatEgress4Key.class, ClatEgress4Value.class); + } catch (ErrnoException e) { + Log.e(TAG, "Cannot create egress4 map: " + e); + return null; + } + } + + /** Checks if the network interface uses an ethernet L2 header. */ + public boolean isEthernet(String iface) throws IOException { + return TcUtils.isEthernet(iface); + } } @VisibleForTesting @@ -268,6 +327,50 @@ public class ClatCoordinator { public ClatCoordinator(@NonNull Dependencies deps) { mDeps = deps; mNetd = mDeps.getNetd(); + mIngressMap = mDeps.getBpfIngress6Map(); + mEgressMap = mDeps.getBpfEgress4Map(); + } + + private void maybeStartBpf(final ClatdTracker tracker) { + if (mIngressMap == null || mEgressMap == null) return; + + final boolean isEthernet; + try { + isEthernet = mDeps.isEthernet(tracker.iface); + } catch (IOException e) { + Log.e(TAG, "Fail to call isEthernet for interface " + tracker.iface); + return; + } + + final ClatEgress4Key txKey = new ClatEgress4Key(tracker.v4ifIndex, tracker.v4); + final ClatEgress4Value txValue = new ClatEgress4Value(tracker.ifIndex, tracker.v6, + tracker.pfx96, (short) (isEthernet ? 1 /* ETHER */ : 0 /* RAWIP */)); + try { + mEgressMap.insertEntry(txKey, txValue); + } catch (ErrnoException | IllegalStateException e) { + Log.e(TAG, "Could not insert entry (" + txKey + ", " + txValue + ") on egress map: " + + e); + return; + } + + final ClatIngress6Key rxKey = new ClatIngress6Key(tracker.ifIndex, tracker.pfx96, + tracker.v6); + final ClatIngress6Value rxValue = new ClatIngress6Value(tracker.v4ifIndex, + tracker.v4); + try { + mIngressMap.insertEntry(rxKey, rxValue); + } catch (ErrnoException | IllegalStateException e) { + Log.e(TAG, "Could not insert entry (" + rxKey + ", " + rxValue + ") ingress map: " + + e); + try { + mEgressMap.deleteEntry(txKey); + } catch (ErrnoException | IllegalStateException e2) { + Log.e(TAG, "Could not delete entry (" + txKey + ") from egress map: " + e2); + } + return; + } + + // TODO: attach program. } /** @@ -454,9 +557,33 @@ public class ClatCoordinator { mClatdTracker = new ClatdTracker(iface, ifIndex, tunIface, tunIfIndex, v4, v6, pfx96, pid, cookie); + // [7] Start BPF + maybeStartBpf(mClatdTracker); + return v6Str; } + private void maybeStopBpf(final ClatdTracker tracker) { + if (mIngressMap == null || mEgressMap == null) return; + + final ClatEgress4Key txKey = new ClatEgress4Key(tracker.v4ifIndex, tracker.v4); + try { + mEgressMap.deleteEntry(txKey); + } catch (ErrnoException | IllegalStateException e) { + Log.e(TAG, "Could not delete entry (" + txKey + "): " + e); + } + + final ClatIngress6Key rxKey = new ClatIngress6Key(tracker.ifIndex, tracker.pfx96, + tracker.v6); + try { + mIngressMap.deleteEntry(rxKey); + } catch (ErrnoException | IllegalStateException e) { + Log.e(TAG, "Could not delete entry (" + rxKey + "): " + e); + } + + // TODO: dettach program. + } + /** * Stop clatd */ @@ -466,6 +593,7 @@ public class ClatCoordinator { } Log.i(TAG, "Stopping clatd pid=" + mClatdTracker.pid + " on " + mClatdTracker.iface); + maybeStopBpf(mClatdTracker); mDeps.stopClatd(mClatdTracker.iface, mClatdTracker.pfx96.getHostAddress(), mClatdTracker.v4.getHostAddress(), mClatdTracker.v6.getHostAddress(), mClatdTracker.pid); diff --git a/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java b/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java index 6c8b545b03..790473d5f7 100644 --- a/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java +++ b/tests/unit/java/com/android/server/connectivity/ClatCoordinatorTest.java @@ -41,6 +41,11 @@ import android.os.ParcelFileDescriptor; import androidx.test.filters.SmallTest; +import com.android.net.module.util.IBpfMap; +import com.android.net.module.util.bpf.ClatEgress4Key; +import com.android.net.module.util.bpf.ClatEgress4Value; +import com.android.net.module.util.bpf.ClatIngress6Key; +import com.android.net.module.util.bpf.ClatIngress6Value; import com.android.testutils.DevSdkIgnoreRule; import com.android.testutils.DevSdkIgnoreRunner; @@ -98,8 +103,19 @@ public class ClatCoordinatorTest { private static final ParcelFileDescriptor PACKET_SOCK_PFD = new ParcelFileDescriptor( new FileDescriptor()); + private static final ClatEgress4Key EGRESS_KEY = new ClatEgress4Key(STACKED_IFINDEX, + INET4_LOCAL4); + private static final ClatEgress4Value EGRESS_VALUE = new ClatEgress4Value(BASE_IFINDEX, + INET6_LOCAL6, INET6_PFX96, (short) 1 /* oifIsEthernet, 1 = true */); + private static final ClatIngress6Key INGRESS_KEY = new ClatIngress6Key(BASE_IFINDEX, + INET6_PFX96, INET6_LOCAL6); + private static final ClatIngress6Value INGRESS_VALUE = new ClatIngress6Value(STACKED_IFINDEX, + INET4_LOCAL4); + @Mock private INetd mNetd; @Spy private TestDependencies mDeps = new TestDependencies(); + @Mock private IBpfMap mIngressMap; + @Mock private IBpfMap mEgressMap; /** * The dependency injection class is used to mock the JNI functions and system functions @@ -298,6 +314,26 @@ public class ClatCoordinatorTest { fail("unsupported arg: " + cookie); } } + + /** Get ingress6 BPF map. */ + @Override + public IBpfMap getBpfIngress6Map() { + return mIngressMap; + } + + /** Get egress4 BPF map. */ + @Override + public IBpfMap getBpfEgress4Map() { + return mEgressMap; + } + + /** Checks if the network interface uses an ethernet L2 header. */ + public boolean isEthernet(String iface) throws IOException { + if (BASE_IFACE.equals(iface)) return true; + + fail("unsupported arg: " + iface); + return false; + } }; @NonNull @@ -322,8 +358,8 @@ public class ClatCoordinatorTest { @Test public void testStartStopClatd() throws Exception { final ClatCoordinator coordinator = makeClatCoordinator(); - final InOrder inOrder = inOrder(mNetd, mDeps); - clearInvocations(mNetd, mDeps); + final InOrder inOrder = inOrder(mNetd, mDeps, mIngressMap, mEgressMap); + clearInvocations(mNetd, mDeps, mIngressMap, mEgressMap); // [1] Start clatd. final String addr6For464xlat = coordinator.clatStart(BASE_IFACE, NETID, NAT64_IP_PREFIX); @@ -379,6 +415,8 @@ public class ClatCoordinatorTest { argThat(fd -> Objects.equals(RAW_SOCK_PFD.getFileDescriptor(), fd)), eq(BASE_IFACE), eq(NAT64_PREFIX_STRING), eq(XLAT_LOCAL_IPV4ADDR_STRING), eq(XLAT_LOCAL_IPV6ADDR_STRING)); + inOrder.verify(mEgressMap).insertEntry(eq(EGRESS_KEY), eq(EGRESS_VALUE)); + inOrder.verify(mIngressMap).insertEntry(eq(INGRESS_KEY), eq(INGRESS_VALUE)); inOrder.verifyNoMoreInteractions(); // [2] Start clatd again failed. @@ -388,6 +426,8 @@ public class ClatCoordinatorTest { // [3] Expect clatd to stop successfully. coordinator.clatStop(); + inOrder.verify(mEgressMap).deleteEntry(eq(EGRESS_KEY)); + inOrder.verify(mIngressMap).deleteEntry(eq(INGRESS_KEY)); inOrder.verify(mDeps).stopClatd(eq(BASE_IFACE), eq(NAT64_PREFIX_STRING), eq(XLAT_LOCAL_IPV4ADDR_STRING), eq(XLAT_LOCAL_IPV6ADDR_STRING), eq(CLATD_PID)); inOrder.verify(mDeps).untagSocket(eq(RAW_SOCK_COOKIE));