Refresh conntrack entry timeout

Needed because the BPF maps offloads IPv4 traffic. The kernel can't
trace the offloaded traffic to keep the conntrack entry.

Bug: 190783768
Test: atest TetheringCoverageTests
Original-Change: https://android-review.googlesource.com/1566871
Merged-In: Idbcf686c9b2124b192944156ac5111be741744fb
Change-Id: Idbcf686c9b2124b192944156ac5111be741744fb

Tethering/src/com/android/networkstack/tethering/BpfCoordinator.java

@@ -41,7 +41,9 @@ import android.net.TetherOffloadRuleParcel;
 import android.net.ip.ConntrackMonitor;
 import android.net.ip.ConntrackMonitor.ConntrackEventConsumer;
 import android.net.ip.IpServer;
+import android.net.netlink.ConntrackMessage;
 import android.net.netlink.NetlinkConstants;
+import android.net.netlink.NetlinkSocket;
 import android.net.netstats.provider.NetworkStatsProvider;
 import android.net.util.InterfaceParams;
 import android.net.util.SharedLog;
@@ -49,6 +51,7 @@ import android.net.util.TetheringUtils.ForwardedStats;
 import android.os.Handler;
 import android.os.SystemClock;
 import android.system.ErrnoException;
+import android.system.OsConstants;
 import android.text.TextUtils;
 import android.util.ArraySet;
 import android.util.Log;
@@ -120,6 +123,13 @@ public class BpfCoordinator {
         return makeMapPath((downstream ? "downstream" : "upstream") + ipVersion);
     }
 
+    @VisibleForTesting
+    static final int POLLING_CONNTRACK_TIMEOUT_MS = 60_000;
+    @VisibleForTesting
+    static final int NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED = 432000;
+    @VisibleForTesting
+    static final int NF_CONNTRACK_UDP_TIMEOUT_STREAM = 180;
+
     @VisibleForTesting
     enum StatsType {
         STATS_PER_IFACE,
@@ -234,11 +244,17 @@ public class BpfCoordinator {
     private int mLastIPv4UpstreamIfindex = 0;
 
     // Runnable that used by scheduling next polling of stats.
-    private final Runnable mScheduledPollingTask = () -> {
+    private final Runnable mScheduledPollingStats = () -> {
         updateForwardedStats();
         maybeSchedulePollingStats();
     };
 
+    // Runnable that used by scheduling next polling of conntrack timeout.
+    private final Runnable mScheduledPollingConntrackTimeout = () -> {
+        maybeRefreshConntrackTimeout();
+        maybeSchedulePollingConntrackTimeout();
+    };
+
     // TODO: add BpfMap<TetherDownstream64Key, TetherDownstream64Value> retrieving function.
     @VisibleForTesting
     public abstract static class Dependencies {
@@ -267,6 +283,13 @@ public class BpfCoordinator {
             return InterfaceParams.getByName(ifName);
         }
 
+        /**
+         * Represents an estimate of elapsed time since boot in nanoseconds.
+         */
+        public long elapsedRealtimeNanos() {
+            return SystemClock.elapsedRealtimeNanos();
+        }
+
         /**
          * Check OS Build at least S.
          *
@@ -274,7 +297,6 @@ public class BpfCoordinator {
          * testing different code flows concurrently.
          */
         public boolean isAtLeastS() {
-            // TODO: consider using ShimUtils.isAtLeastS.
             return SdkLevel.isAtLeastS();
         }
 
@@ -412,6 +434,7 @@ public class BpfCoordinator {
 
         mPollingStarted = true;
         maybeSchedulePollingStats();
+        maybeSchedulePollingConntrackTimeout();
 
         mLog.i("Polling started");
     }
@@ -427,9 +450,13 @@ public class BpfCoordinator {
     public void stopPolling() {
         if (!mPollingStarted) return;
 
-        // Stop scheduled polling tasks and poll the latest stats from BPF maps.
+        // Stop scheduled polling conntrack timeout.
-        if (mHandler.hasCallbacks(mScheduledPollingTask)) {
+        if (mHandler.hasCallbacks(mScheduledPollingConntrackTimeout)) {
-            mHandler.removeCallbacks(mScheduledPollingTask);
+            mHandler.removeCallbacks(mScheduledPollingConntrackTimeout);
+        }
+        // Stop scheduled polling stats and poll the latest stats from BPF maps.
+        if (mHandler.hasCallbacks(mScheduledPollingStats)) {
+            mHandler.removeCallbacks(mScheduledPollingStats);
         }
         updateForwardedStats();
         mPollingStarted = false;
@@ -1412,12 +1439,86 @@ public class BpfCoordinator {
         return addr6;
     }
 
-    // Support raw ip only.
+    @Nullable
-    // TODO: add ether ip support.
+    private Inet4Address ipv4MappedAddressBytesToIpv4Address(final byte[] addr46) {
+        if (addr46.length != 16) return null;
+        if (addr46[0] != 0 || addr46[1] != 0 || addr46[2] != 0 || addr46[3] != 0
+                || addr46[4] != 0 || addr46[5] != 0 || addr46[6] != 0 || addr46[7] != 0
+                || addr46[8] != 0 && addr46[9] != 0 || (addr46[10] & 0xff) != 0xff
+                || (addr46[11] & 0xff) != 0xff) {
+            return null;
+        }
+
+        final byte[] addr4 = new byte[4];
+        addr4[0] = addr46[12];
+        addr4[1] = addr46[13];
+        addr4[2] = addr46[14];
+        addr4[3] = addr46[15];
+
+        return parseIPv4Address(addr4);
+    }
+
     // TODO: parse CTA_PROTOINFO of conntrack event in ConntrackMonitor. For TCP, only add rules
     // while TCP status is established.
     @VisibleForTesting
     class BpfConntrackEventConsumer implements ConntrackEventConsumer {
+        // The upstream4 and downstream4 rules are built as the following tables. Only raw ip
+        // upstream interface is supported. Note that the field "lastUsed" is only updated by
+        // BPF program which records the last used time for a given rule.
+        // TODO: support ether ip upstream interface.
+        //
+        // NAT network topology:
+        //
+        //         public network (rawip)                 private network
+        //                   |                 UE                |
+        // +------------+    V    +------------+------------+    V    +------------+
+        // |   Sever    +---------+  Upstream  | Downstream +---------+   Client   |
+        // +------------+         +------------+------------+         +------------+
+        //
+        // upstream4 key and value:
+        //
+        // +------+------------------------------------------------+
+        // |      |      TetherUpstream4Key                        |
+        // +------+------+------+------+------+------+------+------+
+        // |field |iif   |dstMac|l4prot|src4  |dst4  |srcPor|dstPor|
+        // |      |      |      |o     |      |      |t     |t     |
+        // +------+------+------+------+------+------+------+------+
+        // |value |downst|downst|tcp/  |client|server|client|server|
+        // |      |ream  |ream  |udp   |      |      |      |      |
+        // +------+------+------+------+------+------+------+------+
+        //
+        // +------+---------------------------------------------------------------------+
+        // |      |      TetherUpstream4Value                                           |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        // |field |oif   |ethDst|ethSrc|ethPro|pmtu  |src46 |dst46 |srcPor|dstPor|lastUs|
+        // |      |      |mac   |mac   |to    |      |      |      |t     |t     |ed    |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        // |value |upstre|--    |--    |ETH_P_|1500  |upstre|server|upstre|server|--    |
+        // |      |am    |      |      |IP    |      |am    |      |am    |      |      |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        //
+        // downstream4 key and value:
+        //
+        // +------+------------------------------------------------+
+        // |      |      TetherDownstream4Key                      |
+        // +------+------+------+------+------+------+------+------+
+        // |field |iif   |dstMac|l4prot|src4  |dst4  |srcPor|dstPor|
+        // |      |      |      |o     |      |      |t     |t     |
+        // +------+------+------+------+------+------+------+------+
+        // |value |upstre|--    |tcp/  |server|upstre|server|upstre|
+        // |      |am    |      |udp   |      |am    |      |am    |
+        // +------+------+------+------+------+------+------+------+
+        //
+        // +------+---------------------------------------------------------------------+
+        // |      |      TetherDownstream4Value                                         |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        // |field |oif   |ethDst|ethSrc|ethPro|pmtu  |src46 |dst46 |srcPor|dstPor|lastUs|
+        // |      |      |mac   |mac   |to    |      |      |      |t     |t     |ed    |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        // |value |downst|client|downst|ETH_P_|1500  |server|client|server|client|--    |
+        // |      |ream  |      |ream  |IP    |      |      |      |      |      |      |
+        // +------+------+------+------+------+------+------+------+------+------+------+
+        //
         @NonNull
         private Tether4Key makeTetherUpstream4Key(
                 @NonNull ConntrackEvent e, @NonNull ClientInfo c) {
@@ -1751,14 +1852,89 @@ public class BpfCoordinator {
         return Math.max(DEFAULT_TETHER_OFFLOAD_POLL_INTERVAL_MS, configInterval);
     }
 
+    @Nullable
+    private Inet4Address parseIPv4Address(byte[] addrBytes) {
+        try {
+            final InetAddress ia = Inet4Address.getByAddress(addrBytes);
+            if (ia instanceof Inet4Address) return (Inet4Address) ia;
+        } catch (UnknownHostException | IllegalArgumentException e) {
+            mLog.e("Failed to parse IPv4 address: " + e);
+        }
+        return null;
+    }
+
+    // Update CTA_TUPLE_ORIG timeout for a given conntrack entry. Note that there will also be
+    // coming a conntrack event to notify updated timeout.
+    private void updateConntrackTimeout(byte proto, Inet4Address src4, short srcPort,
+            Inet4Address dst4, short dstPort) {
+        if (src4 == null || dst4 == null) return;
+
+        // TODO: consider acquiring the timeout setting from nf_conntrack_* variables.
+        // - proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established
+        // - proc/sys/net/netfilter/nf_conntrack_udp_timeout_stream
+        // See kernel document nf_conntrack-sysctl.txt.
+        final int timeoutSec = (proto == OsConstants.IPPROTO_TCP)
+                ? NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED
+                : NF_CONNTRACK_UDP_TIMEOUT_STREAM;
+        final byte[] msg = ConntrackMessage.newIPv4TimeoutUpdateRequest(
+                proto, src4, (int) srcPort, dst4, (int) dstPort, timeoutSec);
+        try {
+            NetlinkSocket.sendOneShotKernelMessage(OsConstants.NETLINK_NETFILTER, msg);
+        } catch (ErrnoException e) {
+            mLog.e("Error updating conntrack entry ("
+                    + "proto: " + proto + ", "
+                    + "src4: " + src4 + ", "
+                    + "srcPort: " + Short.toUnsignedInt(srcPort) + ", "
+                    + "dst4: " + dst4 + ", "
+                    + "dstPort: " + Short.toUnsignedInt(dstPort) + "), "
+                    + "msg: " + NetlinkConstants.hexify(msg) + ", "
+                    + "e: " + e);
+        }
+    }
+
+    private void maybeRefreshConntrackTimeout() {
+        final long now = mDeps.elapsedRealtimeNanos();
+
+        // Reverse the source and destination {address, port} from downstream value because
+        // #updateConntrackTimeout refresh the timeout of netlink attribute CTA_TUPLE_ORIG
+        // which is opposite direction for downstream map value.
+        mBpfCoordinatorShim.tetherOffloadRuleForEach(DOWNSTREAM, (k, v) -> {
+            if ((now - v.lastUsed) / 1_000_000 < POLLING_CONNTRACK_TIMEOUT_MS) {
+                updateConntrackTimeout((byte) k.l4proto,
+                        ipv4MappedAddressBytesToIpv4Address(v.dst46), (short) v.dstPort,
+                        ipv4MappedAddressBytesToIpv4Address(v.src46), (short) v.srcPort);
+            }
+        });
+
+        // TODO: Consider ignoring TCP traffic on upstream and monitor on downstream only
+        // because TCP is a bidirectional traffic. Probably don't need to extend timeout by
+        // both directions for TCP.
+        mBpfCoordinatorShim.tetherOffloadRuleForEach(UPSTREAM, (k, v) -> {
+            if ((now - v.lastUsed) / 1_000_000 < POLLING_CONNTRACK_TIMEOUT_MS) {
+                updateConntrackTimeout((byte) k.l4proto, parseIPv4Address(k.src4),
+                        (short) k.srcPort, parseIPv4Address(k.dst4), (short) k.dstPort);
+            }
+        });
+    }
+
     private void maybeSchedulePollingStats() {
         if (!mPollingStarted) return;
 
-        if (mHandler.hasCallbacks(mScheduledPollingTask)) {
+        if (mHandler.hasCallbacks(mScheduledPollingStats)) {
-            mHandler.removeCallbacks(mScheduledPollingTask);
+            mHandler.removeCallbacks(mScheduledPollingStats);
         }
 
-        mHandler.postDelayed(mScheduledPollingTask, getPollingInterval());
+        mHandler.postDelayed(mScheduledPollingStats, getPollingInterval());
+    }
+
+    private void maybeSchedulePollingConntrackTimeout() {
+        if (!mPollingStarted) return;
+
+        if (mHandler.hasCallbacks(mScheduledPollingConntrackTimeout)) {
+            mHandler.removeCallbacks(mScheduledPollingConntrackTimeout);
+        }
+
+        mHandler.postDelayed(mScheduledPollingConntrackTimeout, POLLING_CONNTRACK_TIMEOUT_MS);
     }
 
     // Return forwarding rule map. This is used for testing only.

Tethering/tests/unit/src/com/android/networkstack/tethering/BpfCoordinatorTest.java

@@ -36,9 +36,13 @@ import static android.system.OsConstants.ETH_P_IP;
 import static android.system.OsConstants.ETH_P_IPV6;
 import static android.system.OsConstants.IPPROTO_TCP;
 import static android.system.OsConstants.IPPROTO_UDP;
+import static android.system.OsConstants.NETLINK_NETFILTER;
 
 import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn;
 import static com.android.dx.mockito.inline.extended.ExtendedMockito.staticMockMarker;
+import static com.android.networkstack.tethering.BpfCoordinator.NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED;
+import static com.android.networkstack.tethering.BpfCoordinator.NF_CONNTRACK_UDP_TIMEOUT_STREAM;
+import static com.android.networkstack.tethering.BpfCoordinator.POLLING_CONNTRACK_TIMEOUT_MS;
 import static com.android.networkstack.tethering.BpfCoordinator.StatsType;
 import static com.android.networkstack.tethering.BpfCoordinator.StatsType.STATS_PER_IFACE;
 import static com.android.networkstack.tethering.BpfCoordinator.StatsType.STATS_PER_UID;
@@ -78,7 +82,9 @@ import android.net.TetherStatsParcel;
 import android.net.ip.ConntrackMonitor;
 import android.net.ip.ConntrackMonitor.ConntrackEventConsumer;
 import android.net.ip.IpServer;
+import android.net.netlink.ConntrackMessage;
 import android.net.netlink.NetlinkConstants;
+import android.net.netlink.NetlinkSocket;
 import android.net.util.InterfaceParams;
 import android.net.util.SharedLog;
 import android.os.Build;
@@ -221,6 +227,7 @@ public class BpfCoordinatorTest {
     // it has to access the non-static function of BPF coordinator.
     private BpfConntrackEventConsumer mConsumer;
 
+    private long mElapsedRealtimeNanos = 0;
     private final ArgumentCaptor<ArrayList> mStringArrayCaptor =
             ArgumentCaptor.forClass(ArrayList.class);
     private final TestLooper mTestLooper = new TestLooper();
@@ -260,6 +267,10 @@ public class BpfCoordinatorTest {
                         return mConntrackMonitor;
                     }
 
+                    public long elapsedRealtimeNanos() {
+                        return mElapsedRealtimeNanos;
+                    }
+
                     @Nullable
                     public BpfMap<Tether4Key, Tether4Value> getBpfDownstream4Map() {
                         return mBpfDownstream4Map;
@@ -1343,6 +1354,11 @@ public class BpfCoordinatorTest {
                 PRIVATE_ADDR_V4MAPPED_BYTES, REMOTE_PORT, PRIVATE_PORT, 0 /* lastUsed */);
     }
 
+    @NonNull
+    private Tether4Key makeDownstream4Key() {
+        return makeDownstream4Key(IPPROTO_TCP);
+    }
+
     @NonNull
     private ConntrackEvent makeTestConntrackEvent(short msgType, int proto) {
         if (msgType != IPCTNL_MSG_CT_NEW && msgType != IPCTNL_MSG_CT_DELETE) {
@@ -1504,4 +1520,104 @@ public class BpfCoordinatorTest {
         mConsumer.accept(makeTestConntrackEvent(IPCTNL_MSG_CT_NEW, IPPROTO_UDP));
         verify(mBpfDevMap, never()).updateEntry(any(), any());
     }
+
+    private void setElapsedRealtimeNanos(long nanoSec) {
+        mElapsedRealtimeNanos = nanoSec;
+    }
+
+    private void checkRefreshConntrackTimeout(final TestBpfMap<Tether4Key, Tether4Value> bpfMap,
+            final Tether4Key tcpKey, final Tether4Value tcpValue, final Tether4Key udpKey,
+            final Tether4Value udpValue) throws Exception {
+        // Both system elapsed time since boot and the rule last used time are used to measure
+        // the rule expiration. In this test, all test rules are fixed the last used time to 0.
+        // Set the different testing elapsed time to make the rule to be valid or expired.
+        //
+        // Timeline:
+        // 0                                       60 (seconds)
+        // +---+---+---+---+--...--+---+---+---+---+---+- ..
+        // |      POLLING_CONNTRACK_TIMEOUT_MS     |
+        // +---+---+---+---+--...--+---+---+---+---+---+- ..
+        // |<-          valid diff           ->|
+        // |<-          expired diff                 ->|
+        // ^                                   ^       ^
+        // last used time      elapsed time (valid)    elapsed time (expired)
+        final long validTime = (POLLING_CONNTRACK_TIMEOUT_MS - 1) * 1_000_000L;
+        final long expiredTime = (POLLING_CONNTRACK_TIMEOUT_MS + 1) * 1_000_000L;
+
+        // Static mocking for NetlinkSocket.
+        MockitoSession mockSession = ExtendedMockito.mockitoSession()
+                .mockStatic(NetlinkSocket.class)
+                .startMocking();
+        try {
+            final BpfCoordinator coordinator = makeBpfCoordinator();
+            coordinator.startPolling();
+            bpfMap.insertEntry(tcpKey, tcpValue);
+            bpfMap.insertEntry(udpKey, udpValue);
+
+            // [1] Don't refresh contrack timeout.
+            setElapsedRealtimeNanos(expiredTime);
+            mTestLooper.moveTimeForward(POLLING_CONNTRACK_TIMEOUT_MS);
+            waitForIdle();
+            ExtendedMockito.verifyNoMoreInteractions(staticMockMarker(NetlinkSocket.class));
+            ExtendedMockito.clearInvocations(staticMockMarker(NetlinkSocket.class));
+
+            // [2] Refresh contrack timeout.
+            setElapsedRealtimeNanos(validTime);
+            mTestLooper.moveTimeForward(POLLING_CONNTRACK_TIMEOUT_MS);
+            waitForIdle();
+            final byte[] expectedNetlinkTcp = ConntrackMessage.newIPv4TimeoutUpdateRequest(
+                    IPPROTO_TCP, PRIVATE_ADDR, (int) PRIVATE_PORT, REMOTE_ADDR,
+                    (int) REMOTE_PORT, NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED);
+            final byte[] expectedNetlinkUdp = ConntrackMessage.newIPv4TimeoutUpdateRequest(
+                    IPPROTO_UDP, PRIVATE_ADDR, (int) PRIVATE_PORT, REMOTE_ADDR,
+                    (int) REMOTE_PORT, NF_CONNTRACK_UDP_TIMEOUT_STREAM);
+            ExtendedMockito.verify(() -> NetlinkSocket.sendOneShotKernelMessage(
+                    eq(NETLINK_NETFILTER), eq(expectedNetlinkTcp)));
+            ExtendedMockito.verify(() -> NetlinkSocket.sendOneShotKernelMessage(
+                    eq(NETLINK_NETFILTER), eq(expectedNetlinkUdp)));
+            ExtendedMockito.verifyNoMoreInteractions(staticMockMarker(NetlinkSocket.class));
+            ExtendedMockito.clearInvocations(staticMockMarker(NetlinkSocket.class));
+
+            // [3] Don't refresh contrack timeout if polling stopped.
+            coordinator.stopPolling();
+            mTestLooper.moveTimeForward(POLLING_CONNTRACK_TIMEOUT_MS);
+            waitForIdle();
+            ExtendedMockito.verifyNoMoreInteractions(staticMockMarker(NetlinkSocket.class));
+            ExtendedMockito.clearInvocations(staticMockMarker(NetlinkSocket.class));
+        } finally {
+            mockSession.finishMocking();
+        }
+    }
+
+    @Test
+    @IgnoreUpTo(Build.VERSION_CODES.R)
+    public void testRefreshConntrackTimeout_Upstream4Map() throws Exception {
+        // TODO: Replace the dependencies BPF map with a non-mocked TestBpfMap object.
+        final TestBpfMap<Tether4Key, Tether4Value> bpfUpstream4Map =
+                new TestBpfMap<>(Tether4Key.class, Tether4Value.class);
+        doReturn(bpfUpstream4Map).when(mDeps).getBpfUpstream4Map();
+
+        final Tether4Key tcpKey = makeUpstream4Key(IPPROTO_TCP);
+        final Tether4Key udpKey = makeUpstream4Key(IPPROTO_UDP);
+        final Tether4Value tcpValue = makeUpstream4Value();
+        final Tether4Value udpValue = makeUpstream4Value();
+
+        checkRefreshConntrackTimeout(bpfUpstream4Map, tcpKey, tcpValue, udpKey, udpValue);
+    }
+
+    @Test
+    @IgnoreUpTo(Build.VERSION_CODES.R)
+    public void testRefreshConntrackTimeout_Downstream4Map() throws Exception {
+        // TODO: Replace the dependencies BPF map with a non-mocked TestBpfMap object.
+        final TestBpfMap<Tether4Key, Tether4Value> bpfDownstream4Map =
+                new TestBpfMap<>(Tether4Key.class, Tether4Value.class);
+        doReturn(bpfDownstream4Map).when(mDeps).getBpfDownstream4Map();
+
+        final Tether4Key tcpKey = makeDownstream4Key(IPPROTO_TCP);
+        final Tether4Key udpKey = makeDownstream4Key(IPPROTO_UDP);
+        final Tether4Value tcpValue = makeDownstream4Value();
+        final Tether4Value udpValue = makeDownstream4Value();
+
+        checkRefreshConntrackTimeout(bpfDownstream4Map, tcpKey, tcpValue, udpKey, udpValue);
+    }
 }