Program the upstream IPv6 map in BpfCoordinator.

- Add methods to start and stop IPv6 forwarding upstream
- Populate the upstream IPv6 map when the first rule for any
  upstream/downstream pair is created.
- Clear the upstream IPv6 map when the last rule for any
  upstream/downstream pair is deleted.

Test: Added coverage to IpServerTest and BpfCoordinatorTest
Change-Id: Ib041081e95f5f449489ab63138de034222ffac8f

Tethering/apishim/30/com/android/networkstack/tethering/apishim/api30/BpfCoordinatorShimImpl.java

@@ -17,6 +17,7 @@
 package com.android.networkstack.tethering.apishim.api30;
 
 import android.net.INetd;
+import android.net.MacAddress;
 import android.net.TetherStatsParcel;
 import android.net.util.SharedLog;
 import android.os.RemoteException;
@@ -77,6 +78,17 @@ public class BpfCoordinatorShimImpl
         return true;
     }
 
+    @Override
+    public boolean startUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex,
+            MacAddress srcMac, MacAddress dstMac, int mtu) {
+        return true;
+    }
+
+    @Override
+    public boolean stopUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex) {
+        return true;
+    }
+
     @Override
     @Nullable
     public SparseArray<TetherStatsValue> tetherOffloadGetStats() {

Tethering/apishim/31/com/android/networkstack/tethering/apishim/api31/BpfCoordinatorShimImpl.java

@@ -18,6 +18,7 @@ package com.android.networkstack.tethering.apishim.api31;
 
 import static android.net.netstats.provider.NetworkStatsProvider.QUOTA_UNLIMITED;
 
+import android.net.MacAddress;
 import android.net.util.SharedLog;
 import android.system.ErrnoException;
 import android.system.Os;
@@ -38,6 +39,7 @@ import com.android.networkstack.tethering.TetherLimitKey;
 import com.android.networkstack.tethering.TetherLimitValue;
 import com.android.networkstack.tethering.TetherStatsKey;
 import com.android.networkstack.tethering.TetherStatsValue;
+import com.android.networkstack.tethering.TetherUpstream6Key;
 
 import java.io.FileDescriptor;
 
@@ -68,6 +70,10 @@ public class BpfCoordinatorShimImpl
     @Nullable
     private final BpfMap<TetherDownstream6Key, Tether6Value> mBpfDownstream6Map;
 
+    // BPF map for upstream IPv6 forwarding.
+    @Nullable
+    private final BpfMap<TetherUpstream6Key, Tether6Value> mBpfUpstream6Map;
+
     // BPF map of tethering statistics of the upstream interface since tethering startup.
     @Nullable
     private final BpfMap<TetherStatsKey, TetherStatsValue> mBpfStatsMap;
@@ -81,6 +87,7 @@ public class BpfCoordinatorShimImpl
         mBpfDownstream4Map = deps.getBpfDownstream4Map();
         mBpfUpstream4Map = deps.getBpfUpstream4Map();
         mBpfDownstream6Map = deps.getBpfDownstream6Map();
+        mBpfUpstream6Map = deps.getBpfUpstream6Map();
         mBpfStatsMap = deps.getBpfStatsMap();
         mBpfLimitMap = deps.getBpfLimitMap();
     }
@@ -88,7 +95,7 @@ public class BpfCoordinatorShimImpl
     @Override
     public boolean isInitialized() {
         return mBpfDownstream4Map != null && mBpfUpstream4Map != null && mBpfDownstream6Map != null
-                && mBpfStatsMap != null && mBpfLimitMap != null;
+                && mBpfUpstream6Map != null && mBpfStatsMap != null && mBpfLimitMap != null;
     }
 
     @Override
@@ -124,6 +131,37 @@ public class BpfCoordinatorShimImpl
         return true;
     }
 
+    @Override
+    public boolean startUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex,
+            MacAddress srcMac, MacAddress dstMac, int mtu) {
+        if (!isInitialized()) return false;
+
+        final TetherUpstream6Key key = new TetherUpstream6Key(downstreamIfindex);
+        final Tether6Value value = new Tether6Value(upstreamIfindex, srcMac,
+                dstMac, OsConstants.ETH_P_IPV6, mtu);
+        try {
+            mBpfUpstream6Map.insertEntry(key, value);
+        } catch (ErrnoException | IllegalStateException e) {
+            mLog.e("Could not insert upstream6 entry: " + e);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public boolean stopUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex) {
+        if (!isInitialized()) return false;
+
+        final TetherUpstream6Key key = new TetherUpstream6Key(downstreamIfindex);
+        try {
+            mBpfUpstream6Map.deleteEntry(key);
+        } catch (ErrnoException e) {
+            mLog.e("Could not delete upstream IPv6 entry: " + e);
+            return false;
+        }
+        return true;
+    }
+
     @Override
     @Nullable
     public SparseArray<TetherStatsValue> tetherOffloadGetStats() {
@@ -292,6 +330,8 @@ public class BpfCoordinatorShimImpl
                 + (mBpfDownstream4Map != null ? "initialized" : "not initialized") + "}, "
                 + "mBpfUpstream4Map{"
                 + (mBpfUpstream4Map != null ? "initialized" : "not initialized") + "}, "
+                + "mBpfUpstream6Map{"
+                + (mBpfUpstream6Map != null ? "initialized" : "not initialized") + "}, "
                 + "mBpfDownstream6Map{"
                 + (mBpfDownstream6Map != null ? "initialized" : "not initialized") + "}, "
                 + "mBpfStatsMap{"

Tethering/apishim/common/com/android/networkstack/tethering/apishim/common/BpfCoordinatorShim.java

@@ -16,6 +16,7 @@
 
 package com.android.networkstack.tethering.apishim.common;
 
+import android.net.MacAddress;
 import android.util.SparseArray;
 
 import androidx.annotation.NonNull;
@@ -72,6 +73,27 @@ public abstract class BpfCoordinatorShim {
      */
     public abstract boolean tetherOffloadRuleRemove(@NonNull Ipv6ForwardingRule rule);
 
+    /**
+     * Starts IPv6 forwarding between the specified interfaces.
+
+     * @param downstreamIfindex the downstream interface index
+     * @param upstreamIfindex the upstream interface index
+     * @param srcMac the source MAC address to use for packets
+     * @oaram dstMac the destination MAC address to use for packets
+     * @return true if operation succeeded or was a no-op, false otherwise
+     */
+    public abstract boolean startUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex,
+            MacAddress srcMac, MacAddress dstMac, int mtu);
+
+    /**
+     * Stops IPv6 forwarding between the specified interfaces.
+
+     * @param downstreamIfindex the downstream interface index
+     * @param upstreamIfindex the upstream interface index
+     * @return true if operation succeeded or was a no-op, false otherwise
+     */
+    public abstract boolean stopUpstreamIpv6Forwarding(int downstreamIfindex, int upstreamIfindex);
+
     /**
      * Return BPF tethering offload statistics.
      *

Tethering/src/com/android/networkstack/tethering/BpfCoordinator.java

@@ -280,6 +280,17 @@ public class BpfCoordinator {
             }
         }
 
+        /** Get upstream6 BPF map. */
+        @Nullable public BpfMap<TetherUpstream6Key, Tether6Value> getBpfUpstream6Map() {
+            try {
+                return new BpfMap<>(TETHER_UPSTREAM6_FS_PATH, BpfMap.BPF_F_RDWR,
+                        TetherUpstream6Key.class, Tether6Value.class);
+            } catch (ErrnoException e) {
+                Log.e(TAG, "Cannot create upstream6 map: " + e);
+                return null;
+            }
+        }
+
         /** Get stats BPF map. */
         @Nullable public BpfMap<TetherStatsKey, TetherStatsValue> getBpfStatsMap() {
             try {
@@ -444,7 +455,7 @@ public class BpfCoordinator {
         }
         LinkedHashMap<Inet6Address, Ipv6ForwardingRule> rules = mIpv6ForwardingRules.get(ipServer);
 
-        // Setup the data limit on the given upstream if the first rule is added.
+        // When the first rule is added to an upstream, setup upstream forwarding and data limit.
         final int upstreamIfindex = rule.upstreamIfindex;
         if (!isAnyRuleOnUpstream(upstreamIfindex)) {
             // If failed to set a data limit, probably should not use this upstream, because
@@ -455,6 +466,19 @@ public class BpfCoordinator {
                 final String iface = mInterfaceNames.get(upstreamIfindex);
                 mLog.e("Setting data limit for " + iface + " failed.");
             }
+
+        }
+
+        if (!isAnyRuleFromDownstreamToUpstream(rule.downstreamIfindex, rule.upstreamIfindex)) {
+            final int downstream = rule.downstreamIfindex;
+            final int upstream = rule.upstreamIfindex;
+            // TODO: support upstream forwarding on non-point-to-point interfaces.
+            // TODO: get the MTU from LinkProperties and update the rules when it changes.
+            if (!mBpfCoordinatorShim.startUpstreamIpv6Forwarding(downstream, upstream,
+                    NULL_MAC_ADDRESS, NULL_MAC_ADDRESS, NetworkStackConstants.ETHER_MTU)) {
+                mLog.e("Failed to enable upstream IPv6 forwarding from "
+                        + mInterfaceNames.get(downstream) + " to " + mInterfaceNames.get(upstream));
+            }
         }
 
         // Must update the adding rule after calling #isAnyRuleOnUpstream because it needs to
@@ -487,6 +511,16 @@ public class BpfCoordinator {
             mIpv6ForwardingRules.remove(ipServer);
         }
 
+        // If no more rules between this upstream and downstream, stop upstream forwarding.
+        if (!isAnyRuleFromDownstreamToUpstream(rule.downstreamIfindex, rule.upstreamIfindex)) {
+            final int downstream = rule.downstreamIfindex;
+            final int upstream = rule.upstreamIfindex;
+            if (!mBpfCoordinatorShim.stopUpstreamIpv6Forwarding(downstream, upstream)) {
+                mLog.e("Failed to disable upstream IPv6 forwarding from "
+                        + mInterfaceNames.get(downstream) + " to " + mInterfaceNames.get(upstream));
+            }
+        }
+
         // Do cleanup functionality if there is no more rule on the given upstream.
         final int upstreamIfindex = rule.upstreamIfindex;
         if (!isAnyRuleOnUpstream(upstreamIfindex)) {
@@ -535,12 +569,22 @@ public class BpfCoordinator {
         if (rules == null) return;
 
         // Need to build a rule list because the rule map may be changed in the iteration.
-        for (final Ipv6ForwardingRule rule : new ArrayList<Ipv6ForwardingRule>(rules.values())) {
+        // First remove all the old rules, then add all the new rules. This is because the upstream
+        // forwarding code in tetherOffloadRuleAdd cannot support rules on two upstreams at the
+        // same time. Deleting the rules first ensures that upstream forwarding is disabled on the
+        // old upstream when the last rule is removed from it, and re-enabled on the new upstream
+        // when the first rule is added to it.
+        // TODO: Once the IPv6 client processing code has moved from IpServer to BpfCoordinator, do
+        // something smarter.
+        final ArrayList<Ipv6ForwardingRule> rulesCopy = new ArrayList<>(rules.values());
+        for (final Ipv6ForwardingRule rule : rulesCopy) {
             // Remove the old rule before adding the new one because the map uses the same key for
             // both rules. Reversing the processing order causes that the new rule is removed as
             // unexpected.
             // TODO: Add new rule first to reduce the latency which has no rule.
             tetherOffloadRuleRemove(ipServer, rule);
+        }
+        for (final Ipv6ForwardingRule rule : rulesCopy) {
             tetherOffloadRuleAdd(ipServer, rule.onNewUpstream(newUpstreamIfindex));
         }
     }
@@ -1059,6 +1103,19 @@ public class BpfCoordinator {
         return false;
     }
 
+    private boolean isAnyRuleFromDownstreamToUpstream(int downstreamIfindex, int upstreamIfindex) {
+        for (LinkedHashMap<Inet6Address, Ipv6ForwardingRule> rules : mIpv6ForwardingRules
+                .values()) {
+            for (Ipv6ForwardingRule rule : rules.values()) {
+                if (downstreamIfindex == rule.downstreamIfindex
+                        && upstreamIfindex == rule.upstreamIfindex) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @NonNull
     private NetworkStats buildNetworkStats(@NonNull StatsType type, int ifIndex,
             @NonNull final ForwardedStats diff) {

Tethering/src/com/android/networkstack/tethering/TetherUpstream6Key.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.networkstack.tethering;
+
+import com.android.net.module.util.Struct;
+
+/** Key type for upstream IPv6 forwarding map. */
+public class TetherUpstream6Key extends Struct {
+    @Field(order = 0, type = Type.S32)
+    public final int iif; // The input interface index.
+
+    public TetherUpstream6Key(int iif) {
+        this.iif = iif;
+    }
+}

Tethering/tests/unit/src/android/net/ip/IpServerTest.java

@@ -112,6 +112,7 @@ import com.android.networkstack.tethering.TetherLimitKey;
 import com.android.networkstack.tethering.TetherLimitValue;
 import com.android.networkstack.tethering.TetherStatsKey;
 import com.android.networkstack.tethering.TetherStatsValue;
+import com.android.networkstack.tethering.TetherUpstream6Key;
 import com.android.networkstack.tethering.TetheringConfiguration;
 import com.android.testutils.DevSdkIgnoreRule;
 import com.android.testutils.DevSdkIgnoreRule.IgnoreAfter;
@@ -178,6 +179,7 @@ public class IpServerTest {
     @Mock private BpfMap<Tether4Key, Tether4Value> mBpfDownstream4Map;
     @Mock private BpfMap<Tether4Key, Tether4Value> mBpfUpstream4Map;
     @Mock private BpfMap<TetherDownstream6Key, Tether6Value> mBpfDownstream6Map;
+    @Mock private BpfMap<TetherUpstream6Key, Tether6Value> mBpfUpstream6Map;
     @Mock private BpfMap<TetherStatsKey, TetherStatsValue> mBpfStatsMap;
     @Mock private BpfMap<TetherLimitKey, TetherLimitValue> mBpfLimitMap;
 
@@ -324,6 +326,12 @@ public class IpServerTest {
                         return mBpfDownstream6Map;
                     }
 
+                    @Nullable
+                    public BpfMap<TetherUpstream6Key, Tether6Value>
+                            getBpfUpstream6Map() {
+                        return mBpfUpstream6Map;
+                    }
+
                     @Nullable
                     public BpfMap<TetherStatsKey, TetherStatsValue> getBpfStatsMap() {
                         return mBpfStatsMap;
@@ -865,6 +873,36 @@ public class IpServerTest {
         }
     }
 
+    private void verifyStartUpstreamIpv6Forwarding(@Nullable InOrder inOrder, int upstreamIfindex)
+            throws Exception {
+        if (!mBpfDeps.isAtLeastS()) return;
+        final TetherUpstream6Key key = new TetherUpstream6Key(TEST_IFACE_PARAMS.index);
+        final Tether6Value value = new Tether6Value(upstreamIfindex,
+                MacAddress.ALL_ZEROS_ADDRESS, MacAddress.ALL_ZEROS_ADDRESS,
+                ETH_P_IPV6, NetworkStackConstants.ETHER_MTU);
+        verifyWithOrder(inOrder, mBpfUpstream6Map).insertEntry(key, value);
+    }
+
+    private void verifyStopUpstreamIpv6Forwarding(@Nullable InOrder inOrder)
+            throws Exception {
+        if (!mBpfDeps.isAtLeastS()) return;
+        final TetherUpstream6Key key = new TetherUpstream6Key(TEST_IFACE_PARAMS.index);
+        verifyWithOrder(inOrder, mBpfUpstream6Map).deleteEntry(key);
+    }
+
+    private void verifyNoUpstreamIpv6ForwardingChange(@Nullable InOrder inOrder) throws Exception {
+        if (!mBpfDeps.isAtLeastS()) return;
+        if (inOrder != null) {
+            inOrder.verify(mBpfUpstream6Map, never()).deleteEntry(any());
+            inOrder.verify(mBpfUpstream6Map, never()).insertEntry(any(), any());
+            inOrder.verify(mBpfUpstream6Map, never()).updateEntry(any(), any());
+        } else {
+            verify(mBpfUpstream6Map, never()).deleteEntry(any());
+            verify(mBpfUpstream6Map, never()).insertEntry(any(), any());
+            verify(mBpfUpstream6Map, never()).updateEntry(any(), any());
+        }
+    }
+
     @NonNull
     private static TetherStatsParcel buildEmptyTetherStatsParcel(int ifIndex) {
         TetherStatsParcel parcel = new TetherStatsParcel();
@@ -873,7 +911,9 @@ public class IpServerTest {
     }
 
     private void resetNetdBpfMapAndCoordinator() throws Exception {
-        reset(mNetd, mBpfDownstream6Map, mBpfCoordinator);
+        reset(mNetd, mBpfDownstream6Map, mBpfUpstream6Map, mBpfCoordinator);
+        // When the last rule is removed, tetherOffloadGetAndClearStats will log a WTF (and
+        // potentially crash the test) if the stats map is empty.
         when(mNetd.tetherOffloadGetStats()).thenReturn(new TetherStatsParcel[0]);
         when(mNetd.tetherOffloadGetAndClearStats(UPSTREAM_IFINDEX))
                 .thenReturn(buildEmptyTetherStatsParcel(UPSTREAM_IFINDEX));
@@ -894,7 +934,6 @@ public class IpServerTest {
         final int myIfindex = TEST_IFACE_PARAMS.index;
         final int notMyIfindex = myIfindex - 1;
 
-        final MacAddress myMac = TEST_IFACE_PARAMS.macAddr;
         final InetAddress neighA = InetAddresses.parseNumericAddress("2001:db8::1");
         final InetAddress neighB = InetAddresses.parseNumericAddress("2001:db8::2");
         final InetAddress neighLL = InetAddresses.parseNumericAddress("fe80::1");
@@ -904,33 +943,35 @@ public class IpServerTest {
         final MacAddress macB = MacAddress.fromString("11:22:33:00:00:0b");
 
         resetNetdBpfMapAndCoordinator();
-        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map);
+        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
 
         // TODO: Perhaps verify the interaction of tetherOffloadSetInterfaceQuota and
         // tetherOffloadGetAndClearStats in netd while the rules are changed.
 
         // Events on other interfaces are ignored.
         recvNewNeigh(notMyIfindex, neighA, NUD_REACHABLE, macA);
-        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map);
+        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
 
         // Events on this interface are received and sent to netd.
         recvNewNeigh(myIfindex, neighA, NUD_REACHABLE, macA);
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighA, macA));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neighA, macA);
+        verifyStartUpstreamIpv6Forwarding(null, UPSTREAM_IFINDEX);
         resetNetdBpfMapAndCoordinator();
 
         recvNewNeigh(myIfindex, neighB, NUD_REACHABLE, macB);
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighB, macB));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neighB, macB);
+        verifyNoUpstreamIpv6ForwardingChange(null);
         resetNetdBpfMapAndCoordinator();
 
         // Link-local and multicast neighbors are ignored.
         recvNewNeigh(myIfindex, neighLL, NUD_REACHABLE, macA);
-        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map);
+        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
         recvNewNeigh(myIfindex, neighMC, NUD_REACHABLE, macA);
-        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map);
+        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
 
         // A neighbor that is no longer valid causes the rule to be removed.
         // NUD_FAILED events do not have a MAC address.
@@ -938,6 +979,7 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleRemove(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighA, macNull));
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neighA, macNull);
+        verifyNoUpstreamIpv6ForwardingChange(null);
         resetNetdBpfMapAndCoordinator();
 
         // A neighbor that is deleted causes the rule to be removed.
@@ -945,22 +987,27 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleRemove(
                 mIpServer,  makeForwardingRule(UPSTREAM_IFINDEX, neighB, macNull));
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neighB, macNull);
+        verifyStopUpstreamIpv6Forwarding(null);
         resetNetdBpfMapAndCoordinator();
 
         // Upstream changes result in updating the rules.
         recvNewNeigh(myIfindex, neighA, NUD_REACHABLE, macA);
+        verifyStartUpstreamIpv6Forwarding(null, UPSTREAM_IFINDEX);
         recvNewNeigh(myIfindex, neighB, NUD_REACHABLE, macB);
         resetNetdBpfMapAndCoordinator();
 
-        InOrder inOrder = inOrder(mNetd, mBpfDownstream6Map);
+        InOrder inOrder = inOrder(mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
         LinkProperties lp = new LinkProperties();
         lp.setInterfaceName(UPSTREAM_IFACE2);
         dispatchTetherConnectionChanged(UPSTREAM_IFACE2, lp, -1);
         verify(mBpfCoordinator).tetherOffloadRuleUpdate(mIpServer, UPSTREAM_IFINDEX2);
         verifyTetherOffloadRuleRemove(inOrder, UPSTREAM_IFINDEX, neighA, macA);
-        verifyTetherOffloadRuleAdd(inOrder, UPSTREAM_IFINDEX2, neighA, macA);
         verifyTetherOffloadRuleRemove(inOrder, UPSTREAM_IFINDEX, neighB, macB);
+        verifyStopUpstreamIpv6Forwarding(inOrder);
+        verifyTetherOffloadRuleAdd(inOrder, UPSTREAM_IFINDEX2, neighA, macA);
+        verifyStartUpstreamIpv6Forwarding(inOrder, UPSTREAM_IFINDEX2);
         verifyTetherOffloadRuleAdd(inOrder, UPSTREAM_IFINDEX2, neighB, macB);
+        verifyNoUpstreamIpv6ForwardingChange(inOrder);
         resetNetdBpfMapAndCoordinator();
 
         // When the upstream is lost, rules are removed.
@@ -972,6 +1019,7 @@ public class IpServerTest {
         verify(mBpfCoordinator, times(2)).tetherOffloadRuleClear(mIpServer);
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX2, neighA, macA);
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX2, neighB, macB);
+        verifyStopUpstreamIpv6Forwarding(inOrder);
         resetNetdBpfMapAndCoordinator();
 
         // If the upstream is IPv4-only, no rules are added.
@@ -980,7 +1028,8 @@ public class IpServerTest {
         recvNewNeigh(myIfindex, neighA, NUD_REACHABLE, macA);
         // Clear function is called by #updateIpv6ForwardingRules for the IPv6 upstream is lost.
         verify(mBpfCoordinator).tetherOffloadRuleClear(mIpServer);
-        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map);
+        verifyNoUpstreamIpv6ForwardingChange(null);
+        verifyNoMoreInteractions(mBpfCoordinator, mNetd, mBpfDownstream6Map, mBpfUpstream6Map);
 
         // Rules can be added again once upstream IPv6 connectivity is available.
         lp.setInterfaceName(UPSTREAM_IFACE);
@@ -989,6 +1038,7 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighB, macB));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neighB, macB);
+        verifyStartUpstreamIpv6Forwarding(null, UPSTREAM_IFINDEX);
         verify(mBpfCoordinator, never()).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighA, macA));
         verifyNeverTetherOffloadRuleAdd(UPSTREAM_IFINDEX, neighA, macA);
@@ -998,6 +1048,7 @@ public class IpServerTest {
         dispatchTetherConnectionChanged(UPSTREAM_IFACE, null, 0);
         verify(mBpfCoordinator).tetherOffloadRuleClear(mIpServer);
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neighB, macB);
+        verifyStopUpstreamIpv6Forwarding(null);
 
         // When the interface goes down, rules are removed.
         lp.setInterfaceName(UPSTREAM_IFACE);
@@ -1007,6 +1058,7 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighA, macA));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neighA, macA);
+        verifyStartUpstreamIpv6Forwarding(null, UPSTREAM_IFINDEX);
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neighB, macB));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neighB, macB);
@@ -1017,6 +1069,7 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleClear(mIpServer);
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neighA, macA);
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neighB, macB);
+        verifyStopUpstreamIpv6Forwarding(null);
         verify(mIpNeighborMonitor).stop();
         resetNetdBpfMapAndCoordinator();
     }
@@ -1045,12 +1098,14 @@ public class IpServerTest {
         verify(mBpfCoordinator).tetherOffloadRuleAdd(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neigh, macA));
         verifyTetherOffloadRuleAdd(null, UPSTREAM_IFINDEX, neigh, macA);
+        verifyStartUpstreamIpv6Forwarding(null, UPSTREAM_IFINDEX);
         resetNetdBpfMapAndCoordinator();
 
         recvDelNeigh(myIfindex, neigh, NUD_STALE, macA);
         verify(mBpfCoordinator).tetherOffloadRuleRemove(
                 mIpServer, makeForwardingRule(UPSTREAM_IFINDEX, neigh, macNull));
         verifyTetherOffloadRuleRemove(null, UPSTREAM_IFINDEX, neigh, macNull);
+        verifyStopUpstreamIpv6Forwarding(null);
         resetNetdBpfMapAndCoordinator();
 
         // [2] Disable BPF offload.
@@ -1062,11 +1117,13 @@ public class IpServerTest {
         recvNewNeigh(myIfindex, neigh, NUD_REACHABLE, macA);
         verify(mBpfCoordinator, never()).tetherOffloadRuleAdd(any(), any());
         verifyNeverTetherOffloadRuleAdd();
+        verifyNoUpstreamIpv6ForwardingChange(null);
         resetNetdBpfMapAndCoordinator();
 
         recvDelNeigh(myIfindex, neigh, NUD_STALE, macA);
         verify(mBpfCoordinator, never()).tetherOffloadRuleRemove(any(), any());
         verifyNeverTetherOffloadRuleRemove();
+        verifyNoUpstreamIpv6ForwardingChange(null);
         resetNetdBpfMapAndCoordinator();
     }
 

Tethering/tests/unit/src/com/android/networkstack/tethering/BpfCoordinatorTest.java

@@ -161,6 +161,7 @@ public class BpfCoordinatorTest {
     @Mock private BpfMap<Tether4Key, Tether4Value> mBpfDownstream4Map;
     @Mock private BpfMap<Tether4Key, Tether4Value> mBpfUpstream4Map;
     @Mock private BpfMap<TetherDownstream6Key, Tether6Value> mBpfDownstream6Map;
+    @Mock private BpfMap<TetherUpstream6Key, Tether6Value> mBpfUpstream6Map;
 
     // Late init since methods must be called by the thread that created this object.
     private TestableNetworkStatsProviderCbBinder mTetherStatsProviderCb;
@@ -222,6 +223,12 @@ public class BpfCoordinatorTest {
                         return mBpfDownstream6Map;
                     }
 
+                    @Nullable
+                    public BpfMap<TetherUpstream6Key, Tether6Value>
+                            getBpfUpstream6Map() {
+                        return mBpfUpstream6Map;
+                    }
+
                     @Nullable
                     public BpfMap<TetherStatsKey, TetherStatsValue> getBpfStatsMap() {
                         return mBpfStatsMap;
@@ -362,6 +369,36 @@ public class BpfCoordinatorTest {
         }
     }
 
+    private void verifyStartUpstreamIpv6Forwarding(@Nullable InOrder inOrder, int downstreamIfIndex,
+            int upstreamIfindex) throws Exception {
+        if (!mDeps.isAtLeastS()) return;
+        final TetherUpstream6Key key = new TetherUpstream6Key(downstreamIfIndex);
+        final Tether6Value value = new Tether6Value(upstreamIfindex,
+                MacAddress.ALL_ZEROS_ADDRESS, MacAddress.ALL_ZEROS_ADDRESS,
+                ETH_P_IPV6, NetworkStackConstants.ETHER_MTU);
+        verifyWithOrder(inOrder, mBpfUpstream6Map).insertEntry(key, value);
+    }
+
+    private void verifyStopUpstreamIpv6Forwarding(@Nullable InOrder inOrder, int downstreamIfIndex)
+            throws Exception {
+        if (!mDeps.isAtLeastS()) return;
+        final TetherUpstream6Key key = new TetherUpstream6Key(downstreamIfIndex);
+        verifyWithOrder(inOrder, mBpfUpstream6Map).deleteEntry(key);
+    }
+
+    private void verifyNoUpstreamIpv6ForwardingChange(@Nullable InOrder inOrder) throws Exception {
+        if (!mDeps.isAtLeastS()) return;
+        if (inOrder != null) {
+            inOrder.verify(mBpfUpstream6Map, never()).deleteEntry(any());
+            inOrder.verify(mBpfUpstream6Map, never()).insertEntry(any(), any());
+            inOrder.verify(mBpfUpstream6Map, never()).updateEntry(any(), any());
+        } else {
+            verify(mBpfUpstream6Map, never()).deleteEntry(any());
+            verify(mBpfUpstream6Map, never()).insertEntry(any(), any());
+            verify(mBpfUpstream6Map, never()).updateEntry(any(), any());
+        }
+    }
+
     private void verifyTetherOffloadRuleAdd(@Nullable InOrder inOrder,
             @NonNull Ipv6ForwardingRule rule) throws Exception {
         if (mDeps.isAtLeastS()) {
@@ -804,7 +841,8 @@ public class BpfCoordinatorTest {
         coordinator.addUpstreamNameToLookupTable(ethIfIndex, ethIface);
         coordinator.addUpstreamNameToLookupTable(mobileIfIndex, mobileIface);
 
-        final InOrder inOrder = inOrder(mNetd, mBpfDownstream6Map, mBpfLimitMap, mBpfStatsMap);
+        final InOrder inOrder = inOrder(mNetd, mBpfDownstream6Map, mBpfUpstream6Map, mBpfLimitMap,
+                mBpfStatsMap);
 
         // Before the rule test, here are the additional actions while the rules are changed.
         // - After adding the first rule on a given upstream, the coordinator adds a data limit.
@@ -824,7 +862,7 @@ public class BpfCoordinatorTest {
         verifyTetherOffloadRuleAdd(inOrder, ethernetRuleA);
         verifyTetherOffloadSetInterfaceQuota(inOrder, ethIfIndex, QUOTA_UNLIMITED,
                 true /* isInit */);
-
+        verifyStartUpstreamIpv6Forwarding(inOrder, DOWNSTREAM_IFINDEX, ethIfIndex);
         coordinator.tetherOffloadRuleAdd(mIpServer, ethernetRuleB);
         verifyTetherOffloadRuleAdd(inOrder, ethernetRuleB);
 
@@ -840,11 +878,13 @@ public class BpfCoordinatorTest {
         // by one for updating upstream interface index by #tetherOffloadRuleUpdate.
         coordinator.tetherOffloadRuleUpdate(mIpServer, mobileIfIndex);
         verifyTetherOffloadRuleRemove(inOrder, ethernetRuleA);
+        verifyTetherOffloadRuleRemove(inOrder, ethernetRuleB);
+        verifyStopUpstreamIpv6Forwarding(inOrder, DOWNSTREAM_IFINDEX);
+        verifyTetherOffloadGetAndClearStats(inOrder, ethIfIndex);
         verifyTetherOffloadRuleAdd(inOrder, mobileRuleA);
         verifyTetherOffloadSetInterfaceQuota(inOrder, mobileIfIndex, QUOTA_UNLIMITED,
                 true /* isInit */);
-        verifyTetherOffloadRuleRemove(inOrder, ethernetRuleB);
+        verifyStartUpstreamIpv6Forwarding(inOrder, DOWNSTREAM_IFINDEX, mobileIfIndex);
-        verifyTetherOffloadGetAndClearStats(inOrder, ethIfIndex);
         verifyTetherOffloadRuleAdd(inOrder, mobileRuleB);
 
         // [3] Clear all rules for a given IpServer.
@@ -853,6 +893,7 @@ public class BpfCoordinatorTest {
         coordinator.tetherOffloadRuleClear(mIpServer);
         verifyTetherOffloadRuleRemove(inOrder, mobileRuleA);
         verifyTetherOffloadRuleRemove(inOrder, mobileRuleB);
+        verifyStopUpstreamIpv6Forwarding(inOrder, DOWNSTREAM_IFINDEX);
         verifyTetherOffloadGetAndClearStats(inOrder, mobileIfIndex);
 
         // [4] Force pushing stats update to verify that the last diff of stats is reported on all
@@ -940,6 +981,15 @@ public class BpfCoordinatorTest {
         checkBpfDisabled();
     }
 
+    @Test
+    @IgnoreUpTo(Build.VERSION_CODES.R)
+    public void testBpfDisabledbyNoBpfUpstream6Map() throws Exception {
+        setupFunctioningNetdInterface();
+        doReturn(null).when(mDeps).getBpfUpstream6Map();
+
+        checkBpfDisabled();
+    }
+
     @Test
     @IgnoreUpTo(Build.VERSION_CODES.R)
     public void testBpfDisabledbyNoBpfStatsMap() throws Exception {