IP Connectivity metrics: add connect() statistics
This patch adds a ConnectStats class to aggregate connect() statistics gathered in NetdEventListenerService. ConnectStats is uploaded once a day by IpConnectivityMetrics. Test: $ runtest frameworks-net + new unit test Bug: 32198976 Change-Id: Iea63339035415513a5ba0ff4b8f4d79f75fc652d
This commit is contained in:
@@ -369,7 +369,8 @@ public class IpConnectivityEventBuilderTest extends TestCase {
|
|||||||
|
|
||||||
static void verifySerialization(String want, ConnectivityMetricsEvent... input) {
|
static void verifySerialization(String want, ConnectivityMetricsEvent... input) {
|
||||||
try {
|
try {
|
||||||
byte[] got = IpConnectivityEventBuilder.serialize(0, Arrays.asList(input));
|
byte[] got = IpConnectivityEventBuilder.serialize(0,
|
||||||
|
IpConnectivityEventBuilder.toProto(Arrays.asList(input)));
|
||||||
IpConnectivityLog log = IpConnectivityLog.parseFrom(got);
|
IpConnectivityLog log = IpConnectivityLog.parseFrom(got);
|
||||||
assertEquals(want, log.toString());
|
assertEquals(want, log.toString());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
@@ -292,10 +292,5 @@ public class IpConnectivityMetricsTest extends TestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static final Comparator<ConnectivityMetricsEvent> EVENT_COMPARATOR =
|
static final Comparator<ConnectivityMetricsEvent> EVENT_COMPARATOR =
|
||||||
new Comparator<ConnectivityMetricsEvent>() {
|
Comparator.comparingLong((ev) -> ev.timestamp);
|
||||||
@Override
|
|
||||||
public int compare(ConnectivityMetricsEvent ev1, ConnectivityMetricsEvent ev2) {
|
|
||||||
return (int) (ev1.timestamp - ev2.timestamp);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,25 +16,35 @@
|
|||||||
|
|
||||||
package com.android.server.connectivity;
|
package com.android.server.connectivity;
|
||||||
|
|
||||||
import android.net.ConnectivityManager.NetworkCallback;
|
|
||||||
import android.net.ConnectivityManager;
|
import android.net.ConnectivityManager;
|
||||||
|
import android.net.ConnectivityManager.NetworkCallback;
|
||||||
import android.net.Network;
|
import android.net.Network;
|
||||||
|
import android.net.metrics.ConnectStats;
|
||||||
import android.net.metrics.DnsEvent;
|
import android.net.metrics.DnsEvent;
|
||||||
import android.net.metrics.INetdEventListener;
|
import android.net.metrics.INetdEventListener;
|
||||||
import android.net.metrics.IpConnectivityLog;
|
import android.net.metrics.IpConnectivityLog;
|
||||||
import android.os.RemoteException;
|
import android.os.RemoteException;
|
||||||
|
import android.system.OsConstants;
|
||||||
import android.test.suitebuilder.annotation.SmallTest;
|
import android.test.suitebuilder.annotation.SmallTest;
|
||||||
|
import com.android.server.connectivity.metrics.nano.IpConnectivityLogClass.IpConnectivityEvent;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.OptionalInt;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
import junit.framework.TestCase;
|
import junit.framework.TestCase;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import static org.junit.Assert.assertArrayEquals;
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
import org.mockito.MockitoAnnotations;
|
import org.mockito.MockitoAnnotations;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertArrayEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.mockito.Mockito.any;
|
import static org.mockito.Mockito.any;
|
||||||
import static org.mockito.Mockito.anyInt;
|
import static org.mockito.Mockito.anyInt;
|
||||||
import static org.mockito.Mockito.eq;
|
import static org.mockito.Mockito.eq;
|
||||||
@@ -42,13 +52,6 @@ import static org.mockito.Mockito.timeout;
|
|||||||
import static org.mockito.Mockito.times;
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
import java.io.FileOutputStream;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.OptionalInt;
|
|
||||||
import java.util.stream.IntStream;
|
|
||||||
|
|
||||||
public class NetdEventListenerServiceTest extends TestCase {
|
public class NetdEventListenerServiceTest extends TestCase {
|
||||||
|
|
||||||
// TODO: read from NetdEventListenerService after this constant is read from system property
|
// TODO: read from NetdEventListenerService after this constant is read from system property
|
||||||
@@ -68,45 +71,48 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final String EXAMPLE_IPV4 = "192.0.2.1";
|
||||||
|
private static final String EXAMPLE_IPV6 = "2001:db8:1200::2:1";
|
||||||
|
|
||||||
NetdEventListenerService mNetdEventListenerService;
|
NetdEventListenerService mNetdEventListenerService;
|
||||||
|
|
||||||
@Mock ConnectivityManager mCm;
|
@Mock ConnectivityManager mCm;
|
||||||
@Mock IpConnectivityLog mLog;
|
@Mock IpConnectivityLog mLog;
|
||||||
ArgumentCaptor<NetworkCallback> mCallbackCaptor;
|
ArgumentCaptor<NetworkCallback> mCallbackCaptor;
|
||||||
ArgumentCaptor<DnsEvent> mEvCaptor;
|
ArgumentCaptor<DnsEvent> mDnsEvCaptor;
|
||||||
|
|
||||||
public void setUp() {
|
public void setUp() {
|
||||||
MockitoAnnotations.initMocks(this);
|
MockitoAnnotations.initMocks(this);
|
||||||
mCallbackCaptor = ArgumentCaptor.forClass(NetworkCallback.class);
|
mCallbackCaptor = ArgumentCaptor.forClass(NetworkCallback.class);
|
||||||
mEvCaptor = ArgumentCaptor.forClass(DnsEvent.class);
|
mDnsEvCaptor = ArgumentCaptor.forClass(DnsEvent.class);
|
||||||
mNetdEventListenerService = new NetdEventListenerService(mCm, mLog);
|
mNetdEventListenerService = new NetdEventListenerService(mCm, mLog);
|
||||||
|
|
||||||
verify(mCm, times(1)).registerNetworkCallback(any(), mCallbackCaptor.capture());
|
verify(mCm, times(1)).registerNetworkCallback(any(), mCallbackCaptor.capture());
|
||||||
}
|
}
|
||||||
|
|
||||||
@SmallTest
|
@SmallTest
|
||||||
public void testOneBatch() throws Exception {
|
public void testOneDnsBatch() throws Exception {
|
||||||
log(105, LATENCIES);
|
log(105, LATENCIES);
|
||||||
log(106, Arrays.copyOf(LATENCIES, BATCH_SIZE - 1)); // one lookup short of a batch event
|
log(106, Arrays.copyOf(LATENCIES, BATCH_SIZE - 1)); // one lookup short of a batch event
|
||||||
|
|
||||||
verifyLoggedEvents(new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
verifyLoggedDnsEvents(new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
||||||
|
|
||||||
log(106, Arrays.copyOfRange(LATENCIES, BATCH_SIZE - 1, BATCH_SIZE));
|
log(106, Arrays.copyOfRange(LATENCIES, BATCH_SIZE - 1, BATCH_SIZE));
|
||||||
|
|
||||||
mEvCaptor = ArgumentCaptor.forClass(DnsEvent.class); // reset argument captor
|
mDnsEvCaptor = ArgumentCaptor.forClass(DnsEvent.class); // reset argument captor
|
||||||
verifyLoggedEvents(
|
verifyLoggedDnsEvents(
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
||||||
}
|
}
|
||||||
|
|
||||||
@SmallTest
|
@SmallTest
|
||||||
public void testSeveralBatches() throws Exception {
|
public void testSeveralDmsBatches() throws Exception {
|
||||||
log(105, LATENCIES);
|
log(105, LATENCIES);
|
||||||
log(106, LATENCIES);
|
log(106, LATENCIES);
|
||||||
log(105, LATENCIES);
|
log(105, LATENCIES);
|
||||||
log(107, LATENCIES);
|
log(107, LATENCIES);
|
||||||
|
|
||||||
verifyLoggedEvents(
|
verifyLoggedDnsEvents(
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
@@ -114,7 +120,7 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@SmallTest
|
@SmallTest
|
||||||
public void testBatchAndNetworkLost() throws Exception {
|
public void testDnsBatchAndNetworkLost() throws Exception {
|
||||||
byte[] eventTypes = Arrays.copyOf(EVENT_TYPES, 20);
|
byte[] eventTypes = Arrays.copyOf(EVENT_TYPES, 20);
|
||||||
byte[] returnCodes = Arrays.copyOf(RETURN_CODES, 20);
|
byte[] returnCodes = Arrays.copyOf(RETURN_CODES, 20);
|
||||||
int[] latencies = Arrays.copyOf(LATENCIES, 20);
|
int[] latencies = Arrays.copyOf(LATENCIES, 20);
|
||||||
@@ -124,14 +130,14 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
mCallbackCaptor.getValue().onLost(new Network(105));
|
mCallbackCaptor.getValue().onLost(new Network(105));
|
||||||
log(105, LATENCIES);
|
log(105, LATENCIES);
|
||||||
|
|
||||||
verifyLoggedEvents(
|
verifyLoggedDnsEvents(
|
||||||
new DnsEvent(105, eventTypes, returnCodes, latencies),
|
new DnsEvent(105, eventTypes, returnCodes, latencies),
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
||||||
}
|
}
|
||||||
|
|
||||||
@SmallTest
|
@SmallTest
|
||||||
public void testConcurrentBatchesAndDumps() throws Exception {
|
public void testConcurrentDnsBatchesAndDumps() throws Exception {
|
||||||
final long stop = System.currentTimeMillis() + 100;
|
final long stop = System.currentTimeMillis() + 100;
|
||||||
final PrintWriter pw = new PrintWriter(new FileOutputStream("/dev/null"));
|
final PrintWriter pw = new PrintWriter(new FileOutputStream("/dev/null"));
|
||||||
new Thread() {
|
new Thread() {
|
||||||
@@ -142,27 +148,121 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}
|
}
|
||||||
}.start();
|
}.start();
|
||||||
|
|
||||||
logAsync(105, LATENCIES);
|
logDnsAsync(105, LATENCIES);
|
||||||
logAsync(106, LATENCIES);
|
logDnsAsync(106, LATENCIES);
|
||||||
logAsync(107, LATENCIES);
|
logDnsAsync(107, LATENCIES);
|
||||||
|
|
||||||
verifyLoggedEvents(500,
|
verifyLoggedDnsEvents(500,
|
||||||
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(105, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
new DnsEvent(106, EVENT_TYPES, RETURN_CODES, LATENCIES),
|
||||||
new DnsEvent(107, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
new DnsEvent(107, EVENT_TYPES, RETURN_CODES, LATENCIES));
|
||||||
}
|
}
|
||||||
|
|
||||||
@SmallTest
|
@SmallTest
|
||||||
public void testConcurrentBatchesAndNetworkLoss() throws Exception {
|
public void testConcurrentDnsBatchesAndNetworkLoss() throws Exception {
|
||||||
logAsync(105, LATENCIES);
|
logDnsAsync(105, LATENCIES);
|
||||||
Thread.sleep(10L);
|
Thread.sleep(10L);
|
||||||
// call onLost() asynchronously to logAsync's onDnsEvent() calls.
|
// call onLost() asynchronously to logDnsAsync's onDnsEvent() calls.
|
||||||
mCallbackCaptor.getValue().onLost(new Network(105));
|
mCallbackCaptor.getValue().onLost(new Network(105));
|
||||||
|
|
||||||
// do not verify unpredictable batch
|
// do not verify unpredictable batch
|
||||||
verify(mLog, timeout(500).times(1)).log(any());
|
verify(mLog, timeout(500).times(1)).log(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SmallTest
|
||||||
|
public void testConnectLogging() throws Exception {
|
||||||
|
final int OK = 0;
|
||||||
|
Thread[] logActions = {
|
||||||
|
// ignored
|
||||||
|
connectEventAction(OsConstants.EALREADY, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EALREADY, 0, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OsConstants.EINPROGRESS, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EINPROGRESS, 0, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OsConstants.EINPROGRESS, 0, EXAMPLE_IPV6),
|
||||||
|
// valid latencies
|
||||||
|
connectEventAction(OK, 110, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OK, 23, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OK, 45, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OK, 56, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OK, 523, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OK, 214, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OK, 67, EXAMPLE_IPV6),
|
||||||
|
// errors
|
||||||
|
connectEventAction(OsConstants.EPERM, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EPERM, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EAGAIN, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EACCES, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EACCES, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.EACCES, 0, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OsConstants.EADDRINUSE, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.ETIMEDOUT, 0, EXAMPLE_IPV4),
|
||||||
|
connectEventAction(OsConstants.ETIMEDOUT, 0, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OsConstants.ETIMEDOUT, 0, EXAMPLE_IPV6),
|
||||||
|
connectEventAction(OsConstants.ECONNREFUSED, 0, EXAMPLE_IPV4),
|
||||||
|
};
|
||||||
|
|
||||||
|
for (Thread t : logActions) {
|
||||||
|
t.start();
|
||||||
|
}
|
||||||
|
for (Thread t : logActions) {
|
||||||
|
t.join();
|
||||||
|
}
|
||||||
|
|
||||||
|
List<IpConnectivityEvent> events = new ArrayList<>();
|
||||||
|
mNetdEventListenerService.flushStatistics(events);
|
||||||
|
|
||||||
|
IpConnectivityEvent got = events.get(0);
|
||||||
|
String want = String.join("\n",
|
||||||
|
"time_ms: 0",
|
||||||
|
"transport: 0",
|
||||||
|
"connect_statistics <",
|
||||||
|
" connect_count: 12",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 1",
|
||||||
|
" value: 2",
|
||||||
|
" >",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 11",
|
||||||
|
" value: 1",
|
||||||
|
" >",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 13",
|
||||||
|
" value: 3",
|
||||||
|
" >",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 98",
|
||||||
|
" value: 1",
|
||||||
|
" >",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 110",
|
||||||
|
" value: 3",
|
||||||
|
" >",
|
||||||
|
" errnos_counters <",
|
||||||
|
" key: 111",
|
||||||
|
" value: 1",
|
||||||
|
" >",
|
||||||
|
" ipv6_addr_count: 6",
|
||||||
|
" latencies_ms: 23",
|
||||||
|
" latencies_ms: 45",
|
||||||
|
" latencies_ms: 56",
|
||||||
|
" latencies_ms: 67",
|
||||||
|
" latencies_ms: 110",
|
||||||
|
" latencies_ms: 214",
|
||||||
|
" latencies_ms: 523",
|
||||||
|
">\n");
|
||||||
|
verifyConnectEvent(want, got);
|
||||||
|
}
|
||||||
|
|
||||||
|
Thread connectEventAction(int error, int latencyMs, String ipAddr) {
|
||||||
|
return new Thread(() -> {
|
||||||
|
try {
|
||||||
|
mNetdEventListenerService.onConnectEvent(100, error, latencyMs, ipAddr, 80, 1);
|
||||||
|
} catch (Exception e) {
|
||||||
|
fail(e.toString());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
void log(int netId, int[] latencies) {
|
void log(int netId, int[] latencies) {
|
||||||
try {
|
try {
|
||||||
for (int l : latencies) {
|
for (int l : latencies) {
|
||||||
@@ -174,7 +274,7 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void logAsync(int netId, int[] latencies) {
|
void logDnsAsync(int netId, int[] latencies) {
|
||||||
new Thread() {
|
new Thread() {
|
||||||
public void run() {
|
public void run() {
|
||||||
log(netId, latencies);
|
log(netId, latencies);
|
||||||
@@ -182,15 +282,15 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}.start();
|
}.start();
|
||||||
}
|
}
|
||||||
|
|
||||||
void verifyLoggedEvents(DnsEvent... expected) {
|
void verifyLoggedDnsEvents(DnsEvent... expected) {
|
||||||
verifyLoggedEvents(0, expected);
|
verifyLoggedDnsEvents(0, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
void verifyLoggedEvents(int wait, DnsEvent... expectedEvents) {
|
void verifyLoggedDnsEvents(int wait, DnsEvent... expectedEvents) {
|
||||||
verify(mLog, timeout(wait).times(expectedEvents.length)).log(mEvCaptor.capture());
|
verify(mLog, timeout(wait).times(expectedEvents.length)).log(mDnsEvCaptor.capture());
|
||||||
for (DnsEvent got : mEvCaptor.getAllValues()) {
|
for (DnsEvent got : mDnsEvCaptor.getAllValues()) {
|
||||||
OptionalInt index = IntStream.range(0, expectedEvents.length)
|
OptionalInt index = IntStream.range(0, expectedEvents.length)
|
||||||
.filter(i -> eventsEqual(expectedEvents[i], got))
|
.filter(i -> dnsEventsEqual(expectedEvents[i], got))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
// Don't match same expected event more than once.
|
// Don't match same expected event more than once.
|
||||||
index.ifPresent(i -> expectedEvents[i] = null);
|
index.ifPresent(i -> expectedEvents[i] = null);
|
||||||
@@ -199,11 +299,22 @@ public class NetdEventListenerServiceTest extends TestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** equality function for DnsEvent to avoid overriding equals() and hashCode(). */
|
/** equality function for DnsEvent to avoid overriding equals() and hashCode(). */
|
||||||
static boolean eventsEqual(DnsEvent expected, DnsEvent got) {
|
static boolean dnsEventsEqual(DnsEvent expected, DnsEvent got) {
|
||||||
return (expected == got) || ((expected != null) && (got != null)
|
return (expected == got) || ((expected != null) && (got != null)
|
||||||
&& (expected.netId == got.netId)
|
&& (expected.netId == got.netId)
|
||||||
&& Arrays.equals(expected.eventTypes, got.eventTypes)
|
&& Arrays.equals(expected.eventTypes, got.eventTypes)
|
||||||
&& Arrays.equals(expected.returnCodes, got.returnCodes)
|
&& Arrays.equals(expected.returnCodes, got.returnCodes)
|
||||||
&& Arrays.equals(expected.latenciesMs, got.latenciesMs));
|
&& Arrays.equals(expected.latenciesMs, got.latenciesMs));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void verifyConnectEvent(String expected, IpConnectivityEvent got) {
|
||||||
|
try {
|
||||||
|
Arrays.sort(got.getConnectStatistics().latenciesMs);
|
||||||
|
Arrays.sort(got.getConnectStatistics().errnosCounters,
|
||||||
|
Comparator.comparingInt((p) -> p.key));
|
||||||
|
assertEquals(expected, got.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
fail(e.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user