[9] Fix null targets in IME's 'WM & SF Properties'

Fix incorrect names of properties being extracted from WM Trace: - 'Input Method Control Target' has been replaced with 'IME Control Target', 'Input Method Input Target' with 'IME Input Target', and 'Input Method Target' with 'IME Layering Target'. Bug: 239145867 Test: manual on local build of winscope Change-Id: If067cf73de32d4374b33d1f07f179b63c9fb0c27
2022-07-18 09:16:52 +00:00
parent 8181e2a221
commit cb0d23d9dd
2 changed files with 206 additions and 66 deletions
--- a/tools/winscope/src/ImeAdditionalProperties.vue
+++ b/tools/winscope/src/ImeAdditionalProperties.vue
@@ -20,16 +20,29 @@
  </div>
  <div v-else-if="isImeManagerService">
    <div class="group">
-      <span class="group-header">WMState</span>
+      <button
+          class="text-button group-header"
+          v-if="wmProtoOrNull"
+          :class="{ 'selected': isSelected(wmProtoOrNull) }"
+          @click="onClickShowInPropertiesPanel(wmProtoOrNull)">
+        WMState
+      </button>
+      <span class="group-header" v-else>WMState</span>
      <div class="full-width">
        <span class="value" v-if="entry.wmProperties">{{
            entry.wmProperties.name }}</span>
        <span v-else>There is no corresponding WMState entry.</span>
      </div>
    </div>
-    <div class="group">
-      <span class="group-header">IME Insets Source Provider</span>
+    <div class="group" v-if="wmInsetsSourceProviderOrNull">
+      <button
+          class="text-button group-header"
+          :class="{ 'selected': isSelected(wmInsetsSourceProviderOrNull) }"
+          @click="onClickShowInPropertiesPanel(wmInsetsSourceProviderOrNull)">
+        IME Insets Source Provider
+      </button>
      <div class="full-width">
+        <div />
        <span class="key">Source Frame:</span>
        <CoordinatesTable
            :coordinates="wmInsetsSourceProviderSourceFrameOrNull" />
@@ -56,40 +69,40 @@
      </div>
    </div>
    <div class="group">
-      <span class="group-header">IMControl Target</span>
+      <span class="group-header">Control Target</span>
      <div class="full-width">
        <button
            class="text-button"
-            :class="{ 'selected': isSelected(wmIMControlTargetOrNull) }"
-            v-if="wmIMControlTargetOrNull"
-            @click="onClickShowInPropertiesPanel(wmIMControlTargetOrNull)">
-          Input Method Control Target
+            :class="{ 'selected': isSelected(wmImeControlTargetOrNull) }"
+            v-if="wmImeControlTargetOrNull"
+            @click="onClickShowInPropertiesPanel(wmImeControlTargetOrNull)">
+          IME Control Target
        </button>
        <span class="value" v-else>null</span>
      </div>
    </div>
    <div class="group">
-      <span class="group-header">IMInput Target</span>
+      <span class="group-header">Input Target</span>
      <div class="full-width">
        <button
            class="text-button"
-            :class="{ 'selected': isSelected(wmIMInputTargetOrNull) }"
-            v-if="wmIMInputTargetOrNull"
-            @click="onClickShowInPropertiesPanel(wmIMInputTargetOrNull)">
-          Input Method Input Target
+            :class="{ 'selected': isSelected(wmImeInputTargetOrNull) }"
+            v-if="wmImeInputTargetOrNull"
+            @click="onClickShowInPropertiesPanel(wmImeInputTargetOrNull)">
+          IME Input Target
        </button>
        <span class="value" v-else>null</span>
      </div>
    </div>
    <div class="group">
-      <span class="group-header">IM Target</span>
+      <span class="group-header">Layering Target</span>
      <div class="full-width">
        <button
            class="text-button"
-            :class="{ 'selected': isSelected(wmIMTargetOrNull) }"
-            v-if="wmIMTargetOrNull"
-            @click="onClickShowInPropertiesPanel(wmIMTargetOrNull)">
-          Input Method Target
+            :class="{ 'selected': isSelected(wmImeLayeringTargetOrNull) }"
+            v-if="wmImeLayeringTargetOrNull"
+            @click="onClickShowInPropertiesPanel(wmImeLayeringTargetOrNull)">
+          IME Layering Target
        </button>
        <span class="value" v-else>null</span>
      </div>
@@ -99,7 +112,14 @@
  <div v-else>
    <!-- Ime Client or Ime Service -->
    <div class="group">
-      <span class="group-header">WMState</span>
+      <button
+          class="text-button group-header"
+          v-if="wmProtoOrNull"
+          :class="{ 'selected': isSelected(wmProtoOrNull) }"
+          @click="onClickShowInPropertiesPanel(wmProtoOrNull)">
+        WMState
+      </button>
+      <span class="group-header" v-else>WMState</span>
      <div class="full-width">
        <span class="value" v-if="entry.wmProperties">{{
            entry.wmProperties.name }}</span>
@@ -109,8 +129,8 @@
    <div class="group">
      <span class="group-header">SFLayer</span>
      <div class="full-width">
-        <span class="value" v-if="entry.sfImeContainerProperties">{{
-            entry.sfImeContainerProperties.name }}</span>
+        <span class="value" v-if="entry.sfProperties">{{
+            entry.sfProperties.name }}</span>
        <span v-else>There is no corresponding SFLayer entry.</span>
      </div>
    </div>
@@ -126,29 +146,66 @@
        <span class="key">Focused Window:</span>
        <span class="value">{{ entry.wmProperties.focusedWindow }}</span>
        <div />
-        <span class="key">Frame:</span>
+        <span class="key" v-if="entry.sfProperties">Focused Window Color:</span>
+        <span class="value" v-if="entry.sfProperties">{{
+            entry.sfProperties.focusedWindowRgba
+          }}</span>
+        <div />
+        <span class="key">Input Control Target Frame:</span>
        <CoordinatesTable :coordinates="wmControlTargetFrameOrNull" />
        <div />
      </div>
    </div>
-    <div class="group" v-if="entry.sfImeContainerProperties">
-      <span class="group-header">Ime Container</span>
+    <div class="group">
+      <span class="group-header">Visibility</span>
      <div class="full-width">
-        <span class="key">ScreenBounds:</span>
-        <CoordinatesTable
-            :coordinates="sfImeContainerScreenBoundsOrNull" />
+        <span class="key">InputMethod Window:</span>
+        <span class="value">{{
+            entry.wmProperties.isInputMethodWindowVisible
+          }}</span>
        <div />
-        <span class="key">Rect:</span>
-        <CoordinatesTable
-            :coordinates="sfImeContainerRectOrNull" />
+        <span class="key">InputMethod Surface:</span>
+        <span class="value">{{
+            entry.sfProperties.isInputMethodSurfaceVisible }}</span>
        <div />
+      </div>
+    </div>
+    <div class="group" v-if="entry.sfProperties">
+      <button
+          class="text-button group-header"
+          :class="{ 'selected': isSelected(entry.sfProperties.imeContainer) }"
+          @click="onClickShowInPropertiesPanel(entry.sfProperties.imeContainer)">
+        Ime Container
+      </button>
+      <div class="full-width">
        <span class="key">ZOrderRelativeOfId:</span>
        <span class="value">{{
-            entry.sfImeContainerProperties.zOrderRelativeOfId
+            entry.sfProperties.zOrderRelativeOfId
          }}</span>
        <div />
        <span class="key">Z:</span>
-        <span class="value">{{ entry.sfImeContainerProperties.z }}</span>
+        <span class="value">{{ entry.sfProperties.z }}</span>
+        <div />
+      </div>
+    </div>
+    <div class="group" v-if="entry.sfProperties">
+      <button
+          class="text-button group-header"
+          :class="{
+            'selected': isSelected(entry.sfProperties.inputMethodSurface)
+          }"
+          @click="onClickShowInPropertiesPanel(
+              entry.sfProperties.inputMethodSurface)">
+        Input Method Surface
+      </button>
+      <div class="full-width">
+        <span class="key">ScreenBounds:</span>
+        <CoordinatesTable
+        :coordinates="sfImeContainerScreenBoundsOrNull" />
+        <div />
+        <span class="key">Rect:</span>
+        <CoordinatesTable
+        :coordinates="sfImeContainerRectOrNull" />
        <div />
      </div>
    </div>
@@ -209,6 +266,15 @@ export default {

  },
  computed: {
+    wmProtoOrNull() {
+      return this.entry.wmProperties?.proto;
+    },
+    wmInsetsSourceProviderOrNull() {
+      return this.entry.wmProperties?.imeInsetsSourceProvider ?
+          Object.assign({'name': 'Ime Insets Source Provider'},
+              this.entry.wmProperties.imeInsetsSourceProvider) :
+          null;
+    },
    wmControlTargetFrameOrNull() {
      return this.entry.wmProperties?.imeInsetsSourceProvider
          ?.insetsSourceProvider?.controlTarget?.windowFrames?.frame || 'null';
@@ -237,36 +303,36 @@ export default {
      return this.entry.wmProperties?.imeInsetsSourceProvider
          ?.insetsSourceProvider?.source?.visibleFrame || 'null';
    },
-    wmIMControlTargetOrNull() {
-      return this.entry?.wmProperties?.inputMethodControlTarget ?
-          Object.assign({'name': 'Input Method Control Target'},
-              this.entry.wmProperties.inputMethodControlTarget) :
+    wmImeControlTargetOrNull() {
+      return this.entry?.wmProperties?.imeControlTarget ?
+          Object.assign({'name': 'IME Control Target'},
+              this.entry.wmProperties.imeControlTarget) :
          null;
    },
-    wmIMInputTargetOrNull() {
-      return this.entry?.wmProperties?.inputMethodInputTarget ?
-          Object.assign({'name': 'Input Method Input Target'},
-              this.entry.wmProperties.inputMethodInputTarget) :
+    wmImeInputTargetOrNull() {
+      return this.entry?.wmProperties?.imeInputTarget ?
+          Object.assign({'name': 'IME Input Target'},
+              this.entry.wmProperties.imeInputTarget) :
          null;
    },
-    wmIMTargetOrNull() {
-      return this.entry?.wmProperties?.inputMethodTarget ?
-          Object.assign({'name': 'Input Method Target'},
-              this.entry.wmProperties.inputMethodTarget) :
+    wmImeLayeringTargetOrNull() {
+      return this.entry?.wmProperties?.imeLayeringTarget ?
+          Object.assign({'name': 'IME Layering Target'},
+              this.entry.wmProperties.imeLayeringTarget) :
          null;
    },
    sfImeContainerScreenBoundsOrNull() {
-      return this.entry.sfImeContainerProperties?.screenBounds || 'null';
+      return this.entry.sfProperties?.screenBounds || 'null';
    },
    sfImeContainerRectOrNull() {
-      return this.entry.sfImeContainerProperties?.rect || 'null';
+      return this.entry.sfProperties?.rect || 'null';
    },
    isAllPropertiesNull() {
      if (this.isImeManagerService) {
        return !this.entry.wmProperties;
      } else {
        return !(this.entry.wmProperties ||
-            this.entry.sfImeContainerProperties);
+            this.entry.sfProperties);
      }
    },
  },
@@ -301,6 +367,7 @@ export default {

 .group-header {
  justify-content: center;
+  text-align: left;
  padding: 0px 5px;
  width: 95px;
  display: inline-block;
--- a/tools/winscope/src/ime_processing.js
+++ b/tools/winscope/src/ime_processing.js
@@ -60,6 +60,7 @@ function combineWmSfWithImeDataIfExisting(dataFiles) {
      combineWmSfPropertiesIntoImeData(
          imeTraceFile, filesAsDict[TRACE_TYPES.SURFACE_FLINGER]);
    }
+    processImeAfterCombiningWmAndSfProperties(imeTraceFile);
  }
  console.log('after combining', dataFiles);
 }
@@ -84,17 +85,14 @@ function combineWmSfPropertiesIntoImeData(imeTraceFile, wmOrSfTraceFile) {
      // hasWmSfProperties is added into data because the
      // imeTraceFile object itself is inextensible if it's from file input
    } else if (wmStateOrSfLayer) {
-      const sfImeContainerProperties =
-          extractImeContainerFields(wmStateOrSfLayer);
-      imeTraceFile.data[i].sfImeContainerProperties = Object.assign(
-          {'name': wmStateOrSfLayer.name}, sfImeContainerProperties);
-      // 'name' is the ..d..h..m..s..ms timestamp
+      const sfProperties = extractSfProperties(wmStateOrSfLayer);
+      imeTraceFile.data[i].sfProperties = sfProperties;

      const sfSubtrees = filterSfLayerForIme(
          wmStateOrSfLayer); // for display in Hierarchy sub-panel
      imeTraceFile.data[i].children.push(...sfSubtrees);

-      if (sfImeContainerProperties || sfSubtrees.length > 0) {
+      if (sfProperties || sfSubtrees.length > 0) {
        imeTraceFile.data[0].hasWmSfProperties = true;
      }
    }
@@ -113,6 +111,9 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) {
      wmOrSfIndex++;
      currWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex];
    }
+    // if wmOrSfIndex is 0, i.e. no corresponding entry is found,
+    // we take the first wm / sf entry
+    // intersectWmOrSfIndices.push(Math.max(0, wmOrSfIndex - 1));
    intersectWmOrSfIndices.push(wmOrSfIndex - 1);
  }
  console.log('done matching corresponding timestamps');
@@ -121,21 +122,46 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) {

 function filterWmStateForIme(wmState) {
  // create and return a custom entry that just contains relevant properties
-  const displayContent = wmState.children[0];
+  const displayContent = wmState.root.children[0];
+  const controlTargetActualName = getActualFieldNameFromPossibilities(
+      'controlTarget', displayContent.proto);
+  const inputTargetActualName =
+      getActualFieldNameFromPossibilities('inputTarget', displayContent.proto);
+  const layeringTargetActualName = getActualFieldNameFromPossibilities(
+      'layeringTarget', displayContent.proto);
+  const isInputMethodWindowVisible = findInputMethodVisibility(displayContent);
  return {
    'kind': 'WM State Properties',
-    'name': wmState.name, // this is the ..d..h..m..s..ms timestamp
+    'name': wmState.name,  // this is the ..d..h..m..s..ms timestamp
    'stableId': wmState.stableId,
    'focusedApp': wmState.focusedApp,
    'focusedWindow': wmState.focusedWindow,
    'focusedActivity': wmState.focusedActivity,
-    'inputMethodControlTarget': displayContent.proto.inputMethodControlTarget,
-    'inputMethodInputTarget': displayContent.proto.inputMethodInputTarget,
-    'inputMethodTarget': displayContent.proto.inputMethodTarget,
+    'isInputMethodWindowVisible': isInputMethodWindowVisible,
+    'imeControlTarget': displayContent.proto[controlTargetActualName],
+    'imeInputTarget': displayContent.proto[inputTargetActualName],
+    'imeLayeringTarget': displayContent.proto[layeringTargetActualName],
    'imeInsetsSourceProvider': displayContent.proto.imeInsetsSourceProvider,
+    'proto': wmState,
  };
 }

+function getActualFieldNameFromPossibilities(wantedKey, protoObject) {
+  // for backwards compatibility purposes: find the actual name in the
+  // protoObject out of a list of possible names, as field names may change
+  const possibleNamesMap = {
+    // inputMethod...Target is legacy name, ime...Target is new name
+    'controlTarget': ['inputMethodControlTarget', 'imeControlTarget'],
+    'inputTarget': ['inputMethodInputTarget', 'imeInputTarget'],
+    'layeringTarget': ['inputMethodTarget', 'imeLayeringTarget'],
+  };
+
+  const possibleNames = possibleNamesMap[wantedKey];
+  const actualName =
+      Object.keys(protoObject).find((el) => possibleNames.includes(el));
+  return actualName;
+}
+
 function filterSfLayerForIme(sfLayer) {
  const parentTaskOfImeContainer =
    findParentTaskOfNode(sfLayer, 'ImeContainer');
@@ -175,19 +201,41 @@ function findParentTaskOfNode(curr, nodeName) {
  return null;
 }

-function extractImeContainerFields(curr) {
+function extractSfProperties(curr) {
+  const imeFields = extractImeFields(curr);
+  if (imeFields == null) {
+    return null;
+  }
+  return Object.assign({'name': curr.name, 'proto': curr}, imeFields);
+  // 'name' is the ..d..h..m..s..ms timestamp
+}
+
+function extractImeFields(curr) {
  const isImeContainer = getFilter('ImeContainer');
-  if (isImeContainer(curr)) {
+  const imeContainer = findWindowOrLayerMatch(isImeContainer, curr);
+  if (imeContainer != null) {
+    const isInputMethodSurface = getFilter('InputMethod');
+    const inputMethodSurface =
+        findWindowOrLayerMatch(isInputMethodSurface, imeContainer);
    return {
-      'screenBounds': curr.screenBounds,
-      'rect': curr.rect,
-      'zOrderRelativeOfId': curr.zOrderRelativeOfId,
-      'z': curr.z,
+      'imeContainer': imeContainer,
+      'inputMethodSurface': inputMethodSurface,
+      'screenBounds': inputMethodSurface.screenBounds,
+      'rect': inputMethodSurface.rect,
+      'isInputMethodSurfaceVisible': inputMethodSurface.isVisible,
+      'zOrderRelativeOfId': imeContainer.zOrderRelativeOfId,
+      'z': imeContainer.z,
    };
  }
-  // search for ImeContainer in children
+  return null;
+}
+
+function findWindowOrLayerMatch(filter, curr) {
+  if (filter(curr)) {
+    return curr;
+  }
  for (const child of curr.children) {
-    const result = extractImeContainerFields(child);
+    const result = findWindowOrLayerMatch(filter, child);
    if (result) {
      return result;
    }
@@ -195,4 +243,29 @@ function extractImeContainerFields(curr) {
  return null;
 }

+function findInputMethodVisibility(windowOrLayer) {
+  const isInputMethod = getFilter('InputMethod');
+  const inputMethodWindowOrLayer =
+      findWindowOrLayerMatch(isInputMethod, windowOrLayer);
+  return inputMethodWindowOrLayer ? inputMethodWindowOrLayer.isVisible : false;
+}
+
+function processImeAfterCombiningWmAndSfProperties(imeTraceFile) {
+  for (const imeEntry of imeTraceFile.data) {
+    if (imeEntry.wmProperties?.focusedWindow && imeEntry.sfProperties?.proto) {
+      const focusedWindowRgba = findFocusedWindowRgba(
+          imeEntry.wmProperties.focusedWindow, imeEntry.sfProperties.proto);
+      imeEntry.sfProperties.focusedWindowRgba = focusedWindowRgba;
+    }
+  }
+}
+
+function findFocusedWindowRgba(focusedWindow, sfLayer) {
+  const focusedWindowToken = focusedWindow.token;
+  console.log(focusedWindowToken);
+  const isFocusedWindow = getFilter(focusedWindowToken);
+  const focusedWindowLayer = findWindowOrLayerMatch(isFocusedWindow, sfLayer);
+  return focusedWindowLayer.color;
+}
+
 export {combineWmSfWithImeDataIfExisting};