From cb0d23d9dd34968e894fed9e6e57a914d00ae64f Mon Sep 17 00:00:00 2001 From: Hui Ling Shi Date: Mon, 18 Jul 2022 09:16:52 +0000 Subject: [PATCH] [9] Fix null targets in IME's 'WM & SF Properties' Fix incorrect names of properties being extracted from WM Trace: - 'Input Method Control Target' has been replaced with 'IME Control Target', 'Input Method Input Target' with 'IME Input Target', and 'Input Method Target' with 'IME Layering Target'. Bug: 239145867 Test: manual on local build of winscope Change-Id: If067cf73de32d4374b33d1f07f179b63c9fb0c27 --- .../winscope/src/ImeAdditionalProperties.vue | 161 +++++++++++++----- tools/winscope/src/ime_processing.js | 111 +++++++++--- 2 files changed, 206 insertions(+), 66 deletions(-) diff --git a/tools/winscope/src/ImeAdditionalProperties.vue b/tools/winscope/src/ImeAdditionalProperties.vue index 3bd855c6d..53d6c2731 100644 --- a/tools/winscope/src/ImeAdditionalProperties.vue +++ b/tools/winscope/src/ImeAdditionalProperties.vue @@ -20,16 +20,29 @@
- WMState + + WMState
{{ entry.wmProperties.name }} There is no corresponding WMState entry.
-
- IME Insets Source Provider +
+
+
Source Frame: @@ -56,40 +69,40 @@
- IMControl Target + Control Target
null
- IMInput Target + Input Target
null
- IM Target + Layering Target
null
@@ -99,7 +112,14 @@
- WMState + + WMState
{{ entry.wmProperties.name }} @@ -109,8 +129,8 @@
SFLayer
- {{ - entry.sfImeContainerProperties.name }} + {{ + entry.sfProperties.name }} There is no corresponding SFLayer entry.
@@ -126,29 +146,66 @@ Focused Window: {{ entry.wmProperties.focusedWindow }}
- Frame: + Focused Window Color: + {{ + entry.sfProperties.focusedWindowRgba + }} +
+ Input Control Target Frame:
-
- Ime Container +
+ Visibility
- ScreenBounds: - + InputMethod Window: + {{ + entry.wmProperties.isInputMethodWindowVisible + }}
- Rect: - + InputMethod Surface: + {{ + entry.sfProperties.isInputMethodSurfaceVisible }}
+
+
+
+ +
ZOrderRelativeOfId: {{ - entry.sfImeContainerProperties.zOrderRelativeOfId + entry.sfProperties.zOrderRelativeOfId }}
Z: - {{ entry.sfImeContainerProperties.z }} + {{ entry.sfProperties.z }} +
+
+
+
+ +
+ ScreenBounds: + +
+ Rect: +
@@ -209,6 +266,15 @@ export default { }, computed: { + wmProtoOrNull() { + return this.entry.wmProperties?.proto; + }, + wmInsetsSourceProviderOrNull() { + return this.entry.wmProperties?.imeInsetsSourceProvider ? + Object.assign({'name': 'Ime Insets Source Provider'}, + this.entry.wmProperties.imeInsetsSourceProvider) : + null; + }, wmControlTargetFrameOrNull() { return this.entry.wmProperties?.imeInsetsSourceProvider ?.insetsSourceProvider?.controlTarget?.windowFrames?.frame || 'null'; @@ -237,36 +303,36 @@ export default { return this.entry.wmProperties?.imeInsetsSourceProvider ?.insetsSourceProvider?.source?.visibleFrame || 'null'; }, - wmIMControlTargetOrNull() { - return this.entry?.wmProperties?.inputMethodControlTarget ? - Object.assign({'name': 'Input Method Control Target'}, - this.entry.wmProperties.inputMethodControlTarget) : + wmImeControlTargetOrNull() { + return this.entry?.wmProperties?.imeControlTarget ? + Object.assign({'name': 'IME Control Target'}, + this.entry.wmProperties.imeControlTarget) : null; }, - wmIMInputTargetOrNull() { - return this.entry?.wmProperties?.inputMethodInputTarget ? - Object.assign({'name': 'Input Method Input Target'}, - this.entry.wmProperties.inputMethodInputTarget) : + wmImeInputTargetOrNull() { + return this.entry?.wmProperties?.imeInputTarget ? + Object.assign({'name': 'IME Input Target'}, + this.entry.wmProperties.imeInputTarget) : null; }, - wmIMTargetOrNull() { - return this.entry?.wmProperties?.inputMethodTarget ? - Object.assign({'name': 'Input Method Target'}, - this.entry.wmProperties.inputMethodTarget) : + wmImeLayeringTargetOrNull() { + return this.entry?.wmProperties?.imeLayeringTarget ? + Object.assign({'name': 'IME Layering Target'}, + this.entry.wmProperties.imeLayeringTarget) : null; }, sfImeContainerScreenBoundsOrNull() { - return this.entry.sfImeContainerProperties?.screenBounds || 'null'; + return this.entry.sfProperties?.screenBounds || 'null'; }, sfImeContainerRectOrNull() { - return this.entry.sfImeContainerProperties?.rect || 'null'; + return this.entry.sfProperties?.rect || 'null'; }, isAllPropertiesNull() { if (this.isImeManagerService) { return !this.entry.wmProperties; } else { return !(this.entry.wmProperties || - this.entry.sfImeContainerProperties); + this.entry.sfProperties); } }, }, @@ -301,6 +367,7 @@ export default { .group-header { justify-content: center; + text-align: left; padding: 0px 5px; width: 95px; display: inline-block; diff --git a/tools/winscope/src/ime_processing.js b/tools/winscope/src/ime_processing.js index 4f196364b..8ae164fa1 100644 --- a/tools/winscope/src/ime_processing.js +++ b/tools/winscope/src/ime_processing.js @@ -60,6 +60,7 @@ function combineWmSfWithImeDataIfExisting(dataFiles) { combineWmSfPropertiesIntoImeData( imeTraceFile, filesAsDict[TRACE_TYPES.SURFACE_FLINGER]); } + processImeAfterCombiningWmAndSfProperties(imeTraceFile); } console.log('after combining', dataFiles); } @@ -84,17 +85,14 @@ function combineWmSfPropertiesIntoImeData(imeTraceFile, wmOrSfTraceFile) { // hasWmSfProperties is added into data because the // imeTraceFile object itself is inextensible if it's from file input } else if (wmStateOrSfLayer) { - const sfImeContainerProperties = - extractImeContainerFields(wmStateOrSfLayer); - imeTraceFile.data[i].sfImeContainerProperties = Object.assign( - {'name': wmStateOrSfLayer.name}, sfImeContainerProperties); - // 'name' is the ..d..h..m..s..ms timestamp + const sfProperties = extractSfProperties(wmStateOrSfLayer); + imeTraceFile.data[i].sfProperties = sfProperties; const sfSubtrees = filterSfLayerForIme( wmStateOrSfLayer); // for display in Hierarchy sub-panel imeTraceFile.data[i].children.push(...sfSubtrees); - if (sfImeContainerProperties || sfSubtrees.length > 0) { + if (sfProperties || sfSubtrees.length > 0) { imeTraceFile.data[0].hasWmSfProperties = true; } } @@ -113,6 +111,9 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) { wmOrSfIndex++; currWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; } + // if wmOrSfIndex is 0, i.e. no corresponding entry is found, + // we take the first wm / sf entry + // intersectWmOrSfIndices.push(Math.max(0, wmOrSfIndex - 1)); intersectWmOrSfIndices.push(wmOrSfIndex - 1); } console.log('done matching corresponding timestamps'); @@ -121,21 +122,46 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) { function filterWmStateForIme(wmState) { // create and return a custom entry that just contains relevant properties - const displayContent = wmState.children[0]; + const displayContent = wmState.root.children[0]; + const controlTargetActualName = getActualFieldNameFromPossibilities( + 'controlTarget', displayContent.proto); + const inputTargetActualName = + getActualFieldNameFromPossibilities('inputTarget', displayContent.proto); + const layeringTargetActualName = getActualFieldNameFromPossibilities( + 'layeringTarget', displayContent.proto); + const isInputMethodWindowVisible = findInputMethodVisibility(displayContent); return { 'kind': 'WM State Properties', - 'name': wmState.name, // this is the ..d..h..m..s..ms timestamp + 'name': wmState.name, // this is the ..d..h..m..s..ms timestamp 'stableId': wmState.stableId, 'focusedApp': wmState.focusedApp, 'focusedWindow': wmState.focusedWindow, 'focusedActivity': wmState.focusedActivity, - 'inputMethodControlTarget': displayContent.proto.inputMethodControlTarget, - 'inputMethodInputTarget': displayContent.proto.inputMethodInputTarget, - 'inputMethodTarget': displayContent.proto.inputMethodTarget, + 'isInputMethodWindowVisible': isInputMethodWindowVisible, + 'imeControlTarget': displayContent.proto[controlTargetActualName], + 'imeInputTarget': displayContent.proto[inputTargetActualName], + 'imeLayeringTarget': displayContent.proto[layeringTargetActualName], 'imeInsetsSourceProvider': displayContent.proto.imeInsetsSourceProvider, + 'proto': wmState, }; } +function getActualFieldNameFromPossibilities(wantedKey, protoObject) { + // for backwards compatibility purposes: find the actual name in the + // protoObject out of a list of possible names, as field names may change + const possibleNamesMap = { + // inputMethod...Target is legacy name, ime...Target is new name + 'controlTarget': ['inputMethodControlTarget', 'imeControlTarget'], + 'inputTarget': ['inputMethodInputTarget', 'imeInputTarget'], + 'layeringTarget': ['inputMethodTarget', 'imeLayeringTarget'], + }; + + const possibleNames = possibleNamesMap[wantedKey]; + const actualName = + Object.keys(protoObject).find((el) => possibleNames.includes(el)); + return actualName; +} + function filterSfLayerForIme(sfLayer) { const parentTaskOfImeContainer = findParentTaskOfNode(sfLayer, 'ImeContainer'); @@ -175,19 +201,41 @@ function findParentTaskOfNode(curr, nodeName) { return null; } -function extractImeContainerFields(curr) { +function extractSfProperties(curr) { + const imeFields = extractImeFields(curr); + if (imeFields == null) { + return null; + } + return Object.assign({'name': curr.name, 'proto': curr}, imeFields); + // 'name' is the ..d..h..m..s..ms timestamp +} + +function extractImeFields(curr) { const isImeContainer = getFilter('ImeContainer'); - if (isImeContainer(curr)) { + const imeContainer = findWindowOrLayerMatch(isImeContainer, curr); + if (imeContainer != null) { + const isInputMethodSurface = getFilter('InputMethod'); + const inputMethodSurface = + findWindowOrLayerMatch(isInputMethodSurface, imeContainer); return { - 'screenBounds': curr.screenBounds, - 'rect': curr.rect, - 'zOrderRelativeOfId': curr.zOrderRelativeOfId, - 'z': curr.z, + 'imeContainer': imeContainer, + 'inputMethodSurface': inputMethodSurface, + 'screenBounds': inputMethodSurface.screenBounds, + 'rect': inputMethodSurface.rect, + 'isInputMethodSurfaceVisible': inputMethodSurface.isVisible, + 'zOrderRelativeOfId': imeContainer.zOrderRelativeOfId, + 'z': imeContainer.z, }; } - // search for ImeContainer in children + return null; +} + +function findWindowOrLayerMatch(filter, curr) { + if (filter(curr)) { + return curr; + } for (const child of curr.children) { - const result = extractImeContainerFields(child); + const result = findWindowOrLayerMatch(filter, child); if (result) { return result; } @@ -195,4 +243,29 @@ function extractImeContainerFields(curr) { return null; } +function findInputMethodVisibility(windowOrLayer) { + const isInputMethod = getFilter('InputMethod'); + const inputMethodWindowOrLayer = + findWindowOrLayerMatch(isInputMethod, windowOrLayer); + return inputMethodWindowOrLayer ? inputMethodWindowOrLayer.isVisible : false; +} + +function processImeAfterCombiningWmAndSfProperties(imeTraceFile) { + for (const imeEntry of imeTraceFile.data) { + if (imeEntry.wmProperties?.focusedWindow && imeEntry.sfProperties?.proto) { + const focusedWindowRgba = findFocusedWindowRgba( + imeEntry.wmProperties.focusedWindow, imeEntry.sfProperties.proto); + imeEntry.sfProperties.focusedWindowRgba = focusedWindowRgba; + } + } +} + +function findFocusedWindowRgba(focusedWindow, sfLayer) { + const focusedWindowToken = focusedWindow.token; + console.log(focusedWindowToken); + const isFocusedWindow = getFilter(focusedWindowToken); + const focusedWindowLayer = findWindowOrLayerMatch(isFocusedWindow, sfLayer); + return focusedWindowLayer.color; +} + export {combineWmSfWithImeDataIfExisting};