From cb0d23d9dd34968e894fed9e6e57a914d00ae64f Mon Sep 17 00:00:00 2001 From: Hui Ling Shi Date: Mon, 18 Jul 2022 09:16:52 +0000 Subject: [PATCH 1/5] [9] Fix null targets in IME's 'WM & SF Properties' Fix incorrect names of properties being extracted from WM Trace: - 'Input Method Control Target' has been replaced with 'IME Control Target', 'Input Method Input Target' with 'IME Input Target', and 'Input Method Target' with 'IME Layering Target'. Bug: 239145867 Test: manual on local build of winscope Change-Id: If067cf73de32d4374b33d1f07f179b63c9fb0c27 --- .../winscope/src/ImeAdditionalProperties.vue | 161 +++++++++++++----- tools/winscope/src/ime_processing.js | 111 +++++++++--- 2 files changed, 206 insertions(+), 66 deletions(-) diff --git a/tools/winscope/src/ImeAdditionalProperties.vue b/tools/winscope/src/ImeAdditionalProperties.vue index 3bd855c6d..53d6c2731 100644 --- a/tools/winscope/src/ImeAdditionalProperties.vue +++ b/tools/winscope/src/ImeAdditionalProperties.vue @@ -20,16 +20,29 @@
- WMState + + WMState
{{ entry.wmProperties.name }} There is no corresponding WMState entry.
-
- IME Insets Source Provider +
+
+
Source Frame: @@ -56,40 +69,40 @@
- IMControl Target + Control Target
null
- IMInput Target + Input Target
null
- IM Target + Layering Target
null
@@ -99,7 +112,14 @@
- WMState + + WMState
{{ entry.wmProperties.name }} @@ -109,8 +129,8 @@
SFLayer
- {{ - entry.sfImeContainerProperties.name }} + {{ + entry.sfProperties.name }} There is no corresponding SFLayer entry.
@@ -126,29 +146,66 @@ Focused Window: {{ entry.wmProperties.focusedWindow }}
- Frame: + Focused Window Color: + {{ + entry.sfProperties.focusedWindowRgba + }} +
+ Input Control Target Frame:
-
- Ime Container +
+ Visibility
- ScreenBounds: - + InputMethod Window: + {{ + entry.wmProperties.isInputMethodWindowVisible + }}
- Rect: - + InputMethod Surface: + {{ + entry.sfProperties.isInputMethodSurfaceVisible }}
+
+
+
+ +
ZOrderRelativeOfId: {{ - entry.sfImeContainerProperties.zOrderRelativeOfId + entry.sfProperties.zOrderRelativeOfId }}
Z: - {{ entry.sfImeContainerProperties.z }} + {{ entry.sfProperties.z }} +
+
+
+
+ +
+ ScreenBounds: + +
+ Rect: +
@@ -209,6 +266,15 @@ export default { }, computed: { + wmProtoOrNull() { + return this.entry.wmProperties?.proto; + }, + wmInsetsSourceProviderOrNull() { + return this.entry.wmProperties?.imeInsetsSourceProvider ? + Object.assign({'name': 'Ime Insets Source Provider'}, + this.entry.wmProperties.imeInsetsSourceProvider) : + null; + }, wmControlTargetFrameOrNull() { return this.entry.wmProperties?.imeInsetsSourceProvider ?.insetsSourceProvider?.controlTarget?.windowFrames?.frame || 'null'; @@ -237,36 +303,36 @@ export default { return this.entry.wmProperties?.imeInsetsSourceProvider ?.insetsSourceProvider?.source?.visibleFrame || 'null'; }, - wmIMControlTargetOrNull() { - return this.entry?.wmProperties?.inputMethodControlTarget ? - Object.assign({'name': 'Input Method Control Target'}, - this.entry.wmProperties.inputMethodControlTarget) : + wmImeControlTargetOrNull() { + return this.entry?.wmProperties?.imeControlTarget ? + Object.assign({'name': 'IME Control Target'}, + this.entry.wmProperties.imeControlTarget) : null; }, - wmIMInputTargetOrNull() { - return this.entry?.wmProperties?.inputMethodInputTarget ? - Object.assign({'name': 'Input Method Input Target'}, - this.entry.wmProperties.inputMethodInputTarget) : + wmImeInputTargetOrNull() { + return this.entry?.wmProperties?.imeInputTarget ? + Object.assign({'name': 'IME Input Target'}, + this.entry.wmProperties.imeInputTarget) : null; }, - wmIMTargetOrNull() { - return this.entry?.wmProperties?.inputMethodTarget ? - Object.assign({'name': 'Input Method Target'}, - this.entry.wmProperties.inputMethodTarget) : + wmImeLayeringTargetOrNull() { + return this.entry?.wmProperties?.imeLayeringTarget ? + Object.assign({'name': 'IME Layering Target'}, + this.entry.wmProperties.imeLayeringTarget) : null; }, sfImeContainerScreenBoundsOrNull() { - return this.entry.sfImeContainerProperties?.screenBounds || 'null'; + return this.entry.sfProperties?.screenBounds || 'null'; }, sfImeContainerRectOrNull() { - return this.entry.sfImeContainerProperties?.rect || 'null'; + return this.entry.sfProperties?.rect || 'null'; }, isAllPropertiesNull() { if (this.isImeManagerService) { return !this.entry.wmProperties; } else { return !(this.entry.wmProperties || - this.entry.sfImeContainerProperties); + this.entry.sfProperties); } }, }, @@ -301,6 +367,7 @@ export default { .group-header { justify-content: center; + text-align: left; padding: 0px 5px; width: 95px; display: inline-block; diff --git a/tools/winscope/src/ime_processing.js b/tools/winscope/src/ime_processing.js index 4f196364b..8ae164fa1 100644 --- a/tools/winscope/src/ime_processing.js +++ b/tools/winscope/src/ime_processing.js @@ -60,6 +60,7 @@ function combineWmSfWithImeDataIfExisting(dataFiles) { combineWmSfPropertiesIntoImeData( imeTraceFile, filesAsDict[TRACE_TYPES.SURFACE_FLINGER]); } + processImeAfterCombiningWmAndSfProperties(imeTraceFile); } console.log('after combining', dataFiles); } @@ -84,17 +85,14 @@ function combineWmSfPropertiesIntoImeData(imeTraceFile, wmOrSfTraceFile) { // hasWmSfProperties is added into data because the // imeTraceFile object itself is inextensible if it's from file input } else if (wmStateOrSfLayer) { - const sfImeContainerProperties = - extractImeContainerFields(wmStateOrSfLayer); - imeTraceFile.data[i].sfImeContainerProperties = Object.assign( - {'name': wmStateOrSfLayer.name}, sfImeContainerProperties); - // 'name' is the ..d..h..m..s..ms timestamp + const sfProperties = extractSfProperties(wmStateOrSfLayer); + imeTraceFile.data[i].sfProperties = sfProperties; const sfSubtrees = filterSfLayerForIme( wmStateOrSfLayer); // for display in Hierarchy sub-panel imeTraceFile.data[i].children.push(...sfSubtrees); - if (sfImeContainerProperties || sfSubtrees.length > 0) { + if (sfProperties || sfSubtrees.length > 0) { imeTraceFile.data[0].hasWmSfProperties = true; } } @@ -113,6 +111,9 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) { wmOrSfIndex++; currWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; } + // if wmOrSfIndex is 0, i.e. no corresponding entry is found, + // we take the first wm / sf entry + // intersectWmOrSfIndices.push(Math.max(0, wmOrSfIndex - 1)); intersectWmOrSfIndices.push(wmOrSfIndex - 1); } console.log('done matching corresponding timestamps'); @@ -121,21 +122,46 @@ function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) { function filterWmStateForIme(wmState) { // create and return a custom entry that just contains relevant properties - const displayContent = wmState.children[0]; + const displayContent = wmState.root.children[0]; + const controlTargetActualName = getActualFieldNameFromPossibilities( + 'controlTarget', displayContent.proto); + const inputTargetActualName = + getActualFieldNameFromPossibilities('inputTarget', displayContent.proto); + const layeringTargetActualName = getActualFieldNameFromPossibilities( + 'layeringTarget', displayContent.proto); + const isInputMethodWindowVisible = findInputMethodVisibility(displayContent); return { 'kind': 'WM State Properties', - 'name': wmState.name, // this is the ..d..h..m..s..ms timestamp + 'name': wmState.name, // this is the ..d..h..m..s..ms timestamp 'stableId': wmState.stableId, 'focusedApp': wmState.focusedApp, 'focusedWindow': wmState.focusedWindow, 'focusedActivity': wmState.focusedActivity, - 'inputMethodControlTarget': displayContent.proto.inputMethodControlTarget, - 'inputMethodInputTarget': displayContent.proto.inputMethodInputTarget, - 'inputMethodTarget': displayContent.proto.inputMethodTarget, + 'isInputMethodWindowVisible': isInputMethodWindowVisible, + 'imeControlTarget': displayContent.proto[controlTargetActualName], + 'imeInputTarget': displayContent.proto[inputTargetActualName], + 'imeLayeringTarget': displayContent.proto[layeringTargetActualName], 'imeInsetsSourceProvider': displayContent.proto.imeInsetsSourceProvider, + 'proto': wmState, }; } +function getActualFieldNameFromPossibilities(wantedKey, protoObject) { + // for backwards compatibility purposes: find the actual name in the + // protoObject out of a list of possible names, as field names may change + const possibleNamesMap = { + // inputMethod...Target is legacy name, ime...Target is new name + 'controlTarget': ['inputMethodControlTarget', 'imeControlTarget'], + 'inputTarget': ['inputMethodInputTarget', 'imeInputTarget'], + 'layeringTarget': ['inputMethodTarget', 'imeLayeringTarget'], + }; + + const possibleNames = possibleNamesMap[wantedKey]; + const actualName = + Object.keys(protoObject).find((el) => possibleNames.includes(el)); + return actualName; +} + function filterSfLayerForIme(sfLayer) { const parentTaskOfImeContainer = findParentTaskOfNode(sfLayer, 'ImeContainer'); @@ -175,19 +201,41 @@ function findParentTaskOfNode(curr, nodeName) { return null; } -function extractImeContainerFields(curr) { +function extractSfProperties(curr) { + const imeFields = extractImeFields(curr); + if (imeFields == null) { + return null; + } + return Object.assign({'name': curr.name, 'proto': curr}, imeFields); + // 'name' is the ..d..h..m..s..ms timestamp +} + +function extractImeFields(curr) { const isImeContainer = getFilter('ImeContainer'); - if (isImeContainer(curr)) { + const imeContainer = findWindowOrLayerMatch(isImeContainer, curr); + if (imeContainer != null) { + const isInputMethodSurface = getFilter('InputMethod'); + const inputMethodSurface = + findWindowOrLayerMatch(isInputMethodSurface, imeContainer); return { - 'screenBounds': curr.screenBounds, - 'rect': curr.rect, - 'zOrderRelativeOfId': curr.zOrderRelativeOfId, - 'z': curr.z, + 'imeContainer': imeContainer, + 'inputMethodSurface': inputMethodSurface, + 'screenBounds': inputMethodSurface.screenBounds, + 'rect': inputMethodSurface.rect, + 'isInputMethodSurfaceVisible': inputMethodSurface.isVisible, + 'zOrderRelativeOfId': imeContainer.zOrderRelativeOfId, + 'z': imeContainer.z, }; } - // search for ImeContainer in children + return null; +} + +function findWindowOrLayerMatch(filter, curr) { + if (filter(curr)) { + return curr; + } for (const child of curr.children) { - const result = extractImeContainerFields(child); + const result = findWindowOrLayerMatch(filter, child); if (result) { return result; } @@ -195,4 +243,29 @@ function extractImeContainerFields(curr) { return null; } +function findInputMethodVisibility(windowOrLayer) { + const isInputMethod = getFilter('InputMethod'); + const inputMethodWindowOrLayer = + findWindowOrLayerMatch(isInputMethod, windowOrLayer); + return inputMethodWindowOrLayer ? inputMethodWindowOrLayer.isVisible : false; +} + +function processImeAfterCombiningWmAndSfProperties(imeTraceFile) { + for (const imeEntry of imeTraceFile.data) { + if (imeEntry.wmProperties?.focusedWindow && imeEntry.sfProperties?.proto) { + const focusedWindowRgba = findFocusedWindowRgba( + imeEntry.wmProperties.focusedWindow, imeEntry.sfProperties.proto); + imeEntry.sfProperties.focusedWindowRgba = focusedWindowRgba; + } + } +} + +function findFocusedWindowRgba(focusedWindow, sfLayer) { + const focusedWindowToken = focusedWindow.token; + console.log(focusedWindowToken); + const isFocusedWindow = getFilter(focusedWindowToken); + const focusedWindowLayer = findWindowOrLayerMatch(isFocusedWindow, sfLayer); + return focusedWindowLayer.color; +} + export {combineWmSfWithImeDataIfExisting}; From 948c89706564ec4361b49876d8a060291fd58ab4 Mon Sep 17 00:00:00 2001 From: Hui Ling Shi Date: Wed, 27 Jul 2022 08:52:22 +0000 Subject: [PATCH 2/5] [10] Refine WM & SF Properties IME subpanel Make the following headers clickable; upon clicking, will show proto details in the 'Properties' sub-panel on the right: - WMState - Ime Container - Input Method Surface - IME Insets Source Provider Add the following properties: - Focused window's color / rgba information -- focused window title extracted from WM trace, the corresponding surface's rgba extracted from SF trace - InputMethod Window's visibility - InputMethod Surface's visibility - InputMethod Surface's rect & screenBounds Remove the following properties: - ImeContainer's rect & screenBounds Rename the following properties: - Frame (under 'Focus' heading) --> Input Control Target Frame Screenshots: in buganizer ticket Bug: 236226833 Test: manual on local build of winscope Change-Id: I2a2747b57639f40c8f6bf71d5cb49268e4ee5044 --- tools/winscope/src/ime_processing.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/winscope/src/ime_processing.js b/tools/winscope/src/ime_processing.js index 8ae164fa1..13c0c1d48 100644 --- a/tools/winscope/src/ime_processing.js +++ b/tools/winscope/src/ime_processing.js @@ -132,7 +132,7 @@ function filterWmStateForIme(wmState) { const isInputMethodWindowVisible = findInputMethodVisibility(displayContent); return { 'kind': 'WM State Properties', - 'name': wmState.name, // this is the ..d..h..m..s..ms timestamp + 'name': wmState.name, // 'name' is a timestamp of ..d..h..m..s..ms format 'stableId': wmState.stableId, 'focusedApp': wmState.focusedApp, 'focusedWindow': wmState.focusedWindow, @@ -207,7 +207,7 @@ function extractSfProperties(curr) { return null; } return Object.assign({'name': curr.name, 'proto': curr}, imeFields); - // 'name' is the ..d..h..m..s..ms timestamp + // 'name' is a timestamp of ..d..h..m..s..ms format } function extractImeFields(curr) { @@ -247,7 +247,7 @@ function findInputMethodVisibility(windowOrLayer) { const isInputMethod = getFilter('InputMethod'); const inputMethodWindowOrLayer = findWindowOrLayerMatch(isInputMethod, windowOrLayer); - return inputMethodWindowOrLayer ? inputMethodWindowOrLayer.isVisible : false; + return inputMethodWindowOrLayer && inputMethodWindowOrLayer.isVisible; } function processImeAfterCombiningWmAndSfProperties(imeTraceFile) { From 595c67d790c09c8cd018efd7cc2efa480aeb0b21 Mon Sep 17 00:00:00 2001 From: Hui Ling Shi Date: Wed, 27 Jul 2022 12:29:10 +0000 Subject: [PATCH 3/5] [11] Change matching WM/SF entry to IME entry algo Due to there being some slightly different timestamps between WM/SF trace and IME trace, we want to allow some "fault tolerance" -- even if a WM/SF entry comes after IME entry in time, it may still be the corresponding entry if it's within a manually-defined "fault tolerance" time. For now, fault tolerance is defined to be 200ms. So if a WM/SF entry is within IME entry +- 200ms, it may be considered a matching / corresponding entry. Within this range, we will always try to pick a WM/SF entry that comes before the IME entry. If there is no such WM/SF entry, then we will pick one that comes after. Bug: 236226833 Test: manual on local build of Winscope Change-Id: I0782ec520284517bf964eeffd552e6175758c9f1 --- .../winscope/src/ImeAdditionalProperties.vue | 8 ++--- tools/winscope/src/ime_processing.js | 32 +++++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/tools/winscope/src/ImeAdditionalProperties.vue b/tools/winscope/src/ImeAdditionalProperties.vue index 53d6c2731..0ecd679a5 100644 --- a/tools/winscope/src/ImeAdditionalProperties.vue +++ b/tools/winscope/src/ImeAdditionalProperties.vue @@ -159,13 +159,13 @@
Visibility
- InputMethod Window: - {{ + InputMethod Window: + {{ entry.wmProperties.isInputMethodWindowVisible }}
- InputMethod Surface: - {{ + InputMethod Surface: + {{ entry.sfProperties.isInputMethodSurfaceVisible }}
diff --git a/tools/winscope/src/ime_processing.js b/tools/winscope/src/ime_processing.js index 13c0c1d48..1fe913a78 100644 --- a/tools/winscope/src/ime_processing.js +++ b/tools/winscope/src/ime_processing.js @@ -100,23 +100,35 @@ function combineWmSfPropertiesIntoImeData(imeTraceFile, wmOrSfTraceFile) { } function matchCorrespondingTimestamps(imeTimestamps, wmOrSfTimestamps) { - // find the latest sf / wm timestamp that comes before current ime timestamp + // we want to take the earliest sf / wm entry that is within + // +-[FAULT_TOLERANCE] ns of the ime entry as the corresponding sf / wm entry + const FAULT_TOLERANCE = 200000000; // 200ms in ns + let wmOrSfIndex = 0; const intersectWmOrSfIndices = []; for (let imeIndex = 0; imeIndex < imeTimestamps.length; imeIndex++) { const currImeTimestamp = imeTimestamps[imeIndex]; - - let currWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; - while (currWmOrSfTimestamp < currImeTimestamp) { + let wmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; + while (wmOrSfTimestamp < currImeTimestamp) { wmOrSfIndex++; - currWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; + wmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex]; + } + // wmOrSfIndex should now be at the first entry that is past + // [currImeTimestamp] ns. We want the most recent entry that comes + // before [currImeTimestamp] ns if it's within + // [currImeTimestamp - FAULT_TOLERANCE] ns. Otherwise, we want the most + // recent entry that comes after [currImeTimestamp] ns if it's within + // [currImeTimestamp + FAULT_TOLERANCE] ns. + const previousWmOrSfTimestamp = wmOrSfTimestamps[wmOrSfIndex - 1]; + if (previousWmOrSfTimestamp >= currImeTimestamp - FAULT_TOLERANCE) { + intersectWmOrSfIndices.push(wmOrSfIndex - 1); + } else if (wmOrSfTimestamp <= currImeTimestamp + FAULT_TOLERANCE) { + intersectWmOrSfIndices.push(wmOrSfIndex); + } else { + intersectWmOrSfIndices.push(null); } - // if wmOrSfIndex is 0, i.e. no corresponding entry is found, - // we take the first wm / sf entry - // intersectWmOrSfIndices.push(Math.max(0, wmOrSfIndex - 1)); - intersectWmOrSfIndices.push(wmOrSfIndex - 1); } - console.log('done matching corresponding timestamps'); + console.log('done matching corresponding timestamps', intersectWmOrSfIndices); return intersectWmOrSfIndices; } From 8a507b7a5457c293bccb0f5c2c7dd1d44114b2c9 Mon Sep 17 00:00:00 2001 From: Hui Ling Shi Date: Wed, 27 Jul 2022 13:09:44 +0000 Subject: [PATCH 4/5] Fix SFLayer TypeErrors in Winscope Fix the errors that pop up when clicking `SurfaceFlingerLayer` in SurfaceFlingerTrace panel's Hierarchy panel, by taking care of cases where the property is null. Bug: 240405932 Test: manual on local build of Winscope Change-Id: Ic9e8f6e75dcfde8af97aeb5254d1e2d8f7b3d669 --- tools/winscope/src/SurfaceFlingerTraceView.vue | 2 +- tools/winscope/src/TransformMatrix.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/winscope/src/SurfaceFlingerTraceView.vue b/tools/winscope/src/SurfaceFlingerTraceView.vue index 8cd94da07..602a93b99 100644 --- a/tools/winscope/src/SurfaceFlingerTraceView.vue +++ b/tools/winscope/src/SurfaceFlingerTraceView.vue @@ -37,7 +37,7 @@ export default { summarizer(layer) { const summary = []; - if (layer?.visibilityReason.length > 0) { + if (layer?.visibilityReason?.length > 0) { let reason = ""; if (Array.isArray(layer.visibilityReason)) { reason = layer.visibilityReason.join(", "); diff --git a/tools/winscope/src/TransformMatrix.vue b/tools/winscope/src/TransformMatrix.vue index c5541364a..d8b7a8363 100644 --- a/tools/winscope/src/TransformMatrix.vue +++ b/tools/winscope/src/TransformMatrix.vue @@ -13,7 +13,7 @@ limitations under the License. -->