From f020989c9409d8d098965c420035b2d48fefab28 Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Mon, 29 Aug 2022 14:07:18 +0900 Subject: [PATCH] Generate ninja_inputs.pb as build metrics To monitor ninja inputs, generate ninja_inputs.pb as build metrics. This will count the number of input files and required git projects, with the number of files for each project. Bug: 241033491 Test: $ cp .repo/manifests/default.xml out/dist/manifest_1.xml $ development/multitree/build_vendor.sh -d vsoc_x86_64 \ -p cf_x86_64_phone -r out/dist -i 1 Change-Id: Ie185863f1c8fca313baba6135374d509e2d93937 --- multitree/build_vendor.sh | 4 +- tools/ninja_dependency_analysis/Android.bp | 8 ++++ .../collect_ninja_inputs.py | 12 +++++- .../ninja_metrics_proto/ninja_metrics.proto | 28 ++++++++++++++ .../ninja_metrics_proto/ninja_metrics.py | 37 +++++++++++++++++++ 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.proto create mode 100644 tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.py diff --git a/multitree/build_vendor.sh b/multitree/build_vendor.sh index 790d31044..84afcfdd9 100755 --- a/multitree/build_vendor.sh +++ b/multitree/build_vendor.sh @@ -60,6 +60,8 @@ done cp out/target/product/$device/vendor.img $dist_dir +mkdir -p $dist_dir/logs + out/host/linux-x86/bin/collect_ninja_inputs -n prebuilts/build-tools/linux-x86/bin/ninja \ -f out/combined-$product.ninja -t vendorimage -m $dist_dir/manifest_$build_id.xml \ - > $dist_dir/ninja_inputs.json + --out $dist_dir/logs/ninja_inputs diff --git a/tools/ninja_dependency_analysis/Android.bp b/tools/ninja_dependency_analysis/Android.bp index b8756487f..cda0ba813 100644 --- a/tools/ninja_dependency_analysis/Android.bp +++ b/tools/ninja_dependency_analysis/Android.bp @@ -6,7 +6,15 @@ python_binary_host { name: "collect_ninja_inputs", srcs: [ "collect_ninja_inputs.py", + "ninja_metrics_proto/ninja_metrics.py", + "ninja_metrics_proto/ninja_metrics.proto", ], + libs: [ + "libprotobuf-python", + ], + proto: { + canonical_path_from_root: false, + }, version: { py2: { enabled: false, diff --git a/tools/ninja_dependency_analysis/collect_ninja_inputs.py b/tools/ninja_dependency_analysis/collect_ninja_inputs.py index e7f2234c8..f78357f4a 100755 --- a/tools/ninja_dependency_analysis/collect_ninja_inputs.py +++ b/tools/ninja_dependency_analysis/collect_ninja_inputs.py @@ -23,7 +23,7 @@ import sys import xml.etree.ElementTree as ET from collections import OrderedDict from operator import itemgetter - +from ninja_metrics_proto import ninja_metrics def build_cmd(ninja_binary, ninja_file, target, exempted_file_list): cmd = [ninja_binary, '-f', ninja_file, '-t', 'inputs'] @@ -53,6 +53,7 @@ parser.add_argument('-n', '--ninja_binary', type=pathlib.Path, required=True) parser.add_argument('-f', '--ninja_file', type=pathlib.Path, required=True) parser.add_argument('-t', '--target', type=str, required=True) parser.add_argument('-e', '--exempted_file_list', type=pathlib.Path) +parser.add_argument('-o', '--out', type=pathlib.Path) group = parser.add_mutually_exclusive_group() group.add_argument('-r', '--repo_project_list', type=pathlib.Path) group.add_argument('-m', '--repo_manifest', type=pathlib.Path) @@ -82,4 +83,11 @@ if projects: result['total_project_count'] = len(project_to_count) result['total_input_count'] = len(input_files) -print(json.dumps(result, indent=2)) + +if args.out: + with open(os.path.join(args.out.parent, args.out.name + '.json'), 'w') as json_file: + json.dump(result, json_file, indent=2) + with open(os.path.join(args.out.parent, args.out.name + '.pb'), 'wb') as pb_file: + pb_file.write(ninja_metrics.generate_proto(result).SerializeToString()) +else: + print(json.dumps(result, indent=2)) diff --git a/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.proto b/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.proto new file mode 100644 index 000000000..f6294f07a --- /dev/null +++ b/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.proto @@ -0,0 +1,28 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ninja_metrics; + +message NinjaMetrics { + uint32 num_input_files = 1; + uint32 num_projects = 2; + repeated ProjectInfo project_infos = 3; +} + +message ProjectInfo { + string name = 1; + uint32 num_input_files = 2; +} \ No newline at end of file diff --git a/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.py b/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.py new file mode 100644 index 000000000..6dff394fb --- /dev/null +++ b/tools/ninja_dependency_analysis/ninja_metrics_proto/ninja_metrics.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ninja_metrics_proto import ninja_metrics_pb2 + +def generate_proto(ninja_data: dict): + proto = ninja_metrics_pb2.NinjaMetrics() + proto.num_input_files = ninja_data['total_input_count'] + proto.num_projects = ninja_data['total_project_count'] + for project in ninja_data['project_count']: + project_info = proto.project_infos.add() + project_info.name = project + project_info.num_input_files = ninja_data['project_count'][project] + return proto + +def print_proto_file(proto_file): + proto = ninja_metrics_pb2.NinjaMetrics() + proto.ParseFromString(proto_file.read()) + print("num_input_files: ", proto.num_input_files) + print("num_projects: ", proto.num_projects) + print("project_infos: [") + for project in proto.project_infos: + print(f" {project.name}: {project.num_input_files}") + print("]") \ No newline at end of file