tools/captured_sites/metrics-scraper.py

#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Script that compares the metrics when running Autofill captured site tests
# with some features enabled and disabled.
# Run from the root of the Chromium src directory. -h for help.
# The tool expects that the captured_sites_interactive_tests binary is built.
import argparse, json, os, urllib.parse


# Extracts the names of all non-disabled captured site tests.
def get_all_sites():
  with open("chrome/test/data/autofill/captured_sites/artifacts/testcases.json",
            "r") as f:
    return (site["site_name"] for site in json.loads(f.read())["tests"]
            if not site.get("disabled", False))

# Command line args.
parser = argparse.ArgumentParser(epilog="List of tests: " +
                                 ", ".join(get_all_sites()))
parser.add_argument(
    "target",
    help=("Build target of captured_sites_interactive_tests binary. "
          "For example, Default. The binary should already exist."))
parser.add_argument("features", help="A comma-separated list of feature names.")
parser.add_argument(
    "-r",
    dest="histogram_regex",
    help=("A regex matching the histogram names that should be dumped. If not "
          "specified, the metrics of all histograms dumped."))
parser.add_argument(
    "-o",
    dest="output",
    default="/tmp/",
    help=("Directory to record the metrics into. Creates files per test, named"
          " after the test case."))
parser.add_argument(
    "-t",
    dest="test",
    help="Test case. If no test is specified, all tests are run.")
parser.add_argument("-s",
                    dest="silent",
                    action="store_true",
                    help="Don't print test output.")
args = parser.parse_args()

# The captured_sites_interactive_tests binary should be built.
captured_site_tests = "out/%s/captured_sites_interactive_tests" % args.target
assert os.path.exists(captured_site_tests)


# Runs the capture site test for `site` and scrapes the metrics.
# args.features is enabled/disabled depending on `features_enabled`.
def run_test(site, features_enabled):
  cmd = "./" + captured_site_tests
  cmd += (" --gtest_filter="
          "All/AutofillCapturedSitesInteractiveTest.Recipe/" + site)
  # Enable scraping tools. Special characters need to be escaped.
  cmd += " --enable-features=AutofillCapturedSiteTestsMetricsScraper"
  cmd += ":output_dir/" + urllib.parse.quote(args.output, safe="")
  if args.histogram_regex is not None:
    cmd += "/histogram_regex/" + urllib.parse.quote(args.histogram_regex,
                                                    safe="")
  # En- or disable features.
  if features_enabled:
    cmd += "," + args.features
  else:
    cmd += " --disable-features=" + args.features
  # Random arguments that the captured site tests recommend.
  cmd += " --enable-pixel-output-in-tests"
  cmd += " --test-launcher-timeout=10000000"
  cmd += " --ui-test-action-max-timeout=10000000"
  cmd += (" --vmodule=captured_sites_test_utils=2\,"
          "autofill_captured_sites_interactive_uitest=1")
  # Maybe disable output.
  if args.silent:
    cmd += " > /dev/null 2>&1"
  # Run
  os.system(cmd)


# Runs the captured site test `site` twice. Once with `args.features` enabled
# and once with the feature disabled.
# Diffs the metrics collected.
def run_tests_and_diff(site):
  print("Testing %s..." % site)

  def file_name(infix):
    return "%s/%s%s.txt" % (args.output, site, infix)

  # `output` is where the captured site test will write to. Rename the file
  # afterwards to distinguish between enabled/disabled state.
  output = file_name("")
  result_enabled = file_name("_enabled")
  result_disabled = file_name("_disabled")

  print("Running with features enabled. Results at " + result_enabled)
  run_test(site, True)
  os.rename(output, result_enabled)

  print("Running with features disabled. Results at " + result_disabled)
  run_test(site, False)
  os.rename(output, result_disabled)

  print("Comparing metrics (no output means no diff)")
  os.system("diff %s %s" % (result_enabled, result_disabled))
  print("")

# If a test is specified, only run that specific test. Otherwise run all.
if args.test is None:
  for site in get_all_sites():
    run_tests_and_diff(site)
else:
  run_tests_and_diff(args.test)