bench.py

#!/usr/bin/env python3
"""Benchcoin - Bitcoin Core benchmarking toolkit.

A CLI for building, benchmarking, analyzing, and reporting on Bitcoin Core
performance. PR results are compared against nightly baseline data.

Usage:
    bench.py experiment run MANIFEST Run a declarative benchmark experiment
    bench.py build COMMIT            Build bitcoind at a commit
    bench.py analyze COMMIT LOGFILE  Generate plots from debug.log
    bench.py report OUTPUT           Generate HTML report with nightly comparison
    bench.py nightly append-experiment ... Append experiment output to nightly history
    bench.py nightly chart ...       Generate nightly chart HTML

Examples:
    # Run a PR-style experiment
    bench.py experiment run bench/experiments/pr.toml --datadir /data

    # Build at HEAD
    bench.py build HEAD:pr

    # Generate HTML report with nightly comparison
    bench.py report --experiment-output ./experiment-output --nightly-history ./nightly-history.json ./output

    # Append nightly result and regenerate chart
    bench.py nightly append-experiment ./experiment-output abc123
    bench.py nightly chart ./index.html
"""

from __future__ import annotations

import argparse
import logging
import sys
from pathlib import Path

from bench.capabilities import detect_capabilities
from bench.config import build_config

logging.basicConfig(
    level=logging.INFO,
    format="%(levelname)s: %(message)s",
)
logger = logging.getLogger(__name__)


def cmd_build(args: argparse.Namespace) -> int:
    """Build bitcoind at a commit."""
    from bench.build import BuildPhase
    from bench.environment import BuildEnvironment

    capabilities = detect_capabilities()
    config = build_config(
        cli_args={
            "binaries_dir": args.output_dir,
            "skip_existing": args.skip_existing,
            "dry_run": args.dry_run,
            "verbose": args.verbose,
        },
        config_file=Path(args.config) if args.config else None,
        profile=args.profile,
    )

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    environment = BuildEnvironment.from_config(config)
    phase = BuildPhase(environment, capabilities)

    try:
        result = phase.run(
            args.commit,
            output_dir=Path(args.output_dir) if args.output_dir else None,
        )
        logger.info(f"Built binary: {result.binary.name} at {result.binary.path}")
        return 0
    except Exception as e:
        logger.error(f"Build failed: {e}")
        return 1


def cmd_analyze(args: argparse.Namespace) -> int:
    """Generate plots from debug.log."""
    from bench.analyze import AnalyzePhase

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    log_file = Path(args.log_file)
    output_dir = Path(args.output_dir)

    if not log_file.exists():
        logger.error(f"Log file not found: {log_file}")
        return 1

    phase = AnalyzePhase()

    try:
        result = phase.run(
            commit=args.commit,
            log_file=log_file,
            output_dir=output_dir,
        )
        logger.info(f"Generated {len(result.plots)} plots in {result.output_dir}")
        return 0
    except Exception as e:
        logger.error(f"Analysis failed: {e}")
        if args.verbose:
            import traceback

            traceback.print_exc()
        return 1


def cmd_report(args: argparse.Namespace) -> int:
    """Generate HTML report from benchmark results."""
    from bench.report import ReportPhase

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    output_dir = Path(args.output_dir)
    nightly_history_file = Path(args.nightly_history) if args.nightly_history else None
    phase = ReportPhase(nightly_history_file=nightly_history_file)

    try:
        # CI experiment mode
        if args.experiment_output:
            experiment_output = Path(args.experiment_output)
            if not experiment_output.exists():
                logger.error(f"Experiment output not found: {experiment_output}")
                return 1

            result = phase.run_experiment(
                experiment_dir=experiment_output,
                output_dir=output_dir,
                title=args.title or "Benchmark Results",
                pr_number=args.pr_number,
                run_id=args.run_id,
                commit=args.commit,
            )

            # Update results index if we have a results directory
            # Note: This writes to results/index.html, not the main index.html
            # The main index.html is generated by the nightly benchmark chart
            if args.update_index:
                results_base = output_dir.parent.parent  # Go up from pr-N/run-id
                if results_base.exists():
                    phase.update_index(results_base, results_base / "index.html")
        else:
            # Standard single-directory mode
            input_dir = Path(args.input_dir)

            if not input_dir.exists():
                logger.error(f"Input directory not found: {input_dir}")
                return 1

            result = phase.run(
                input_dir=input_dir,
                output_dir=output_dir,
                title=args.title or "Benchmark Results",
            )

        # Print nightly comparison (speedups vs nightly)
        if result.speedups:
            logger.info("Comparison to nightly:")
            for config, speedup in result.speedups.items():
                sign = "+" if speedup > 0 else ""
                logger.info(f"  {config}: {sign}{speedup}%")

        return 0
    except Exception as e:
        logger.error(f"Report generation failed: {e}")
        if args.verbose:
            import traceback

            traceback.print_exc()
        return 1


def cmd_nightly(args: argparse.Namespace) -> int:
    """Manage nightly benchmark history and charts."""
    from bench.nightly import NightlyPhase

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    if not args.nightly_command:
        logger.error(
            "No nightly subcommand specified. Use 'append', "
            "'append-experiment', or 'chart'."
        )
        return 1

    history_file = Path(args.history_file)
    phase = NightlyPhase(history_file)

    try:
        if args.nightly_command == "append":
            machine_specs_file = (
                Path(args.machine_specs) if args.machine_specs else None
            )
            phase.append(
                results_file=Path(args.results_file),
                commit=args.commit,
                dbcache=args.dbcache,
                date_str=args.date,
                experiment_config_file=Path(args.experiment_config)
                if args.experiment_config
                else None,
                profile_name=args.profile_name,
                instrumentation=args.instrumentation,
                machine_specs_file=machine_specs_file,
                run_date=args.run_date or "",
                trigger=args.trigger,
            )
            logger.info(f"Appended result to {history_file}")
        elif args.nightly_command == "append-experiment":
            machine_specs_file = (
                Path(args.machine_specs) if args.machine_specs else None
            )
            count = phase.append_experiment(
                experiment_dir=Path(args.experiment_dir),
                commit=args.commit,
                date_str=args.date,
                machine_specs_file=machine_specs_file,
                run_date=args.run_date or "",
                trigger=args.trigger,
            )
            logger.info(f"Appended {count} experiment result(s) to {history_file}")
        elif args.nightly_command == "chart":
            phase.chart(output_file=Path(args.output_file))
            logger.info(f"Generated chart at {args.output_file}")
        return 0
    except Exception as e:
        logger.error(f"Nightly operation failed: {e}")
        if args.verbose:
            import traceback

            traceback.print_exc()
        return 1


def cmd_experiment(args: argparse.Namespace) -> int:
    """Run an experiment manifest."""
    from bench.experiment import Experiment, ExperimentRunner
    from bench.environment import ExperimentEnvironment

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    capabilities = detect_capabilities()
    config = build_config(
        cli_args={
            "binaries_dir": args.binaries_dir,
            "output_dir": args.output_dir,
            "skip_existing": args.skip_existing,
            "no_cache_drop": args.no_cache_drop,
            "dry_run": args.dry_run,
            "verbose": args.verbose,
        },
        config_file=Path(args.config) if args.config else None,
        profile=args.profile,
    )

    try:
        experiment = Experiment.from_toml(Path(args.manifest))
        environment = ExperimentEnvironment.from_config(config)
        runner = ExperimentRunner(environment, capabilities)
        result = runner.run(
            experiment=experiment,
            datadir=Path(args.datadir) if args.datadir else None,
            tmp_dir=Path(args.tmp_dir) if args.tmp_dir else None,
            subject_names=args.subject_name,
            profile_names=args.profile_name,
        )
        logger.info(f"Experiment outputs saved to: {result.output_dir}")
        logger.info(f"Completed {len(result.runs)} benchmark runs")
        if result.comparisons:
            logger.info(f"Generated {len(result.comparisons)} comparison artifacts")
        return 0
    except Exception as e:
        logger.error(f"Experiment failed: {e}")
        if args.verbose:
            import traceback

            traceback.print_exc()
        return 1


def main() -> int:
    """Main entry point."""
    parser = argparse.ArgumentParser(
        description="Benchcoin - Bitcoin Core benchmarking toolkit",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )

    parser.add_argument(
        "--config",
        metavar="PATH",
        help="Config file (default: bench.toml)",
    )
    parser.add_argument(
        "--profile",
        choices=["quick", "full", "ci"],
        default="full",
        help="Configuration profile (default: full)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Verbose output",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what would be done without executing",
    )

    subparsers = parser.add_subparsers(dest="command", help="Commands")

    # Build command
    build_parser = subparsers.add_parser(
        "build",
        help="Build bitcoind at a commit",
        description="Build bitcoind binary from a git commit. "
        "Optionally provide a name suffix: COMMIT:NAME",
    )
    build_parser.add_argument(
        "commit",
        metavar="COMMIT[:NAME]",
        help="Commit to build. Format: COMMIT or COMMIT:NAME (e.g., HEAD:pr, abc123:test)",
    )
    build_parser.add_argument(
        "-o",
        "--output-dir",
        metavar="PATH",
        help="Where to store binaries (default: ./binaries)",
    )
    build_parser.add_argument(
        "--skip-existing",
        action="store_true",
        help="Skip build if binary already exists",
    )
    build_parser.set_defaults(func=cmd_build)

    # Analyze command
    analyze_parser = subparsers.add_parser(
        "analyze", help="Generate plots from debug.log"
    )
    analyze_parser.add_argument("commit", help="Commit hash (for naming)")
    analyze_parser.add_argument("log_file", help="Path to debug.log")
    analyze_parser.add_argument(
        "--output-dir",
        default="./plots",
        metavar="PATH",
        help="Output directory for plots",
    )
    analyze_parser.set_defaults(func=cmd_analyze)

    # Report command
    report_parser = subparsers.add_parser(
        "report",
        help="Generate HTML report",
        description="Generate HTML report from benchmark results. "
        "Use --experiment-output for CI experiment reports.",
    )
    report_parser.add_argument(
        "input_dir",
        nargs="?",
        help="Directory with results.json (for single-run mode)",
    )
    report_parser.add_argument("output_dir", help="Output directory for report")
    report_parser.add_argument(
        "--title",
        help="Report title",
    )
    # CI experiment options
    report_parser.add_argument(
        "--experiment-output",
        metavar="PATH",
        help="Experiment output directory containing artifacts.json",
    )
    report_parser.add_argument(
        "--pr-number",
        metavar="N",
        help="PR number (for CI reports)",
    )
    report_parser.add_argument(
        "--run-id",
        metavar="ID",
        help="Run ID (for CI reports)",
    )
    report_parser.add_argument(
        "--update-index",
        action="store_true",
        help="Update main index.html (for CI reports)",
    )
    report_parser.add_argument(
        "--nightly-history",
        metavar="PATH",
        help="Path to nightly-history.json for comparison against nightly baseline",
    )
    report_parser.add_argument(
        "--commit",
        metavar="SHA",
        help="PR commit hash (for chart display)",
    )
    report_parser.set_defaults(func=cmd_report)

    # Nightly command
    nightly_parser = subparsers.add_parser(
        "nightly",
        help="Manage nightly benchmark history and charts",
        description="Commands for managing nightly benchmark results history "
        "and generating the historical trend chart.",
    )
    nightly_parser.add_argument(
        "--history-file",
        default="nightly-history.json",
        metavar="PATH",
        help="Path to nightly history JSON file (default: nightly-history.json)",
    )
    nightly_subparsers = nightly_parser.add_subparsers(
        dest="nightly_command", help="Nightly commands"
    )

    # nightly append
    nightly_append = nightly_subparsers.add_parser(
        "append",
        help="Append a result to the nightly history",
        description="Parse a hyperfine results.json file and append the result "
        "to the nightly history JSON file. Machine specs are automatically captured.",
    )
    nightly_append.add_argument(
        "results_file",
        help="Path to hyperfine results.json file",
    )
    nightly_append.add_argument(
        "commit",
        help="Git commit hash",
    )
    nightly_append.add_argument(
        "dbcache",
        type=int,
        help="DB cache size in MB (450 or 32000)",
    )
    nightly_append.add_argument(
        "--date",
        metavar="YYYY-MM-DD",
        help="Date for this result (default: today)",
    )
    nightly_append.add_argument(
        "--experiment-config",
        metavar="PATH",
        help="Experiment TOML file to store with results",
    )
    nightly_append.add_argument(
        "--profile-name",
        metavar="NAME",
        help="Profile name from --experiment-config",
    )
    nightly_append.add_argument(
        "--instrumentation",
        default="uninstrumented",
        choices=["uninstrumented", "instrumented"],
        help="Instrumentation mode (default: uninstrumented)",
    )
    nightly_append.add_argument(
        "--machine-specs",
        metavar="PATH",
        help="Path to pre-captured machine specs JSON (default: detect current machine)",
    )
    nightly_append.add_argument(
        "--run-date",
        metavar="YYYY-MM-DD",
        help="Date when benchmark was executed (default: today). Stored for reference.",
    )
    nightly_append.add_argument(
        "--trigger",
        default="scheduled",
        choices=["scheduled", "manual"],
        help="How the benchmark was triggered (default: scheduled). "
        "Scheduled runs dedup by commit; manual runs are always kept.",
    )

    # nightly append-experiment
    nightly_append_experiment = nightly_subparsers.add_parser(
        "append-experiment",
        help="Append experiment output to the nightly history",
        description="Read artifacts.json from an experiment output directory and "
        "append every run result to the nightly history.",
    )
    nightly_append_experiment.add_argument(
        "experiment_dir",
        help="Experiment output directory containing artifacts.json",
    )
    nightly_append_experiment.add_argument(
        "commit",
        help="Git commit hash",
    )
    nightly_append_experiment.add_argument(
        "--date",
        metavar="YYYY-MM-DD",
        help="Date for this result (default: today)",
    )
    nightly_append_experiment.add_argument(
        "--machine-specs",
        metavar="PATH",
        help="Path to pre-captured machine specs JSON (default: detect current machine)",
    )
    nightly_append_experiment.add_argument(
        "--run-date",
        metavar="YYYY-MM-DD",
        help="Date when benchmark was executed (default: today). Stored for reference.",
    )
    nightly_append_experiment.add_argument(
        "--trigger",
        default="scheduled",
        choices=["scheduled", "manual"],
        help="How the benchmark was triggered (default: scheduled). "
        "Scheduled runs dedup by commit; manual runs are always kept.",
    )

    # nightly chart
    nightly_chart = nightly_subparsers.add_parser(
        "chart",
        help="Generate the nightly trend chart HTML",
        description="Generate an HTML page with an interactive Plotly chart "
        "showing nightly benchmark results over time.",
    )
    nightly_chart.add_argument(
        "output_file",
        help="Path to write the chart HTML (typically index.html)",
    )

    nightly_parser.set_defaults(func=cmd_nightly)

    # Experiment command
    experiment_parser = subparsers.add_parser(
        "experiment",
        help="Run declarative benchmark experiments",
        description="Commands for running experiment manifests.",
    )
    experiment_subparsers = experiment_parser.add_subparsers(
        dest="experiment_command", help="Experiment commands"
    )
    experiment_run = experiment_subparsers.add_parser(
        "run",
        help="Run an experiment manifest",
        description="Build subjects, run profiles, and derive comparison artifacts.",
    )
    experiment_run.add_argument(
        "manifest",
        help="Path to experiment TOML file",
    )
    experiment_run.add_argument(
        "--datadir",
        metavar="PATH",
        help="Source datadir with blockchain snapshot (omit for fresh sync)",
    )
    experiment_run.add_argument(
        "--tmp-dir",
        metavar="PATH",
        help="Base temp directory for benchmark datadirs",
    )
    experiment_run.add_argument(
        "--subject-name",
        action="append",
        metavar="NAME",
        help="Run only a named subject from the manifest (repeatable)",
    )
    experiment_run.add_argument(
        "--profile-name",
        action="append",
        metavar="NAME",
        help="Run only a named profile from the manifest (repeatable)",
    )
    experiment_run.add_argument(
        "-o",
        "--output-dir",
        metavar="PATH",
        help="Output directory for experiment artifacts",
    )
    experiment_run.add_argument(
        "--binaries-dir",
        metavar="PATH",
        help="Where to store or find built commit binaries",
    )
    experiment_run.add_argument(
        "--skip-existing",
        action="store_true",
        help="Skip commit builds if the subject binary already exists",
    )
    experiment_run.add_argument(
        "--no-cache-drop",
        action="store_true",
        help="Skip cache dropping between runs",
    )
    experiment_run.set_defaults(func=cmd_experiment)

    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        return 1

    if not hasattr(args, "func"):
        parser.print_help()
        return 1

    return args.func(args)


if __name__ == "__main__":
    sys.exit(main())