Run the in-run analyses during algorithm warm-up, skipping only the speed sampling

Author: jhonabreul

Engine/Results/Analysis/InRunResultsAnalyzer.cs

@@ -44,18 +44,16 @@ public class InRunResultsAnalyzer : ResultsAnalyzer
 
         private readonly AlgorithmSpeedTracker _speed = new();
 
-        private readonly QCAlgorithm _algorithm;
-
         /// <summary>
         /// The number of order events already consumed by previous runs. The order events
-        /// in the result passed to <see cref="Run(Result, IReadOnlyList{string}, int, int)"/>
+        /// in the result passed to <see cref="Run(Result, IReadOnlyList{string}, System.Nullable{AlgorithmSpeedSample}, int, int)"/>
         /// are expected to start at this position.
         /// </summary>
         public int OrderEventsPosition { get; private set; }
 
         /// <summary>
         /// The number of log entries already consumed by previous runs. The logs passed to
-        /// <see cref="Run(Result, IReadOnlyList{string}, int, int)"/> are expected to start
+        /// <see cref="Run(Result, IReadOnlyList{string}, System.Nullable{AlgorithmSpeedSample}, int, int)"/> are expected to start
         /// at this position.
         /// </summary>
         public int LogsPosition { get; private set; }
@@ -76,14 +74,13 @@ public class InRunResultsAnalyzer : ResultsAnalyzer
         /// <summary>
         /// Initializes a new instance of the <see cref="InRunResultsAnalyzer"/> class.
         /// The instance is expected to be kept alive for the duration of the backtest,
-        /// receiving fresh data on each <see cref="Run(Result, IReadOnlyList{string}, int, int)"/> call.
+        /// receiving fresh data on each <see cref="Run(Result, IReadOnlyList{string}, System.Nullable{AlgorithmSpeedSample}, int, int)"/> call.
         /// </summary>
         /// <param name="algorithm">The algorithm instance used for history requests and settings.</param>
         /// <param name="language">The programming language the algorithm is written in.</param>
         public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language)
             : base(null, algorithm, language, null)
         {
-            _algorithm = algorithm;
         }
 
         /// <summary>
@@ -94,25 +91,17 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language)
         /// Findings from analyses scanning the order event and log streams are accumulated
         /// (first sample kept, counts totaled), while findings from state-based analyses are
         /// replaced on every run.
-        /// While the algorithm is warming up, nothing is analyzed or consumed and no findings are reported.
         /// </summary>
         /// <param name="result">A snapshot of the current intermediate backtest result, holding only new order events.</param>
         /// <param name="logs">The log lines produced since the previous run.</param>
-        /// <param name="speedSample">A sample of the engine speed counters for the algorithm speed analysis, when available.</param>
+        /// <param name="speedSample">A sample of the engine speed counters for the algorithm speed analysis.
+        /// Null when the counters should not be sampled, like while the algorithm warms up.</param>
         /// <param name="timeLimitSeconds">Wall-clock seconds allowed for the full chain before early exit.</param>
         /// <param name="maxFailedAnalyses">Maximum number of failing analyses to return.</param>
         /// <returns>The accumulated findings, ranked by analysis weight.</returns>
         public IReadOnlyList<QuantConnect.Analysis> Run(Result result, IReadOnlyList<string> logs, AlgorithmSpeedSample? speedSample = null,
             int timeLimitSeconds = 1, int maxFailedAnalyses = 10)
         {
-            // Nothing is analyzed during the algorithm warm-up period: trading hasn't started, and
-            // sampling the warm-up pace would skew the speed metrics. The positions don't advance,
-            // so the order events and logs produced during warm-up are analyzed by the first run after it ends.
-            if (_algorithm?.IsWarmingUp == true)
-            {
-                return [];
-            }
-
             SetAnalysisData(result, logs);
             if (speedSample.HasValue)
             {

Engine/Results/BacktestingResultHandler.cs

@@ -464,13 +464,6 @@ protected void SendFinalResult()
         {
             try
             {
-                // Nothing to analyze until trading starts: skip building the snapshot altogether.
-                // The analyzer catches up on the warm-up order events and logs on the first run after warm-up ends.
-                if (Algorithm.IsWarmingUp)
-                {
-                    return null;
-                }
-
                 if (AlgorithmInstance == null)
                 {
                     return null;
@@ -485,13 +478,18 @@ protected void SendFinalResult()
 
                 _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language);
 
-                // Sample the engine speed counters for the algorithm speed analysis
-                var speedSample = new AlgorithmSpeedSample(
-                    DateTime.UtcNow - StartTime,
-                    PerformanceTrackingTool?.DataPoints ?? 0,
-                    PerformanceTrackingTool?.HistoryDataPoints ?? 0,
-                    _progressMonitor?.ProcessedDays ?? 0,
-                    _progressMonitor?.TotalDays ?? 0);
+                // Sample the engine speed counters for the algorithm speed analysis, but not while the
+                // algorithm is warming up: the warm-up pace would skew the speed metrics. The analyses
+                // themselves do run during warm-up, so conditions like orders submitted while warming up
+                // surface without waiting for warm-up to end
+                AlgorithmSpeedSample? speedSample = Algorithm.IsWarmingUp
+                    ? null
+                    : new AlgorithmSpeedSample(
+                        DateTime.UtcNow - StartTime,
+                        PerformanceTrackingTool?.DataPoints ?? 0,
+                        PerformanceTrackingTool?.HistoryDataPoints ?? 0,
+                        _progressMonitor?.ProcessedDays ?? 0,
+                        _progressMonitor?.TotalDays ?? 0);
 
                 // Only the order events and logs produced since the previous run are analyzed,
                 // the analyzer accumulates findings across runs

Tests/Engine/Results/InRunResultsAnalyzerTests.cs

@@ -19,7 +19,6 @@
 using System.Linq;
 using System.Threading;
 using NUnit.Framework;
-using QuantConnect.Algorithm;
 using QuantConnect.Lean.Engine.Results.Analysis;
 using QuantConnect.Lean.Engine.Results.Analysis.Analyses;
 using QuantConnect.Orders;
@@ -179,33 +178,22 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName()
         }
 
         [Test]
-        public void NothingIsAnalyzedOrConsumedDuringAlgorithmWarmUp()
+        public void SpeedSamplesAreTrackedOnlyWhenProvided()
         {
-            // A fresh algorithm is warming up until the engine flips it
-            var algorithm = new QCAlgorithm();
-            var ran = false;
-            var fake = new FakeAnalysisA(10)
-            {
-                Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 1),
-                OnRun = () => ran = true
-            };
-            var analyzer = new TestInRunResultsAnalyzer(algorithm, fake);
-
-            var findings = analyzer.Run(MakeResult(2), new[] { "log" });
+            AlgorithmSpeedTracker speed = null;
+            var fake = new FakeAnalysisA(10) { OnParameters = parameters => speed = parameters.Speed };
+            var analyzer = new TestInRunResultsAnalyzer(fake);
 
-            Assert.IsFalse(ran);
-            Assert.IsEmpty(findings);
-            Assert.AreEqual(0, analyzer.OrderEventsPosition);
-            Assert.AreEqual(0, analyzer.LogsPosition);
+            analyzer.Run(MakeResult(1), new[] { "log" });
+            Assert.IsNotNull(speed);
+            Assert.AreEqual(0, speed.SampleCount);
 
-            // Once warm-up finishes, the analysis catches up on the unconsumed order events and logs
-            algorithm.SetFinishedWarmingUp();
-            findings = analyzer.Run(MakeResult(2), new[] { "log" });
+            analyzer.Run(MakeResult(1), new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10));
+            Assert.AreEqual(1, speed.SampleCount);
 
-            Assert.IsTrue(ran);
-            Assert.AreEqual("sample", findings.Single().Sample);
-            Assert.AreEqual(2, analyzer.OrderEventsPosition);
-            Assert.AreEqual(1, analyzer.LogsPosition);
+            // No sample provided (e.g. while the algorithm warms up): the tracker is left untouched
+            analyzer.Run(MakeResult(1), new[] { "log" });
+            Assert.AreEqual(1, speed.SampleCount);
         }
 
         [Test]
@@ -257,12 +245,7 @@ private class TestInRunResultsAnalyzer : InRunResultsAnalyzer
             private readonly IReadOnlyCollection<BaseResultsAnalysis> _analyses;
 
             public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses)
-                : this(null, analyses)
-            {
-            }
-
-            public TestInRunResultsAnalyzer(QCAlgorithm algorithm, params BaseResultsAnalysis[] analyses)
-                : base(algorithm, Language.CSharp)
+                : base(null, Language.CSharp)
             {
                 _analyses = analyses;
             }
@@ -282,6 +265,8 @@ private class FakeAnalysis : BaseResultsAnalysis
 
             public Action OnRun { get; set; }
 
+            public Action<ResultsAnalysisRunParameters> OnParameters { get; set; }
+
             protected FakeAnalysis(int weight)
             {
                 _weight = weight;
@@ -290,6 +275,7 @@ protected FakeAnalysis(int weight)
             public override IReadOnlyList<QuantConnect.Analysis> Run(ResultsAnalysisRunParameters parameters)
             {
                 OnRun?.Invoke();
+                OnParameters?.Invoke(parameters);
                 return Findings();
             }
         }