From a54061f757e6a5825a3b445508e0b45dc5cf8e23 Mon Sep 17 00:00:00 2001 From: Alex Hope-O'Connor Date: Thu, 7 May 2026 05:22:47 +1000 Subject: [PATCH] Release v0.0.4: BOM spatial map scraping and radar cache reliability. Align selectors and workflow with the step-through scrubber, share scrub display timestamp parsing via IScrubDisplayTimestampParser, fix pause post-click label verification, and stop cancelling background cache updates when the HTTP request ends. Version bumped to 0.0.4. --- BomLocalService.csproj | 1 + Models/JavaScriptTemplatesConfig.cs | 3 +- Models/TextPatternsConfig.cs | 2 +- Program.cs | 1 - Services/BomRadarService.cs | 5 +- .../IScrubDisplayTimestampParser.cs | 27 ++ .../Scraping/ScrubDisplayTimestampParser.cs | 215 ++++++++++++++++ .../Steps/Capture/CaptureFramesStep.cs | 238 +----------------- Services/Scraping/Steps/Map/PauseRadarStep.cs | 39 ++- .../Steps/Map/ResetToFirstFrameStep.cs | 152 ++++++----- .../Steps/Metadata/ExtractMetadataStep.cs | 66 +++-- .../Workflows/RadarScrapingWorkflow.cs | 2 +- Services/SelectorService.cs | 14 +- appsettings.json | 25 +- 14 files changed, 462 insertions(+), 328 deletions(-) create mode 100644 Services/Interfaces/IScrubDisplayTimestampParser.cs create mode 100644 Services/Scraping/ScrubDisplayTimestampParser.cs diff --git a/BomLocalService.csproj b/BomLocalService.csproj index 084858f..24d357b 100644 --- a/BomLocalService.csproj +++ b/BomLocalService.csproj @@ -4,6 +4,7 @@ net9.0 enable enable + 0.0.4 diff --git a/Models/JavaScriptTemplatesConfig.cs b/Models/JavaScriptTemplatesConfig.cs index 883e436..6b96096 100644 --- a/Models/JavaScriptTemplatesConfig.cs +++ b/Models/JavaScriptTemplatesConfig.cs @@ -53,6 +53,7 @@ public class JavaScriptTemplatesConfig public string CheckActiveFrameSegment { get; set; } = @"() => { const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); + if (segments.length === 0) return true; const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; @@ -73,7 +74,7 @@ public class JavaScriptTemplatesConfig let minutes = null; // Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm"" - const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i); + const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]+),?\s+(\d{1,2}):(\d{2})\s*(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); diff --git a/Models/TextPatternsConfig.cs b/Models/TextPatternsConfig.cs index 1ab9077..75e10ba 100644 --- a/Models/TextPatternsConfig.cs +++ b/Models/TextPatternsConfig.cs @@ -6,7 +6,7 @@ namespace BomLocalService.Models; public class TextPatternsConfig { public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)"; - public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)"; + public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]+,?\s+\d{1,2}:\d{2}\s*(?:am|pm)"; public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago"; public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago"; public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago"; diff --git a/Program.cs b/Program.cs index f6d395d..58eb3e5 100644 --- a/Program.cs +++ b/Program.cs @@ -1,6 +1,5 @@ using BomLocalService.Extensions; using BomLocalService.Services.Interfaces; -using BomLocalService.Services.Scraping; var builder = WebApplication.CreateBuilder(args); diff --git a/Services/BomRadarService.cs b/Services/BomRadarService.cs index 6fa2a31..efeb11f 100644 --- a/Services/BomRadarService.cs +++ b/Services/BomRadarService.cs @@ -185,6 +185,9 @@ public class BomRadarService : IBomRadarService, IDisposable if (needsUpdate) { // Trigger async update (fire and forget) + // Do not pass the caller's cancellationToken to Task.Run: for HTTP that token is tied to + // RequestAborted and often fires as soon as the response is sent, so the update never starts. + // FetchAndCacheScreenshotAsync already uses CancellationToken.None for the scrape work. _ = Task.Run(async () => { try @@ -195,7 +198,7 @@ public class BomRadarService : IBomRadarService, IDisposable { _logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state); } - }, cancellationToken); + }); status.UpdateTriggered = true; status.Message = status.CacheExists diff --git a/Services/Interfaces/IScrubDisplayTimestampParser.cs b/Services/Interfaces/IScrubDisplayTimestampParser.cs new file mode 100644 index 0000000..c98ff37 --- /dev/null +++ b/Services/Interfaces/IScrubDisplayTimestampParser.cs @@ -0,0 +1,27 @@ +using BomLocalService.Services.Interfaces.Registration; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Parses the BOM scrubber time display label to UTC; shared by capture and reset steps. +/// +public interface IScrubDisplayTimestampParser : ISingletonService +{ + Task ReadUtcAsync( + IPage page, + ISelectorService selectorService, + ScrapingContext? context = null, + CancellationToken cancellationToken = default); + + bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc); + + Task WaitForDisplayLabelChangeAsync( + IPage page, + ISelectorService selectorService, + DateTime currentTimestamp, + ScrapingContext? context = null, + int maxWaitMs = 5000, + CancellationToken cancellationToken = default); +} diff --git a/Services/Scraping/ScrubDisplayTimestampParser.cs b/Services/Scraping/ScrubDisplayTimestampParser.cs new file mode 100644 index 0000000..5ca7a06 --- /dev/null +++ b/Services/Scraping/ScrubDisplayTimestampParser.cs @@ -0,0 +1,215 @@ +using System.Text.RegularExpressions; +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping; + +/// +/// Parses the BOM scrubber time label (.bom-scrub-display-label) to UTC. +/// Shared by capture and reset steps so behaviour stays in sync. +/// +public class ScrubDisplayTimestampParser : IScrubDisplayTimestampParser +{ + private readonly IConfiguration _configuration; + private readonly ScrapingSelectorsConfig _selectors; + private readonly TextPatternsConfig _textPatterns; + private readonly JavaScriptTemplatesConfig _javaScriptTemplates; + private readonly ILogger _logger; + + public ScrubDisplayTimestampParser( + IConfiguration configuration, + ILogger logger) + { + _configuration = configuration; + _selectors = configuration.GetSection("Scraping:Selectors").Get() ?? new(); + _textPatterns = configuration.GetSection("Scraping:TextPatterns").Get() ?? new(); + _javaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get() ?? new(); + _logger = logger; + } + + public async Task ReadUtcAsync( + IPage page, + ISelectorService selectorService, + ScrapingContext? context = null, + CancellationToken cancellationToken = default) + { + _ = cancellationToken; + try + { + var timeLabelLocator = selectorService.GetLocator(page, _selectors.TimeDisplayLabel); + var timeLabel = await timeLabelLocator.TextContentAsync(); + if (string.IsNullOrEmpty(timeLabel)) + { + _logger.LogDebug("Time display label is empty"); + return null; + } + + var trimmedLabel = timeLabel.Trim(); + _logger.LogDebug("Scrub display label: '{Label}'", trimmedLabel); + + var timestampMatch = Regex.Match(trimmedLabel, _textPatterns.TimestampPattern, RegexOptions.IgnoreCase); + if (!timestampMatch.Success) + { + _logger.LogWarning( + "Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", + trimmedLabel, + _textPatterns.TimestampPattern); + return null; + } + + var timestampStr = timestampMatch.Groups[0].Value; + + string? detectedTimezone = null; + if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) + detectedTimezone = "AEDT"; + else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase)) + detectedTimezone = "AEST"; + + if (detectedTimezone == null && context != null) + { + try + { + var metadataText = await page.EvaluateAsync(_javaScriptTemplates.ExtractWeatherMetadata); + if (!string.IsNullOrEmpty(metadataText)) + { + if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) + detectedTimezone = "AEDT"; + else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase)) + detectedTimezone = "AEST"; + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone"); + } + } + + if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc)) + return frameTimestampUtc; + + _logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr); + return null; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to extract timestamp from display label"); + return null; + } + } + + public bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc) + { + timestampUtc = DateTime.MinValue; + + try + { + TimeZoneInfo timeZoneInfo; + + if (!string.IsNullOrEmpty(timezoneAbbreviation)) + { + if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase)) + timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane"); + else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) + timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney"); + else + { + var timezone = _configuration.GetValue("Timezone"); + if (string.IsNullOrEmpty(timezone)) + return false; + timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone); + } + } + else + { + var timezone = _configuration.GetValue("Timezone"); + if (string.IsNullOrEmpty(timezone)) + return false; + timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone); + } + + timestampStr = Regex.Replace(timestampStr.Trim(), @"\s+", " "); + + var formats = new[] + { + "dddd d MMM, h:mm tt", + "d MMM, h:mm tt", + "dddd d MMM, hh:mm tt", + "d MMM, hh:mm tt", + "dddd dd MMM, h:mm tt", + "dd MMM, h:mm tt", + "dddd d MMM, h:mmtt", + "d MMM, h:mmtt", + "dddd d MMM, hh:mmtt", + "d MMM, hh:mmtt", + "dddd dd MMM, h:mmtt", + "dd MMM, h:mmtt", + "dddd dd MMM, hh:mmtt", + "dd MMM, hh:mmtt" + }; + + var culture = new System.Globalization.CultureInfo("en-AU"); + DateTime localTime = default; + var parsed = false; + + foreach (var format in formats) + { + if (DateTime.TryParseExact(timestampStr, format, culture, + System.Globalization.DateTimeStyles.None, out localTime)) + { + parsed = true; + break; + } + } + + if (!parsed) + return false; + + if (localTime.Year == 1) + { + localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day, + localTime.Hour, localTime.Minute, localTime.Second); + } + + timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr); + return false; + } + } + + public async Task WaitForDisplayLabelChangeAsync( + IPage page, + ISelectorService selectorService, + DateTime currentTimestamp, + ScrapingContext? context = null, + int maxWaitMs = 5000, + CancellationToken cancellationToken = default) + { + _ = cancellationToken; + try + { + var startTime = DateTime.UtcNow; + while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs) + { + var newTimestamp = await ReadUtcAsync(page, selectorService, context, cancellationToken); + if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp) + return; + + await page.WaitForTimeoutAsync(200); + } + + _logger.LogDebug( + "Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms", + currentTimestamp, + maxWaitMs); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Error waiting for display label to change"); + } + } +} diff --git a/Services/Scraping/Steps/Capture/CaptureFramesStep.cs b/Services/Scraping/Steps/Capture/CaptureFramesStep.cs index 921035c..f5857c6 100644 --- a/Services/Scraping/Steps/Capture/CaptureFramesStep.cs +++ b/Services/Scraping/Steps/Capture/CaptureFramesStep.cs @@ -3,13 +3,13 @@ using BomLocalService.Services.Interfaces; using BomLocalService.Services.Scraping; using BomLocalService.Utilities; using Microsoft.Playwright; -using System.Text.RegularExpressions; namespace BomLocalService.Services.Scraping.Steps.Capture; public class CaptureFramesStep : BaseScrapingStep { private readonly ICacheService _cacheService; + private readonly IScrubDisplayTimestampParser _scrubDisplayTime; private readonly int _tileRenderWaitMs; private readonly ScreenshotCropConfig _cropConfig; @@ -21,10 +21,12 @@ public class CaptureFramesStep : BaseScrapingStep ISelectorService selectorService, IDebugService debugService, IConfiguration configuration, - ICacheService cacheService) + ICacheService cacheService, + IScrubDisplayTimestampParser scrubDisplayTime) : base(logger, selectorService, debugService, configuration) { _cacheService = cacheService; + _scrubDisplayTime = scrubDisplayTime; var tileRenderWaitMsConfig = configuration.GetValue("Screenshot:TileRenderWaitMs"); if (!tileRenderWaitMsConfig.HasValue) @@ -91,12 +93,12 @@ public class CaptureFramesStep : BaseScrapingStep await context.Page.WaitForTimeoutAsync(300); // Try extracting timestamp with a retry in case the label is still updating - var frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); + var frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); if (frameTimestamp == null) { // Retry once after a short wait in case label was updating await context.Page.WaitForTimeoutAsync(200); - frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); + frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); } // Fallback: calculate expected timestamp from observation time and frame index if we can't parse it @@ -112,8 +114,9 @@ public class CaptureFramesStep : BaseScrapingStep { Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same timestamp ({Timestamp}) as previous frame. Waiting for display to update...", Name, frameIndex, frameTimestamp); - await WaitForDisplayLabelToChangeAsync(context.Page, previousTimestamp.Value, context); - frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); + await _scrubDisplayTime.WaitForDisplayLabelChangeAsync( + context.Page, SelectorService, previousTimestamp.Value, context, cancellationToken: cancellationToken); + frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); if (frameTimestamp == null || frameTimestamp == previousTimestamp.Value) { if (context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count) @@ -161,13 +164,14 @@ public class CaptureFramesStep : BaseScrapingStep { await DismissModalOverlaysAsync(context.Page); - var currentTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); + var currentTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true }); if (currentTimestamp.HasValue) { - await WaitForDisplayLabelToChangeAsync(context.Page, currentTimestamp.Value); + await _scrubDisplayTime.WaitForDisplayLabelChangeAsync( + context.Page, SelectorService, currentTimestamp.Value, context, cancellationToken: cancellationToken); } else { @@ -198,224 +202,6 @@ public class CaptureFramesStep : BaseScrapingStep } } - /// - /// Extracts the UTC timestamp from the frame display label - /// - private async Task ExtractTimestampFromDisplayAsync(IPage page, ScrapingContext? context = null) - { - try - { - var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel); - var timeLabel = await timeLabelLocator.TextContentAsync(); - if (string.IsNullOrEmpty(timeLabel)) - { - Logger.LogDebug("Time display label is empty"); - return null; - } - - var trimmedLabel = timeLabel.Trim(); - Logger.LogInformation("Extracting timestamp from display label: '{Label}'", trimmedLabel); - - // Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" - Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern); - var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase); - if (timestampMatch.Success) - { - var timestampStr = timestampMatch.Groups[0].Value; - - // Check for timezone abbreviation in the full label (not just the matched timestamp) - // BOM website may display timezone elsewhere in the label text - string? detectedTimezone = null; - if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) - { - detectedTimezone = "AEDT"; - } - else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase)) - { - detectedTimezone = "AEST"; - } - - // Fallback: If timezone not found in frame label, try to extract it from metadata - // The metadata has the timezone (e.g., "9:40 pm AEST"), so we can use that for frames too - if (detectedTimezone == null && context != null) - { - try - { - var metadataText = await page.EvaluateAsync(JavaScriptTemplates.ExtractWeatherMetadata); - if (!string.IsNullOrEmpty(metadataText)) - { - if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) - { - detectedTimezone = "AEDT"; - Logger.LogDebug("Detected timezone AEDT from metadata text"); - } - else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase)) - { - detectedTimezone = "AEST"; - Logger.LogDebug("Detected timezone AEST from metadata text"); - } - } - } - catch (Exception ex) - { - Logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone"); - } - } - - Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}' (detected timezone: {Tz})", - timestampStr, trimmedLabel, detectedTimezone ?? "none"); - - if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc)) - { - Logger.LogInformation("Successfully parsed frame timestamp: {Timestamp} UTC", frameTimestampUtc); - return frameTimestampUtc; - } - else - { - Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr); - } - } - else - { - Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern); - } - - return null; - } - catch (Exception ex) - { - Logger.LogDebug(ex, "Failed to extract timestamp from display label"); - return null; - } - } - - private bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc) - { - timestampUtc = DateTime.MinValue; - - try - { - // Determine timezone based on detected abbreviation or configured default - TimeZoneInfo timeZoneInfo; - - if (!string.IsNullOrEmpty(timezoneAbbreviation)) - { - // Map timezone abbreviations to actual timezones (same logic as TimeParsingService) - // - "AEST" = UTC+10 (Australian Eastern Standard Time) - Brisbane year-round, Sydney/Melbourne in winter - // - "AEDT" = UTC+11 (Australian Eastern Daylight Time) - Sydney/Melbourne in summer (Oct-Apr), never Brisbane - if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase)) - { - timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane"); - Logger.LogDebug("Using Brisbane timezone (AEST, UTC+10) for frame timestamp"); - } - else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) - { - timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney"); - Logger.LogDebug("Using Sydney timezone (AEDT, UTC+11) for frame timestamp"); - } - else - { - // Fallback to configured timezone if abbreviation is unrecognized - var timezone = Configuration.GetValue("Timezone"); - if (string.IsNullOrEmpty(timezone)) - { - Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly"); - return false; - } - timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone); - Logger.LogDebug("Using configured timezone '{Timezone}' for frame timestamp", timezone); - } - } - else - { - // No timezone detected - use configured default - var timezone = Configuration.GetValue("Timezone"); - if (string.IsNullOrEmpty(timezone)) - { - Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly"); - return false; - } - timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone); - Logger.LogDebug("No timezone abbreviation detected, using configured timezone '{Timezone}' for frame timestamp", timezone); - } - - // Try common Australian date formats - // Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" - var formats = new[] - { - "dddd d MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm - "d MMM, h:mm tt", // 17 Dec, 11:05 pm - "dddd d MMM, hh:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero) - "d MMM, hh:mm tt", // 17 Dec, 11:05 pm (with leading zero) - "dddd dd MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero day) - "dd MMM, h:mm tt" // 17 Dec, 11:05 pm (with leading zero day) - }; - - var culture = new System.Globalization.CultureInfo("en-AU"); - DateTime localTime = default; - bool parsed = false; - - foreach (var format in formats) - { - if (DateTime.TryParseExact(timestampStr, format, culture, - System.Globalization.DateTimeStyles.None, out localTime)) - { - parsed = true; - break; - } - } - - if (!parsed) - { - return false; - } - - // If year is not specified, assume current year - if (localTime.Year == 1) - { - localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day, - localTime.Hour, localTime.Minute, localTime.Second); - } - - // The parsed timestamp includes both date and time (e.g., "23 Dec, 11:45 pm") - // localTime already contains the complete date and time from the parsed string - // We treat it as being in the target timezone (DateTimeKind.Unspecified), then convert to UTC - timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo); - - Logger.LogDebug("Parsed frame timestamp '{TimestampStr}' with timezone '{TzAbbrev}' as {LocalTime} local ({UtcTime} UTC)", - timestampStr, timezoneAbbreviation ?? "default", localTime, timestampUtc); - - return true; - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr); - return false; - } - } - - private async Task WaitForDisplayLabelToChangeAsync(IPage page, DateTime currentTimestamp, ScrapingContext? context = null, int maxWaitMs = 5000) - { - try - { - var startTime = DateTime.UtcNow; - while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs) - { - var newTimestamp = await ExtractTimestampFromDisplayAsync(page, context); - if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp) - { - return; - } - await page.WaitForTimeoutAsync(200); - } - Logger.LogDebug("Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms", currentTimestamp, maxWaitMs); - } - catch (Exception ex) - { - Logger.LogDebug(ex, "Error waiting for display label to change"); - } - } - private async Task DismissModalOverlaysAsync(IPage page) { try diff --git a/Services/Scraping/Steps/Map/PauseRadarStep.cs b/Services/Scraping/Steps/Map/PauseRadarStep.cs index fb0d681..3063729 100644 --- a/Services/Scraping/Steps/Map/PauseRadarStep.cs +++ b/Services/Scraping/Steps/Map/PauseRadarStep.cs @@ -31,16 +31,43 @@ public class PauseRadarStep : BaseScrapingStep var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton); await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); - - var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); - if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true) + + // New spatial map often uses one step-through control; label may be on the button or in .bom-scrub-action__label + string? buttonLabel = null; + try + { + var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First; + if (await labelLoc.CountAsync() > 0) + buttonLabel = (await labelLoc.TextContentAsync())?.Trim(); + } + catch + { + /* use inner text */ + } + + buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim(); + + if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true) { Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name); await playPauseButton.ClickAsync(); await context.Page.WaitForTimeoutAsync(300); - - buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); - if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true) + + // Clear so post-click read always runs; otherwise stale "Pause" skips the ??= InnerText fallback when the label child is missing + buttonLabel = null; + try + { + var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First; + if (await labelLoc.CountAsync() > 0) + buttonLabel = (await labelLoc.TextContentAsync())?.Trim(); + } + catch + { + buttonLabel = null; + } + + buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim(); + if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true) { Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name); } diff --git a/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs b/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs index 9a99a48..618b693 100644 --- a/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs +++ b/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs @@ -1,99 +1,134 @@ +using BomLocalService.Models; using BomLocalService.Services.Interfaces; using BomLocalService.Services.Scraping; +using BomLocalService.Utilities; using Microsoft.Playwright; namespace BomLocalService.Services.Scraping.Steps.Map; +/// +/// Aligns the radar scrubber to the oldest frame (historical frame 0) without clicking timeline segments. +/// The BOM UI now often exposes a single step-through control; segment pills and bom-scrub-tl are unreliable. +/// Strategy: while paused, read from the display label, step forward +/// through one full loop to find the minimum time, then step until that time shows again. +/// public class ResetToFirstFrameStep : BaseScrapingStep { + private readonly IScrubDisplayTimestampParser _scrubDisplayTime; + public override string Name => "ResetToFirstFrame"; - public override string[] Prerequisites => new[] { "PauseRadar" }; - + public override string[] Prerequisites => new[] { "ExtractMetadata" }; + public ResetToFirstFrameStep( ILogger logger, ISelectorService selectorService, IDebugService debugService, - IConfiguration configuration) + IConfiguration configuration, + IScrubDisplayTimestampParser scrubDisplayTime) : base(logger, selectorService, debugService, configuration) { + _scrubDisplayTime = scrubDisplayTime; } - + public override bool CanExecute(ScrapingContext context) { - return context.CurrentState >= PageState.SlideshowPaused; + return context.CurrentState >= PageState.SlideshowPaused + && context.LastUpdatedInfo != null; } - + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) { try { - Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name); - - try + Logger.LogInformation("Step {Step}: Aligning to oldest frame via step-through + display label (no segment clicks)", Name); + + var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar); + var elementWaitMs = Configuration.GetValue("Scraping:Timeouts:ElementWait") ?? 10000; + + var stepBtn = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton); + await stepBtn.WaitForAsync(new LocatorWaitForOptions { State = WaitForSelectorState.Visible, Timeout = elementWaitMs }); + + // Phase 1 — sample one full loop of display times while paused; oldest = frame 0 target + var samples = new List(); + for (var i = 0; i < frameCount; i++) { - // Try JavaScript click first to avoid thumb interception - var clicked = await context.Page.EvaluateAsync(@"() => { - const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]'); - if (segment) { - segment.click(); - return true; + var t = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); + if (t.HasValue) + samples.Add(t.Value); + + if (i == frameCount - 1) + break; + + var beforeStep = t; + await stepBtn.ClickAsync(new LocatorClickOptions { Force = true }); + if (beforeStep.HasValue) + { + await _scrubDisplayTime.WaitForDisplayLabelChangeAsync( + context.Page, SelectorService, beforeStep.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken); + } + else + await context.Page.WaitForTimeoutAsync(800); + } + + DateTime? frame0Utc = null; + if (samples.Count > 0) + { + frame0Utc = samples.Min(); + Logger.LogInformation("Step {Step}: Oldest time in sampled loop (frame 0 target): {Utc:o} UTC", Name, frame0Utc); + } + else + { + Logger.LogWarning("Step {Step}: Could not read any display timestamps; tile timing may be wrong", Name); + } + + // Phase 2 — step until the display matches the oldest sample + if (frame0Utc.HasValue) + { + const double matchMinutes = 2.0; + for (var attempt = 0; attempt < frameCount + 4; attempt++) + { + var cur = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken); + if (cur.HasValue && Math.Abs((cur.Value - frame0Utc.Value).TotalMinutes) <= matchMinutes) + { + Logger.LogInformation("Step {Step}: Display matches frame 0 target (≤{MatchMinutes:F0} min)", Name, matchMinutes); + break; } - return false; - }"); - - if (clicked) - { - await context.Page.WaitForTimeoutAsync(500); - Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name); - } - else - { - // Fallback to locator click with force - var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First; - await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); - await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true }); - await context.Page.WaitForTimeoutAsync(500); - Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name); + + await stepBtn.ClickAsync(new LocatorClickOptions { Force = true }); + if (cur.HasValue) + { + await _scrubDisplayTime.WaitForDisplayLabelChangeAsync( + context.Page, SelectorService, cur.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken); + } + else + await context.Page.WaitForTimeoutAsync(800); } } - catch (Exception ex) - { - Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name); - } - - await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken); - - // Verify scrubber is at position 0 - Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name); + + await SaveDebugAsync(context, 9, "frame_0_aligned", cancellationToken); + + // Best-effort only: segment-based checks are unreliable on the new spatial map layout try { - var activeSegment = await context.Page.EvaluateAsync(JavaScriptTemplates.CheckActiveFrameSegment); - - if (activeSegment) - { - Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name); - } - else - { - Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name); - } + var onSegmentZero = await context.Page.EvaluateAsync(JavaScriptTemplates.CheckActiveFrameSegment); + if (onSegmentZero) + Logger.LogDebug("Step {Step}: Segment check reports frame 0 (informational)", Name); } catch (Exception ex) { - Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name); + Logger.LogDebug(ex, "Step {Step}: Segment position check skipped", Name); } - - await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken); - - // Wait for frame 0 tiles to fully render + + await SaveDebugAsync(context, 10, "scrubber_frame0", cancellationToken); + var tileRenderWaitMsConfig = Configuration.GetValue("Screenshot:TileRenderWaitMs"); if (!tileRenderWaitMsConfig.HasValue) { throw new InvalidOperationException("Screenshot:TileRenderWaitMs configuration is required. Set it in appsettings.json or via SCREENSHOT__TILERENDERWAITMS environment variable."); } - var tileRenderWaitMs = tileRenderWaitMsConfig.Value; - await context.Page.WaitForTimeoutAsync(tileRenderWaitMs); - + + await context.Page.WaitForTimeoutAsync(tileRenderWaitMsConfig.Value); + context.CurrentState = PageState.Frame0Selected; return ScrapingStepResult.Successful(); } @@ -105,4 +140,3 @@ public class ResetToFirstFrameStep : BaseScrapingStep } } } - diff --git a/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs b/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs index be1076e..7cabd9f 100644 --- a/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs +++ b/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs @@ -1,15 +1,18 @@ +using System.Text.Json; +using BomLocalService.Models; using BomLocalService.Services.Interfaces; using BomLocalService.Services.Scraping; +using BomLocalService.Utilities; namespace BomLocalService.Services.Scraping.Steps.Metadata; public class ExtractMetadataStep : BaseScrapingStep { private readonly ITimeParsingService _timeParsingService; - + public override string Name => "ExtractMetadata"; - public override string[] Prerequisites => new[] { "ResetToFirstFrame" }; - + public override string[] Prerequisites => new[] { "PauseRadar" }; + public ExtractMetadataStep( ILogger logger, ISelectorService selectorService, @@ -20,41 +23,67 @@ public class ExtractMetadataStep : BaseScrapingStep { _timeParsingService = timeParsingService; } - + public override bool CanExecute(ScrapingContext context) { return context.IsMapReady; } - + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) { try { Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name); - + var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page); context.LastUpdatedInfo = lastUpdatedInfo; - - // Extract frame info + + var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar); + var defaults = Enumerable.Range(0, frameCount) + .Select(i => (index: i, minutesAgo: 40 - i * 5)) + .ToList(); + try { - var frameInfo = await context.Page.EvaluateAsync(JavaScriptTemplates.ExtractFrameInfo); - var result = new List<(int index, int minutesAgo)>(); - for (int i = 0; i < 7; i++) + var element = await context.Page.EvaluateAsync(JavaScriptTemplates.ExtractFrameInfo); + if (element.ValueKind == JsonValueKind.Array && element.GetArrayLength() > 0) { - var minutesAgo = 40 - (i * 5); - result.Add((i, minutesAgo)); + var byIndex = new Dictionary(); + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + continue; + + var idx = item.GetProperty("index").GetInt32(); + if (item.TryGetProperty("minutesAgo", out var m) && m.ValueKind == JsonValueKind.Number) + byIndex[idx] = m.GetInt32(); + } + + if (byIndex.Count > 0) + { + context.FrameInfo = Enumerable.Range(0, frameCount) + .Select(i => (i, byIndex.TryGetValue(i, out var mm) ? mm : defaults[i].minutesAgo)) + .ToList(); + Logger.LogInformation("Step {Step}: Parsed frame offsets from segment aria-labels where available", Name); + } + else + { + context.FrameInfo = defaults; + Logger.LogInformation("Step {Step}: Segment times not parsed; using default ladder", Name); + } + } + else + { + context.FrameInfo = defaults; + Logger.LogInformation("Step {Step}: No segment array from page; using default ladder", Name); } - context.FrameInfo = result; } catch (Exception ex) { Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name); - context.FrameInfo = Enumerable.Range(0, 7) - .Select(i => (i, 40 - (i * 5))) - .ToList(); + context.FrameInfo = defaults; } - + return ScrapingStepResult.Successful(); } catch (Exception ex) @@ -65,4 +94,3 @@ public class ExtractMetadataStep : BaseScrapingStep } } } - diff --git a/Services/Scraping/Workflows/RadarScrapingWorkflow.cs b/Services/Scraping/Workflows/RadarScrapingWorkflow.cs index 5600b9a..53f021a 100644 --- a/Services/Scraping/Workflows/RadarScrapingWorkflow.cs +++ b/Services/Scraping/Workflows/RadarScrapingWorkflow.cs @@ -28,8 +28,8 @@ public class RadarScrapingWorkflow : IRadarScrapingWorkflow "ClickRadarLink", "WaitForMapReady", "PauseRadar", - "ResetToFirstFrame", "ExtractMetadata", + "ResetToFirstFrame", "CalculateMapBounds", "CaptureFrames" }; diff --git a/Services/SelectorService.cs b/Services/SelectorService.cs index e82ee04..bd87b7f 100644 --- a/Services/SelectorService.cs +++ b/Services/SelectorService.cs @@ -52,9 +52,17 @@ public class SelectorService : ISelectorService { throw new ArgumentException($"No selectors configured for {config.Name}"); } - - // Returns first selector as locator (for cases where we know it exists) - return page.Locator(config.Selectors[0]).First; + + // Match any configured selector (BOM occasionally renames test ids; list is tried in order) + var combined = page.Locator(config.Selectors[0]).First; + for (var i = 1; i < config.Selectors.Length; i++) + { + combined = combined.Or(page.Locator(config.Selectors[i]).First); + } + + // Or() unions alternatives; BOM sometimes renders duplicates (e.g. responsive + hidden). + // Pin to a single element so Playwright strict mode does not reject clicks. + return combined.First; } } diff --git a/appsettings.json b/appsettings.json index fa748d0..929b070 100644 --- a/appsettings.json +++ b/appsettings.json @@ -61,11 +61,9 @@ "Name": "Search Button", "Selectors": [ "button[data-testid='searchLabel']", - "button[aria-label='Search for a location']", "button.search-location__trigger-button", - "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))", - "button:has-text('Search for a location')", - "[data-testid='showSearchModal'] button" + "button[aria-label='Search for a location'].search-location__trigger-button", + "button.search-location__trigger-button.bom-button:has(span.bom-button__label:has-text('Search for a location'))" ], "TimeoutMs": 10000, "Required": true, @@ -137,7 +135,10 @@ }, "PlayPauseButton": { "Name": "Play Pause Button", - "Selectors": ["button[data-testid='bom-time-scrub-play-pause']"], + "Selectors": [ + "button[data-testid='bom-time-scrub-step-through']", + "button[data-testid='bom-time-scrub-play-pause']" + ], "TimeoutMs": 5000, "Required": true }, @@ -155,7 +156,11 @@ }, "StepForwardButton": { "Name": "Step Forward Button", - "Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"], + "Selectors": [ + "button[data-testid='bom-time-scrub-step-through']", + "button.bom-scrub-action--step-through", + "button[data-testid='bom-scrub-utils__right__step-forward']" + ], "TimeoutMs": 5000, "Required": true }, @@ -184,9 +189,9 @@ "ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }", "WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }", "WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }", - "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }", + "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); if (segments.length === 0) return true; const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }", "WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }", - "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }", + "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]+),?\\s+(\\d{1,2}):(\\d{2})\\s*(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }", "CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }", "CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }", "GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })", @@ -194,7 +199,7 @@ }, "TextPatterns": { "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)", - "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)", + "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]+,?\\s+\\d{1,2}:\\d{2}\\s*(?:am|pm)", "ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago", "ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago", "ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago", @@ -218,8 +223,8 @@ "ClickRadarLink": { "Enabled": true }, "WaitForMapReady": { "Enabled": true }, "PauseRadar": { "Enabled": true }, - "ResetToFirstFrame": { "Enabled": true }, "ExtractMetadata": { "Enabled": true }, + "ResetToFirstFrame": { "Enabled": true }, "CalculateMapBounds": { "Enabled": true }, "CaptureFrames": { "Enabled": true,