ElixirForge/bom-local-service
1
0
Fork 0
mirror of https://github.com/alexhopeoconnor/bom-local-service.git synced 2026-10-03 21:21:59 +10:00
Code Releases 7 Activity

Release v0.0.4: BOM spatial map scraping and radar cache reliability.

Browse Source 
Align selectors and workflow with the step-through scrubber, share scrub display
timestamp parsing via IScrubDisplayTimestampParser, fix pause post-click label
verification, and stop cancelling background cache updates when the HTTP request
ends. Version bumped to 0.0.4.
This commit is contained in:
alex
2026-05-07 05:23:18 +10:00
parent 0b6d266e79
commit a54061f757
14 changed files with 462 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BomLocalService.csproj
+1
View File
@@ -4,6 +4,7 @@
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<Version>0.0.4</Version>
</PropertyGroup>
<ItemGroup>
Models/JavaScriptTemplatesConfig.cs
+2 -1
View File
@@ -53,6 +53,7 @@ public class JavaScriptTemplatesConfig
public string CheckActiveFrameSegment { get; set; } = @"() => {
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
if (segments.length === 0) return true;
const activeSegment = segments.find(s => {
const style = window.getComputedStyle(s);
return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
@@ -73,7 +74,7 @@ public class JavaScriptTemplatesConfig
let minutes = null;
// Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm""
const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i);
const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]+),?\s+(\d{1,2}):(\d{2})\s*(am|pm)/i);
if (timestampMatch) {
try {
const day = parseInt(timestampMatch[1]);
Models/TextPatternsConfig.cs
+1 -1
View File
@@ -6,7 +6,7 @@ namespace BomLocalService.Models;
public class TextPatternsConfig
{
public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)";
public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)";
public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]+,?\s+\d{1,2}:\d{2}\s*(?:am|pm)";
public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago";
public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago";
public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago";
Program.cs
-1
View File
@@ -1,6 +1,5 @@
using BomLocalService.Extensions;
using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping;
var builder = WebApplication.CreateBuilder(args);
Services/BomRadarService.cs
+4 -1
View File
@@ -185,6 +185,9 @@ public class BomRadarService : IBomRadarService, IDisposable
if (needsUpdate)
{
// Trigger async update (fire and forget)
// Do not pass the caller's cancellationToken to Task.Run: for HTTP that token is tied to
// RequestAborted and often fires as soon as the response is sent, so the update never starts.
// FetchAndCacheScreenshotAsync already uses CancellationToken.None for the scrape work.
_ = Task.Run(async () =>
{
try
@@ -195,7 +198,7 @@ public class BomRadarService : IBomRadarService, IDisposable
{
_logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state);
}
}, cancellationToken);
});
status.UpdateTriggered = true;
status.Message = status.CacheExists
Services/Interfaces/IScrubDisplayTimestampParser.cs
+27
View File
@@ -0,0 +1,27 @@
using BomLocalService.Services.Interfaces.Registration;
using BomLocalService.Services.Scraping;
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Parses the BOM scrubber time display label to UTC; shared by capture and reset steps.
/// </summary>
public interface IScrubDisplayTimestampParser : ISingletonService
{
Task<DateTime?> ReadUtcAsync(
IPage page,
ISelectorService selectorService,
ScrapingContext? context = null,
CancellationToken cancellationToken = default);
bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc);
Task WaitForDisplayLabelChangeAsync(
IPage page,
ISelectorService selectorService,
DateTime currentTimestamp,
ScrapingContext? context = null,
int maxWaitMs = 5000,
CancellationToken cancellationToken = default);
}
Services/Scraping/ScrubDisplayTimestampParser.cs
+215
View File
@@ -0,0 +1,215 @@
using System.Text.RegularExpressions;
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using Microsoft.Playwright;
namespace BomLocalService.Services.Scraping;
/// <summary>
/// Parses the BOM scrubber time label (.bom-scrub-display-label) to UTC.
/// Shared by capture and reset steps so behaviour stays in sync.
/// </summary>
public class ScrubDisplayTimestampParser : IScrubDisplayTimestampParser
{
private readonly IConfiguration _configuration;
private readonly ScrapingSelectorsConfig _selectors;
private readonly TextPatternsConfig _textPatterns;
private readonly JavaScriptTemplatesConfig _javaScriptTemplates;
private readonly ILogger<ScrubDisplayTimestampParser> _logger;
public ScrubDisplayTimestampParser(
IConfiguration configuration,
ILogger<ScrubDisplayTimestampParser> logger)
{
_configuration = configuration;
_selectors = configuration.GetSection("Scraping:Selectors").Get<ScrapingSelectorsConfig>() ?? new();
_textPatterns = configuration.GetSection("Scraping:TextPatterns").Get<TextPatternsConfig>() ?? new();
_javaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get<JavaScriptTemplatesConfig>() ?? new();
_logger = logger;
}
public async Task<DateTime?> ReadUtcAsync(
IPage page,
ISelectorService selectorService,
ScrapingContext? context = null,
CancellationToken cancellationToken = default)
{
_ = cancellationToken;
try
{
var timeLabelLocator = selectorService.GetLocator(page, _selectors.TimeDisplayLabel);
var timeLabel = await timeLabelLocator.TextContentAsync();
if (string.IsNullOrEmpty(timeLabel))
{
_logger.LogDebug("Time display label is empty");
return null;
}
var trimmedLabel = timeLabel.Trim();
_logger.LogDebug("Scrub display label: '{Label}'", trimmedLabel);
var timestampMatch = Regex.Match(trimmedLabel, _textPatterns.TimestampPattern, RegexOptions.IgnoreCase);
if (!timestampMatch.Success)
{
_logger.LogWarning(
"Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'",
trimmedLabel,
_textPatterns.TimestampPattern);
return null;
}
var timestampStr = timestampMatch.Groups[0].Value;
string? detectedTimezone = null;
if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEDT";
else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEST";
if (detectedTimezone == null && context != null)
{
try
{
var metadataText = await page.EvaluateAsync<string>(_javaScriptTemplates.ExtractWeatherMetadata);
if (!string.IsNullOrEmpty(metadataText))
{
if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEDT";
else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEST";
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone");
}
}
if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc))
return frameTimestampUtc;
_logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
return null;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to extract timestamp from display label");
return null;
}
}
public bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc)
{
timestampUtc = DateTime.MinValue;
try
{
TimeZoneInfo timeZoneInfo;
if (!string.IsNullOrEmpty(timezoneAbbreviation))
{
if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase))
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney");
else
{
var timezone = _configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
return false;
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
}
}
else
{
var timezone = _configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
return false;
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
}
timestampStr = Regex.Replace(timestampStr.Trim(), @"\s+", " ");
var formats = new[]
{
"dddd d MMM, h:mm tt",
"d MMM, h:mm tt",
"dddd d MMM, hh:mm tt",
"d MMM, hh:mm tt",
"dddd dd MMM, h:mm tt",
"dd MMM, h:mm tt",
"dddd d MMM, h:mmtt",
"d MMM, h:mmtt",
"dddd d MMM, hh:mmtt",
"d MMM, hh:mmtt",
"dddd dd MMM, h:mmtt",
"dd MMM, h:mmtt",
"dddd dd MMM, hh:mmtt",
"dd MMM, hh:mmtt"
};
var culture = new System.Globalization.CultureInfo("en-AU");
DateTime localTime = default;
var parsed = false;
foreach (var format in formats)
{
if (DateTime.TryParseExact(timestampStr, format, culture,
System.Globalization.DateTimeStyles.None, out localTime))
{
parsed = true;
break;
}
}
if (!parsed)
return false;
if (localTime.Year == 1)
{
localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day,
localTime.Hour, localTime.Minute, localTime.Second);
}
timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr);
return false;
}
}
public async Task WaitForDisplayLabelChangeAsync(
IPage page,
ISelectorService selectorService,
DateTime currentTimestamp,
ScrapingContext? context = null,
int maxWaitMs = 5000,
CancellationToken cancellationToken = default)
{
_ = cancellationToken;
try
{
var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
{
var newTimestamp = await ReadUtcAsync(page, selectorService, context, cancellationToken);
if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp)
return;
await page.WaitForTimeoutAsync(200);
}
_logger.LogDebug(
"Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms",
currentTimestamp,
maxWaitMs);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Error waiting for display label to change");
}
}
}
Services/Scraping/Steps/Capture/CaptureFramesStep.cs
+12 -226
View File
@@ -3,13 +3,13 @@ using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping;
using BomLocalService.Utilities;
using Microsoft.Playwright;
using System.Text.RegularExpressions;
namespace BomLocalService.Services.Scraping.Steps.Capture;
public class CaptureFramesStep : BaseScrapingStep
{
private readonly ICacheService _cacheService;
private readonly IScrubDisplayTimestampParser _scrubDisplayTime;
private readonly int _tileRenderWaitMs;
private readonly ScreenshotCropConfig _cropConfig;
@@ -21,10 +21,12 @@ public class CaptureFramesStep : BaseScrapingStep
ISelectorService selectorService,
IDebugService debugService,
IConfiguration configuration,
ICacheService cacheService)
ICacheService cacheService,
IScrubDisplayTimestampParser scrubDisplayTime)
: base(logger, selectorService, debugService, configuration)
{
_cacheService = cacheService;
_scrubDisplayTime = scrubDisplayTime;
var tileRenderWaitMsConfig = configuration.GetValue<int?>("Screenshot:TileRenderWaitMs");
if (!tileRenderWaitMsConfig.HasValue)
@@ -91,12 +93,12 @@ public class CaptureFramesStep : BaseScrapingStep
await context.Page.WaitForTimeoutAsync(300);
// Try extracting timestamp with a retry in case the label is still updating
var frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context);
var frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (frameTimestamp == null)
{
// Retry once after a short wait in case label was updating
await context.Page.WaitForTimeoutAsync(200);
frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context);
frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
}
// Fallback: calculate expected timestamp from observation time and frame index if we can't parse it
@@ -112,8 +114,9 @@ public class CaptureFramesStep : BaseScrapingStep
{
Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same timestamp ({Timestamp}) as previous frame. Waiting for display to update...",
Name, frameIndex, frameTimestamp);
await WaitForDisplayLabelToChangeAsync(context.Page, previousTimestamp.Value, context);
frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context);
await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, previousTimestamp.Value, context, cancellationToken: cancellationToken);
frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (frameTimestamp == null || frameTimestamp == previousTimestamp.Value)
{
if (context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
@@ -161,13 +164,14 @@ public class CaptureFramesStep : BaseScrapingStep
{
await DismissModalOverlaysAsync(context.Page);
var currentTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context);
var currentTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
if (currentTimestamp.HasValue)
{
await WaitForDisplayLabelToChangeAsync(context.Page, currentTimestamp.Value);
await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, currentTimestamp.Value, context, cancellationToken: cancellationToken);
}
else
{
@@ -198,224 +202,6 @@ public class CaptureFramesStep : BaseScrapingStep
}
}
/// <summary>
/// Extracts the UTC timestamp from the frame display label
/// </summary>
private async Task<DateTime?> ExtractTimestampFromDisplayAsync(IPage page, ScrapingContext? context = null)
{
try
{
var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel);
var timeLabel = await timeLabelLocator.TextContentAsync();
if (string.IsNullOrEmpty(timeLabel))
{
Logger.LogDebug("Time display label is empty");
return null;
}
var trimmedLabel = timeLabel.Trim();
Logger.LogInformation("Extracting timestamp from display label: '{Label}'", trimmedLabel);
// Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern);
var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase);
if (timestampMatch.Success)
{
var timestampStr = timestampMatch.Groups[0].Value;
// Check for timezone abbreviation in the full label (not just the matched timestamp)
// BOM website may display timezone elsewhere in the label text
string? detectedTimezone = null;
if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEDT";
}
else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEST";
}
// Fallback: If timezone not found in frame label, try to extract it from metadata
// The metadata has the timezone (e.g., "9:40 pm AEST"), so we can use that for frames too
if (detectedTimezone == null && context != null)
{
try
{
var metadataText = await page.EvaluateAsync<string>(JavaScriptTemplates.ExtractWeatherMetadata);
if (!string.IsNullOrEmpty(metadataText))
{
if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEDT";
Logger.LogDebug("Detected timezone AEDT from metadata text");
}
else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEST";
Logger.LogDebug("Detected timezone AEST from metadata text");
}
}
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone");
}
}
Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}' (detected timezone: {Tz})",
timestampStr, trimmedLabel, detectedTimezone ?? "none");
if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc))
{
Logger.LogInformation("Successfully parsed frame timestamp: {Timestamp} UTC", frameTimestampUtc);
return frameTimestampUtc;
}
else
{
Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
}
}
else
{
Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern);
}
return null;
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Failed to extract timestamp from display label");
return null;
}
}
private bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc)
{
timestampUtc = DateTime.MinValue;
try
{
// Determine timezone based on detected abbreviation or configured default
TimeZoneInfo timeZoneInfo;
if (!string.IsNullOrEmpty(timezoneAbbreviation))
{
// Map timezone abbreviations to actual timezones (same logic as TimeParsingService)
// - "AEST" = UTC+10 (Australian Eastern Standard Time) - Brisbane year-round, Sydney/Melbourne in winter
// - "AEDT" = UTC+11 (Australian Eastern Daylight Time) - Sydney/Melbourne in summer (Oct-Apr), never Brisbane
if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
Logger.LogDebug("Using Brisbane timezone (AEST, UTC+10) for frame timestamp");
}
else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney");
Logger.LogDebug("Using Sydney timezone (AEDT, UTC+11) for frame timestamp");
}
else
{
// Fallback to configured timezone if abbreviation is unrecognized
var timezone = Configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
{
Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly");
return false;
}
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
Logger.LogDebug("Using configured timezone '{Timezone}' for frame timestamp", timezone);
}
}
else
{
// No timezone detected - use configured default
var timezone = Configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
{
Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly");
return false;
}
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
Logger.LogDebug("No timezone abbreviation detected, using configured timezone '{Timezone}' for frame timestamp", timezone);
}
// Try common Australian date formats
// Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
var formats = new[]
{
"dddd d MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm
"d MMM, h:mm tt", // 17 Dec, 11:05 pm
"dddd d MMM, hh:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero)
"d MMM, hh:mm tt", // 17 Dec, 11:05 pm (with leading zero)
"dddd dd MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero day)
"dd MMM, h:mm tt" // 17 Dec, 11:05 pm (with leading zero day)
};
var culture = new System.Globalization.CultureInfo("en-AU");
DateTime localTime = default;
bool parsed = false;
foreach (var format in formats)
{
if (DateTime.TryParseExact(timestampStr, format, culture,
System.Globalization.DateTimeStyles.None, out localTime))
{
parsed = true;
break;
}
}
if (!parsed)
{
return false;
}
// If year is not specified, assume current year
if (localTime.Year == 1)
{
localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day,
localTime.Hour, localTime.Minute, localTime.Second);
}
// The parsed timestamp includes both date and time (e.g., "23 Dec, 11:45 pm")
// localTime already contains the complete date and time from the parsed string
// We treat it as being in the target timezone (DateTimeKind.Unspecified), then convert to UTC
timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo);
Logger.LogDebug("Parsed frame timestamp '{TimestampStr}' with timezone '{TzAbbrev}' as {LocalTime} local ({UtcTime} UTC)",
timestampStr, timezoneAbbreviation ?? "default", localTime, timestampUtc);
return true;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr);
return false;
}
}
private async Task WaitForDisplayLabelToChangeAsync(IPage page, DateTime currentTimestamp, ScrapingContext? context = null, int maxWaitMs = 5000)
{
try
{
var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
{
var newTimestamp = await ExtractTimestampFromDisplayAsync(page, context);
if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp)
{
return;
}
await page.WaitForTimeoutAsync(200);
}
Logger.LogDebug("Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms", currentTimestamp, maxWaitMs);
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Error waiting for display label to change");
}
}
private async Task DismissModalOverlaysAsync(IPage page)
{
try
Services/Scraping/Steps/Map/PauseRadarStep.cs
+33 -6
View File
@@ -31,16 +31,43 @@ public class PauseRadarStep : BaseScrapingStep
var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton);
await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true)
// New spatial map often uses one step-through control; label may be on the button or in .bom-scrub-action__label
string? buttonLabel = null;
try
{
var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First;
if (await labelLoc.CountAsync() > 0)
buttonLabel = (await labelLoc.TextContentAsync())?.Trim();
}
catch
{
/* use inner text */
}
buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim();
if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true)
{
Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name);
await playPauseButton.ClickAsync();
await context.Page.WaitForTimeoutAsync(300);
buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true)
// Clear so post-click read always runs; otherwise stale "Pause" skips the ??= InnerText fallback when the label child is missing
buttonLabel = null;
try
{
var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First;
if (await labelLoc.CountAsync() > 0)
buttonLabel = (await labelLoc.TextContentAsync())?.Trim();
}
catch
{
buttonLabel = null;
}
buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim();
if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true)
{
Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name);
}
Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs
+93 -59
View File
@@ -1,99 +1,134 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping;
using BomLocalService.Utilities;
using Microsoft.Playwright;
namespace BomLocalService.Services.Scraping.Steps.Map;
/// <summary>
/// Aligns the radar scrubber to the oldest frame (historical frame 0) without clicking timeline segments.
/// The BOM UI now often exposes a single step-through control; segment pills and <c>bom-scrub-tl</c> are unreliable.
/// Strategy: while paused, read <see cref="IScrubDisplayTimestampParser"/> from the display label, step forward
/// through one full loop to find the minimum time, then step until that time shows again.
/// </summary>
public class ResetToFirstFrameStep : BaseScrapingStep
{
private readonly IScrubDisplayTimestampParser _scrubDisplayTime;
public override string Name => "ResetToFirstFrame";
public override string[] Prerequisites => new[] { "PauseRadar" };
public override string[] Prerequisites => new[] { "ExtractMetadata" };
public ResetToFirstFrameStep(
ILogger<ResetToFirstFrameStep> logger,
ISelectorService selectorService,
IDebugService debugService,
IConfiguration configuration)
IConfiguration configuration,
IScrubDisplayTimestampParser scrubDisplayTime)
: base(logger, selectorService, debugService, configuration)
{
_scrubDisplayTime = scrubDisplayTime;
}
public override bool CanExecute(ScrapingContext context)
{
return context.CurrentState >= PageState.SlideshowPaused;
return context.CurrentState >= PageState.SlideshowPaused
&& context.LastUpdatedInfo != null;
}
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
{
try
{
Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name);
try
Logger.LogInformation("Step {Step}: Aligning to oldest frame via step-through + display label (no segment clicks)", Name);
var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
var elementWaitMs = Configuration.GetValue<int?>("Scraping:Timeouts:ElementWait") ?? 10000;
var stepBtn = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton);
await stepBtn.WaitForAsync(new LocatorWaitForOptions { State = WaitForSelectorState.Visible, Timeout = elementWaitMs });
// Phase 1 — sample one full loop of display times while paused; oldest = frame 0 target
var samples = new List<DateTime>();
for (var i = 0; i < frameCount; i++)
{
// Try JavaScript click first to avoid thumb interception
var clicked = await context.Page.EvaluateAsync<bool>(@"() => {
const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]');
if (segment) {
segment.click();
return true;
var t = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (t.HasValue)
samples.Add(t.Value);
if (i == frameCount - 1)
break;
var beforeStep = t;
await stepBtn.ClickAsync(new LocatorClickOptions { Force = true });
if (beforeStep.HasValue)
{
await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, beforeStep.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken);
}
else
await context.Page.WaitForTimeoutAsync(800);
}
DateTime? frame0Utc = null;
if (samples.Count > 0)
{
frame0Utc = samples.Min();
Logger.LogInformation("Step {Step}: Oldest time in sampled loop (frame 0 target): {Utc:o} UTC", Name, frame0Utc);
}
else
{
Logger.LogWarning("Step {Step}: Could not read any display timestamps; tile timing may be wrong", Name);
}
// Phase 2 — step until the display matches the oldest sample
if (frame0Utc.HasValue)
{
const double matchMinutes = 2.0;
for (var attempt = 0; attempt < frameCount + 4; attempt++)
{
var cur = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (cur.HasValue && Math.Abs((cur.Value - frame0Utc.Value).TotalMinutes) <= matchMinutes)
{
Logger.LogInformation("Step {Step}: Display matches frame 0 target (≤{MatchMinutes:F0} min)", Name, matchMinutes);
break;
}
return false;
}");
if (clicked)
{
await context.Page.WaitForTimeoutAsync(500);
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name);
}
else
{
// Fallback to locator click with force
var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true });
await context.Page.WaitForTimeoutAsync(500);
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name);
await stepBtn.ClickAsync(new LocatorClickOptions { Force = true });
if (cur.HasValue)
{
await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, cur.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken);
}
else
await context.Page.WaitForTimeoutAsync(800);
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name);
}
await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken);
// Verify scrubber is at position 0
Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name);
await SaveDebugAsync(context, 9, "frame_0_aligned", cancellationToken);
// Best-effort only: segment-based checks are unreliable on the new spatial map layout
try
{
var activeSegment = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment);
if (activeSegment)
{
Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name);
}
else
{
Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name);
}
var onSegmentZero = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment);
if (onSegmentZero)
Logger.LogDebug("Step {Step}: Segment check reports frame 0 (informational)", Name);
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name);
Logger.LogDebug(ex, "Step {Step}: Segment position check skipped", Name);
}
await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken);
// Wait for frame 0 tiles to fully render
await SaveDebugAsync(context, 10, "scrubber_frame0", cancellationToken);
var tileRenderWaitMsConfig = Configuration.GetValue<int?>("Screenshot:TileRenderWaitMs");
if (!tileRenderWaitMsConfig.HasValue)
{
throw new InvalidOperationException("Screenshot:TileRenderWaitMs configuration is required. Set it in appsettings.json or via SCREENSHOT__TILERENDERWAITMS environment variable.");
}
var tileRenderWaitMs = tileRenderWaitMsConfig.Value;
await context.Page.WaitForTimeoutAsync(tileRenderWaitMs);
await context.Page.WaitForTimeoutAsync(tileRenderWaitMsConfig.Value);
context.CurrentState = PageState.Frame0Selected;
return ScrapingStepResult.Successful();
}
@@ -105,4 +140,3 @@ public class ResetToFirstFrameStep : BaseScrapingStep
}
}
}
Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs
+47 -19
View File
@@ -1,15 +1,18 @@
using System.Text.Json;
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping;
using BomLocalService.Utilities;
namespace BomLocalService.Services.Scraping.Steps.Metadata;
public class ExtractMetadataStep : BaseScrapingStep
{
private readonly ITimeParsingService _timeParsingService;
public override string Name => "ExtractMetadata";
public override string[] Prerequisites => new[] { "ResetToFirstFrame" };
public override string[] Prerequisites => new[] { "PauseRadar" };
public ExtractMetadataStep(
ILogger<ExtractMetadataStep> logger,
ISelectorService selectorService,
@@ -20,41 +23,67 @@ public class ExtractMetadataStep : BaseScrapingStep
{
_timeParsingService = timeParsingService;
}
public override bool CanExecute(ScrapingContext context)
{
return context.IsMapReady;
}
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
{
try
{
Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name);
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page);
context.LastUpdatedInfo = lastUpdatedInfo;
// Extract frame info
var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
var defaults = Enumerable.Range(0, frameCount)
.Select(i => (index: i, minutesAgo: 40 - i * 5))
.ToList();
try
{
var frameInfo = await context.Page.EvaluateAsync<object[]>(JavaScriptTemplates.ExtractFrameInfo);
var result = new List<(int index, int minutesAgo)>();
for (int i = 0; i < 7; i++)
var element = await context.Page.EvaluateAsync<JsonElement>(JavaScriptTemplates.ExtractFrameInfo);
if (element.ValueKind == JsonValueKind.Array && element.GetArrayLength() > 0)
{
var minutesAgo = 40 - (i * 5);
result.Add((i, minutesAgo));
var byIndex = new Dictionary<int, int>();
foreach (var item in element.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Object)
continue;
var idx = item.GetProperty("index").GetInt32();
if (item.TryGetProperty("minutesAgo", out var m) && m.ValueKind == JsonValueKind.Number)
byIndex[idx] = m.GetInt32();
}
if (byIndex.Count > 0)
{
context.FrameInfo = Enumerable.Range(0, frameCount)
.Select(i => (i, byIndex.TryGetValue(i, out var mm) ? mm : defaults[i].minutesAgo))
.ToList();
Logger.LogInformation("Step {Step}: Parsed frame offsets from segment aria-labels where available", Name);
}
else
{
context.FrameInfo = defaults;
Logger.LogInformation("Step {Step}: Segment times not parsed; using default ladder", Name);
}
}
else
{
context.FrameInfo = defaults;
Logger.LogInformation("Step {Step}: No segment array from page; using default ladder", Name);
}
context.FrameInfo = result;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name);
context.FrameInfo = Enumerable.Range(0, 7)
.Select(i => (i, 40 - (i * 5)))
.ToList();
context.FrameInfo = defaults;
}
return ScrapingStepResult.Successful();
}
catch (Exception ex)
@@ -65,4 +94,3 @@ public class ExtractMetadataStep : BaseScrapingStep
}
}
}
Services/Scraping/Workflows/RadarScrapingWorkflow.cs
+1 -1
View File
@@ -28,8 +28,8 @@ public class RadarScrapingWorkflow : IRadarScrapingWorkflow
"ClickRadarLink",
"WaitForMapReady",
"PauseRadar",
"ResetToFirstFrame",
"ExtractMetadata",
"ResetToFirstFrame",
"CalculateMapBounds",
"CaptureFrames"
};
Services/SelectorService.cs
+11 -3
View File
@@ -52,9 +52,17 @@ public class SelectorService : ISelectorService
{
throw new ArgumentException($"No selectors configured for {config.Name}");
}
// Returns first selector as locator (for cases where we know it exists)
return page.Locator(config.Selectors[0]).First;
// Match any configured selector (BOM occasionally renames test ids; list is tried in order)
var combined = page.Locator(config.Selectors[0]).First;
for (var i = 1; i < config.Selectors.Length; i++)
{
combined = combined.Or(page.Locator(config.Selectors[i]).First);
}
// Or() unions alternatives; BOM sometimes renders duplicates (e.g. responsive + hidden).
// Pin to a single element so Playwright strict mode does not reject clicks.
return combined.First;
}
}
appsettings.json
+15 -10
View File
@@ -61,11 +61,9 @@
"Name": "Search Button",
"Selectors": [
"button[data-testid='searchLabel']",
"button[aria-label='Search for a location']",
"button.search-location__trigger-button",
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
"button:has-text('Search for a location')",
"[data-testid='showSearchModal'] button"
"button[aria-label='Search for a location'].search-location__trigger-button",
"button.search-location__trigger-button.bom-button:has(span.bom-button__label:has-text('Search for a location'))"
],
"TimeoutMs": 10000,
"Required": true,
@@ -137,7 +135,10 @@
},
"PlayPauseButton": {
"Name": "Play Pause Button",
"Selectors": ["button[data-testid='bom-time-scrub-play-pause']"],
"Selectors": [
"button[data-testid='bom-time-scrub-step-through']",
"button[data-testid='bom-time-scrub-play-pause']"
],
"TimeoutMs": 5000,
"Required": true
},
@@ -155,7 +156,11 @@
},
"StepForwardButton": {
"Name": "Step Forward Button",
"Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"],
"Selectors": [
"button[data-testid='bom-time-scrub-step-through']",
"button.bom-scrub-action--step-through",
"button[data-testid='bom-scrub-utils__right__step-forward']"
],
"TimeoutMs": 5000,
"Required": true
},
@@ -184,9 +189,9 @@
"ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }",
"WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }",
"WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }",
"CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
"CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); if (segments.length === 0) return true; const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
"WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }",
"ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
"ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]+),?\\s+(\\d{1,2}):(\\d{2})\\s*(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
"CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }",
"CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }",
"GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })",
@@ -194,7 +199,7 @@
},
"TextPatterns": {
"ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]+,?\\s+\\d{1,2}:\\d{2}\\s*(?:am|pm)",
"ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago",
"ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago",
"ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago",
@@ -218,8 +223,8 @@
"ClickRadarLink": { "Enabled": true },
"WaitForMapReady": { "Enabled": true },
"PauseRadar": { "Enabled": true },
"ResetToFirstFrame": { "Enabled": true },
"ExtractMetadata": { "Enabled": true },
"ResetToFirstFrame": { "Enabled": true },
"CalculateMapBounds": { "Enabled": true },
"CaptureFrames": {
"Enabled": true,