4 Commits
Author SHA1 Message Date
alex 8eecce00a6 Release v0.0.6: resilient BOM metadata parsing.
Read C07_WeatherMetadata via innerText, accept observation prefixes (a minute,
N minutes, an hour, N hours) and forecast N hours ago. Version 0.0.6.
2026-05-07 05:49:51 +10:00
alex 28e8cb4fd1 Bump version to 0.0.5 for release (v0.0.4 already exists on GitHub). 2026-05-07 05:23:49 +10:00
alex a54061f757 Release v0.0.4: BOM spatial map scraping and radar cache reliability.
Align selectors and workflow with the step-through scrubber, share scrub display
timestamp parsing via IScrubDisplayTimestampParser, fix pause post-click label
verification, and stop cancelling background cache updates when the HTTP request
ends. Version bumped to 0.0.4.
2026-05-07 05:23:18 +10:00
alex 0b6d266e79 Replace large image with smaller icon in README 2025-12-24 02:15:35 +10:00
16 changed files with 490 additions and 352 deletions
+1
View File
@@ -4,6 +4,7 @@
<TargetFramework>net9.0</TargetFramework> <TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Version>0.0.6</Version>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
+2 -1
View File
@@ -53,6 +53,7 @@ public class JavaScriptTemplatesConfig
public string CheckActiveFrameSegment { get; set; } = @"() => { public string CheckActiveFrameSegment { get; set; } = @"() => {
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
if (segments.length === 0) return true;
const activeSegment = segments.find(s => { const activeSegment = segments.find(s => {
const style = window.getComputedStyle(s); const style = window.getComputedStyle(s);
return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
@@ -73,7 +74,7 @@ public class JavaScriptTemplatesConfig
let minutes = null; let minutes = null;
// Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm"" // Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm""
const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i); const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]+),?\s+(\d{1,2}):(\d{2})\s*(am|pm)/i);
if (timestampMatch) { if (timestampMatch) {
try { try {
const day = parseInt(timestampMatch[1]); const day = parseInt(timestampMatch[1]);
+1 -1
View File
@@ -6,7 +6,7 @@ namespace BomLocalService.Models;
public class TextPatternsConfig public class TextPatternsConfig
{ {
public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)"; public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)";
public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)"; public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]+,?\s+\d{1,2}:\d{2}\s*(?:am|pm)";
public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago"; public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago";
public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago"; public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago";
public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago"; public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago";
-1
View File
@@ -1,6 +1,5 @@
using BomLocalService.Extensions; using BomLocalService.Extensions;
using BomLocalService.Services.Interfaces; using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
+1 -5
View File
@@ -1,9 +1,5 @@
<p align="center"> <p align="center">
<img <img width="256" height="auto" alt="icon" src="https://github.com/user-attachments/assets/6f712fbd-4f42-49d4-9ddb-1767bb43551a" />
src="https://github.com/user-attachments/assets/684b73cd-a374-49e6-aaf1-ccc4c5697b47"
width="500"
alt="f67325fe-33ae-4fd8-9e57-38b46899d09c"
/>
</p> </p>
# BOM Local Service # BOM Local Service
+4 -1
View File
@@ -185,6 +185,9 @@ public class BomRadarService : IBomRadarService, IDisposable
if (needsUpdate) if (needsUpdate)
{ {
// Trigger async update (fire and forget) // Trigger async update (fire and forget)
// Do not pass the caller's cancellationToken to Task.Run: for HTTP that token is tied to
// RequestAborted and often fires as soon as the response is sent, so the update never starts.
// FetchAndCacheScreenshotAsync already uses CancellationToken.None for the scrape work.
_ = Task.Run(async () => _ = Task.Run(async () =>
{ {
try try
@@ -195,7 +198,7 @@ public class BomRadarService : IBomRadarService, IDisposable
{ {
_logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state); _logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state);
} }
}, cancellationToken); });
status.UpdateTriggered = true; status.UpdateTriggered = true;
status.Message = status.CacheExists status.Message = status.CacheExists
@@ -0,0 +1,27 @@
using BomLocalService.Services.Interfaces.Registration;
using BomLocalService.Services.Scraping;
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Parses the BOM scrubber time display label to UTC; shared by capture and reset steps.
/// </summary>
public interface IScrubDisplayTimestampParser : ISingletonService
{
Task<DateTime?> ReadUtcAsync(
IPage page,
ISelectorService selectorService,
ScrapingContext? context = null,
CancellationToken cancellationToken = default);
bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc);
Task WaitForDisplayLabelChangeAsync(
IPage page,
ISelectorService selectorService,
DateTime currentTimestamp,
ScrapingContext? context = null,
int maxWaitMs = 5000,
CancellationToken cancellationToken = default);
}
@@ -0,0 +1,215 @@
using System.Text.RegularExpressions;
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using Microsoft.Playwright;
namespace BomLocalService.Services.Scraping;
/// <summary>
/// Parses the BOM scrubber time label (.bom-scrub-display-label) to UTC.
/// Shared by capture and reset steps so behaviour stays in sync.
/// </summary>
public class ScrubDisplayTimestampParser : IScrubDisplayTimestampParser
{
private readonly IConfiguration _configuration;
private readonly ScrapingSelectorsConfig _selectors;
private readonly TextPatternsConfig _textPatterns;
private readonly JavaScriptTemplatesConfig _javaScriptTemplates;
private readonly ILogger<ScrubDisplayTimestampParser> _logger;
public ScrubDisplayTimestampParser(
IConfiguration configuration,
ILogger<ScrubDisplayTimestampParser> logger)
{
_configuration = configuration;
_selectors = configuration.GetSection("Scraping:Selectors").Get<ScrapingSelectorsConfig>() ?? new();
_textPatterns = configuration.GetSection("Scraping:TextPatterns").Get<TextPatternsConfig>() ?? new();
_javaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get<JavaScriptTemplatesConfig>() ?? new();
_logger = logger;
}
public async Task<DateTime?> ReadUtcAsync(
IPage page,
ISelectorService selectorService,
ScrapingContext? context = null,
CancellationToken cancellationToken = default)
{
_ = cancellationToken;
try
{
var timeLabelLocator = selectorService.GetLocator(page, _selectors.TimeDisplayLabel);
var timeLabel = await timeLabelLocator.TextContentAsync();
if (string.IsNullOrEmpty(timeLabel))
{
_logger.LogDebug("Time display label is empty");
return null;
}
var trimmedLabel = timeLabel.Trim();
_logger.LogDebug("Scrub display label: '{Label}'", trimmedLabel);
var timestampMatch = Regex.Match(trimmedLabel, _textPatterns.TimestampPattern, RegexOptions.IgnoreCase);
if (!timestampMatch.Success)
{
_logger.LogWarning(
"Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'",
trimmedLabel,
_textPatterns.TimestampPattern);
return null;
}
var timestampStr = timestampMatch.Groups[0].Value;
string? detectedTimezone = null;
if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEDT";
else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEST";
if (detectedTimezone == null && context != null)
{
try
{
var metadataText = await page.EvaluateAsync<string>(_javaScriptTemplates.ExtractWeatherMetadata);
if (!string.IsNullOrEmpty(metadataText))
{
if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEDT";
else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase))
detectedTimezone = "AEST";
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone");
}
}
if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc))
return frameTimestampUtc;
_logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
return null;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to extract timestamp from display label");
return null;
}
}
public bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc)
{
timestampUtc = DateTime.MinValue;
try
{
TimeZoneInfo timeZoneInfo;
if (!string.IsNullOrEmpty(timezoneAbbreviation))
{
if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase))
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney");
else
{
var timezone = _configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
return false;
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
}
}
else
{
var timezone = _configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
return false;
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
}
timestampStr = Regex.Replace(timestampStr.Trim(), @"\s+", " ");
var formats = new[]
{
"dddd d MMM, h:mm tt",
"d MMM, h:mm tt",
"dddd d MMM, hh:mm tt",
"d MMM, hh:mm tt",
"dddd dd MMM, h:mm tt",
"dd MMM, h:mm tt",
"dddd d MMM, h:mmtt",
"d MMM, h:mmtt",
"dddd d MMM, hh:mmtt",
"d MMM, hh:mmtt",
"dddd dd MMM, h:mmtt",
"dd MMM, h:mmtt",
"dddd dd MMM, hh:mmtt",
"dd MMM, hh:mmtt"
};
var culture = new System.Globalization.CultureInfo("en-AU");
DateTime localTime = default;
var parsed = false;
foreach (var format in formats)
{
if (DateTime.TryParseExact(timestampStr, format, culture,
System.Globalization.DateTimeStyles.None, out localTime))
{
parsed = true;
break;
}
}
if (!parsed)
return false;
if (localTime.Year == 1)
{
localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day,
localTime.Hour, localTime.Minute, localTime.Second);
}
timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr);
return false;
}
}
public async Task WaitForDisplayLabelChangeAsync(
IPage page,
ISelectorService selectorService,
DateTime currentTimestamp,
ScrapingContext? context = null,
int maxWaitMs = 5000,
CancellationToken cancellationToken = default)
{
_ = cancellationToken;
try
{
var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
{
var newTimestamp = await ReadUtcAsync(page, selectorService, context, cancellationToken);
if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp)
return;
await page.WaitForTimeoutAsync(200);
}
_logger.LogDebug(
"Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms",
currentTimestamp,
maxWaitMs);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Error waiting for display label to change");
}
}
}
@@ -3,13 +3,13 @@ using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping; using BomLocalService.Services.Scraping;
using BomLocalService.Utilities; using BomLocalService.Utilities;
using Microsoft.Playwright; using Microsoft.Playwright;
using System.Text.RegularExpressions;
namespace BomLocalService.Services.Scraping.Steps.Capture; namespace BomLocalService.Services.Scraping.Steps.Capture;
public class CaptureFramesStep : BaseScrapingStep public class CaptureFramesStep : BaseScrapingStep
{ {
private readonly ICacheService _cacheService; private readonly ICacheService _cacheService;
private readonly IScrubDisplayTimestampParser _scrubDisplayTime;
private readonly int _tileRenderWaitMs; private readonly int _tileRenderWaitMs;
private readonly ScreenshotCropConfig _cropConfig; private readonly ScreenshotCropConfig _cropConfig;
@@ -21,10 +21,12 @@ public class CaptureFramesStep : BaseScrapingStep
ISelectorService selectorService, ISelectorService selectorService,
IDebugService debugService, IDebugService debugService,
IConfiguration configuration, IConfiguration configuration,
ICacheService cacheService) ICacheService cacheService,
IScrubDisplayTimestampParser scrubDisplayTime)
: base(logger, selectorService, debugService, configuration) : base(logger, selectorService, debugService, configuration)
{ {
_cacheService = cacheService; _cacheService = cacheService;
_scrubDisplayTime = scrubDisplayTime;
var tileRenderWaitMsConfig = configuration.GetValue<int?>("Screenshot:TileRenderWaitMs"); var tileRenderWaitMsConfig = configuration.GetValue<int?>("Screenshot:TileRenderWaitMs");
if (!tileRenderWaitMsConfig.HasValue) if (!tileRenderWaitMsConfig.HasValue)
@@ -91,12 +93,12 @@ public class CaptureFramesStep : BaseScrapingStep
await context.Page.WaitForTimeoutAsync(300); await context.Page.WaitForTimeoutAsync(300);
// Try extracting timestamp with a retry in case the label is still updating // Try extracting timestamp with a retry in case the label is still updating
var frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); var frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (frameTimestamp == null) if (frameTimestamp == null)
{ {
// Retry once after a short wait in case label was updating // Retry once after a short wait in case label was updating
await context.Page.WaitForTimeoutAsync(200); await context.Page.WaitForTimeoutAsync(200);
frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
} }
// Fallback: calculate expected timestamp from observation time and frame index if we can't parse it // Fallback: calculate expected timestamp from observation time and frame index if we can't parse it
@@ -112,8 +114,9 @@ public class CaptureFramesStep : BaseScrapingStep
{ {
Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same timestamp ({Timestamp}) as previous frame. Waiting for display to update...", Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same timestamp ({Timestamp}) as previous frame. Waiting for display to update...",
Name, frameIndex, frameTimestamp); Name, frameIndex, frameTimestamp);
await WaitForDisplayLabelToChangeAsync(context.Page, previousTimestamp.Value, context); await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); context.Page, SelectorService, previousTimestamp.Value, context, cancellationToken: cancellationToken);
frameTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (frameTimestamp == null || frameTimestamp == previousTimestamp.Value) if (frameTimestamp == null || frameTimestamp == previousTimestamp.Value)
{ {
if (context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count) if (context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
@@ -161,13 +164,14 @@ public class CaptureFramesStep : BaseScrapingStep
{ {
await DismissModalOverlaysAsync(context.Page); await DismissModalOverlaysAsync(context.Page);
var currentTimestamp = await ExtractTimestampFromDisplayAsync(context.Page, context); var currentTimestamp = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true }); await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
if (currentTimestamp.HasValue) if (currentTimestamp.HasValue)
{ {
await WaitForDisplayLabelToChangeAsync(context.Page, currentTimestamp.Value); await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, currentTimestamp.Value, context, cancellationToken: cancellationToken);
} }
else else
{ {
@@ -198,224 +202,6 @@ public class CaptureFramesStep : BaseScrapingStep
} }
} }
/// <summary>
/// Extracts the UTC timestamp from the frame display label
/// </summary>
private async Task<DateTime?> ExtractTimestampFromDisplayAsync(IPage page, ScrapingContext? context = null)
{
try
{
var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel);
var timeLabel = await timeLabelLocator.TextContentAsync();
if (string.IsNullOrEmpty(timeLabel))
{
Logger.LogDebug("Time display label is empty");
return null;
}
var trimmedLabel = timeLabel.Trim();
Logger.LogInformation("Extracting timestamp from display label: '{Label}'", trimmedLabel);
// Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern);
var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase);
if (timestampMatch.Success)
{
var timestampStr = timestampMatch.Groups[0].Value;
// Check for timezone abbreviation in the full label (not just the matched timestamp)
// BOM website may display timezone elsewhere in the label text
string? detectedTimezone = null;
if (trimmedLabel.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEDT";
}
else if (trimmedLabel.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEST";
}
// Fallback: If timezone not found in frame label, try to extract it from metadata
// The metadata has the timezone (e.g., "9:40 pm AEST"), so we can use that for frames too
if (detectedTimezone == null && context != null)
{
try
{
var metadataText = await page.EvaluateAsync<string>(JavaScriptTemplates.ExtractWeatherMetadata);
if (!string.IsNullOrEmpty(metadataText))
{
if (metadataText.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEDT";
Logger.LogDebug("Detected timezone AEDT from metadata text");
}
else if (metadataText.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
detectedTimezone = "AEST";
Logger.LogDebug("Detected timezone AEST from metadata text");
}
}
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Failed to extract timezone from metadata, using configured timezone");
}
}
Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}' (detected timezone: {Tz})",
timestampStr, trimmedLabel, detectedTimezone ?? "none");
if (TryParseTimestamp(timestampStr, detectedTimezone, out var frameTimestampUtc))
{
Logger.LogInformation("Successfully parsed frame timestamp: {Timestamp} UTC", frameTimestampUtc);
return frameTimestampUtc;
}
else
{
Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
}
}
else
{
Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern);
}
return null;
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Failed to extract timestamp from display label");
return null;
}
}
private bool TryParseTimestamp(string timestampStr, string? timezoneAbbreviation, out DateTime timestampUtc)
{
timestampUtc = DateTime.MinValue;
try
{
// Determine timezone based on detected abbreviation or configured default
TimeZoneInfo timeZoneInfo;
if (!string.IsNullOrEmpty(timezoneAbbreviation))
{
// Map timezone abbreviations to actual timezones (same logic as TimeParsingService)
// - "AEST" = UTC+10 (Australian Eastern Standard Time) - Brisbane year-round, Sydney/Melbourne in winter
// - "AEDT" = UTC+11 (Australian Eastern Daylight Time) - Sydney/Melbourne in summer (Oct-Apr), never Brisbane
if (timezoneAbbreviation.Contains("AEST", StringComparison.OrdinalIgnoreCase))
{
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
Logger.LogDebug("Using Brisbane timezone (AEST, UTC+10) for frame timestamp");
}
else if (timezoneAbbreviation.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney");
Logger.LogDebug("Using Sydney timezone (AEDT, UTC+11) for frame timestamp");
}
else
{
// Fallback to configured timezone if abbreviation is unrecognized
var timezone = Configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
{
Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly");
return false;
}
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
Logger.LogDebug("Using configured timezone '{Timezone}' for frame timestamp", timezone);
}
}
else
{
// No timezone detected - use configured default
var timezone = Configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
{
Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly");
return false;
}
timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
Logger.LogDebug("No timezone abbreviation detected, using configured timezone '{Timezone}' for frame timestamp", timezone);
}
// Try common Australian date formats
// Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
var formats = new[]
{
"dddd d MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm
"d MMM, h:mm tt", // 17 Dec, 11:05 pm
"dddd d MMM, hh:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero)
"d MMM, hh:mm tt", // 17 Dec, 11:05 pm (with leading zero)
"dddd dd MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero day)
"dd MMM, h:mm tt" // 17 Dec, 11:05 pm (with leading zero day)
};
var culture = new System.Globalization.CultureInfo("en-AU");
DateTime localTime = default;
bool parsed = false;
foreach (var format in formats)
{
if (DateTime.TryParseExact(timestampStr, format, culture,
System.Globalization.DateTimeStyles.None, out localTime))
{
parsed = true;
break;
}
}
if (!parsed)
{
return false;
}
// If year is not specified, assume current year
if (localTime.Year == 1)
{
localTime = new DateTime(DateTime.UtcNow.Year, localTime.Month, localTime.Day,
localTime.Hour, localTime.Minute, localTime.Second);
}
// The parsed timestamp includes both date and time (e.g., "23 Dec, 11:45 pm")
// localTime already contains the complete date and time from the parsed string
// We treat it as being in the target timezone (DateTimeKind.Unspecified), then convert to UTC
timestampUtc = TimeZoneInfo.ConvertTimeToUtc(localTime, timeZoneInfo);
Logger.LogDebug("Parsed frame timestamp '{TimestampStr}' with timezone '{TzAbbrev}' as {LocalTime} local ({UtcTime} UTC)",
timestampStr, timezoneAbbreviation ?? "default", localTime, timestampUtc);
return true;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr);
return false;
}
}
private async Task WaitForDisplayLabelToChangeAsync(IPage page, DateTime currentTimestamp, ScrapingContext? context = null, int maxWaitMs = 5000)
{
try
{
var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
{
var newTimestamp = await ExtractTimestampFromDisplayAsync(page, context);
if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp)
{
return;
}
await page.WaitForTimeoutAsync(200);
}
Logger.LogDebug("Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms", currentTimestamp, maxWaitMs);
}
catch (Exception ex)
{
Logger.LogDebug(ex, "Error waiting for display label to change");
}
}
private async Task DismissModalOverlaysAsync(IPage page) private async Task DismissModalOverlaysAsync(IPage page)
{ {
try try
+33 -6
View File
@@ -31,16 +31,43 @@ public class PauseRadarStep : BaseScrapingStep
var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton); var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton);
await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); // New spatial map often uses one step-through control; label may be on the button or in .bom-scrub-action__label
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true) string? buttonLabel = null;
try
{
var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First;
if (await labelLoc.CountAsync() > 0)
buttonLabel = (await labelLoc.TextContentAsync())?.Trim();
}
catch
{
/* use inner text */
}
buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim();
if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true)
{ {
Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name); Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name);
await playPauseButton.ClickAsync(); await playPauseButton.ClickAsync();
await context.Page.WaitForTimeoutAsync(300); await context.Page.WaitForTimeoutAsync(300);
buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); // Clear so post-click read always runs; otherwise stale "Pause" skips the ??= InnerText fallback when the label child is missing
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true) buttonLabel = null;
try
{
var labelLoc = playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).First;
if (await labelLoc.CountAsync() > 0)
buttonLabel = (await labelLoc.TextContentAsync())?.Trim();
}
catch
{
buttonLabel = null;
}
buttonLabel ??= (await playPauseButton.InnerTextAsync())?.Trim();
if (buttonLabel?.Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true)
{ {
Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name); Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name);
} }
@@ -1,99 +1,134 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces; using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping; using BomLocalService.Services.Scraping;
using BomLocalService.Utilities;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Scraping.Steps.Map; namespace BomLocalService.Services.Scraping.Steps.Map;
/// <summary>
/// Aligns the radar scrubber to the oldest frame (historical frame 0) without clicking timeline segments.
/// The BOM UI now often exposes a single step-through control; segment pills and <c>bom-scrub-tl</c> are unreliable.
/// Strategy: while paused, read <see cref="IScrubDisplayTimestampParser"/> from the display label, step forward
/// through one full loop to find the minimum time, then step until that time shows again.
/// </summary>
public class ResetToFirstFrameStep : BaseScrapingStep public class ResetToFirstFrameStep : BaseScrapingStep
{ {
private readonly IScrubDisplayTimestampParser _scrubDisplayTime;
public override string Name => "ResetToFirstFrame"; public override string Name => "ResetToFirstFrame";
public override string[] Prerequisites => new[] { "PauseRadar" }; public override string[] Prerequisites => new[] { "ExtractMetadata" };
public ResetToFirstFrameStep( public ResetToFirstFrameStep(
ILogger<ResetToFirstFrameStep> logger, ILogger<ResetToFirstFrameStep> logger,
ISelectorService selectorService, ISelectorService selectorService,
IDebugService debugService, IDebugService debugService,
IConfiguration configuration) IConfiguration configuration,
IScrubDisplayTimestampParser scrubDisplayTime)
: base(logger, selectorService, debugService, configuration) : base(logger, selectorService, debugService, configuration)
{ {
_scrubDisplayTime = scrubDisplayTime;
} }
public override bool CanExecute(ScrapingContext context) public override bool CanExecute(ScrapingContext context)
{ {
return context.CurrentState >= PageState.SlideshowPaused; return context.CurrentState >= PageState.SlideshowPaused
&& context.LastUpdatedInfo != null;
} }
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
{ {
try try
{ {
Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name); Logger.LogInformation("Step {Step}: Aligning to oldest frame via step-through + display label (no segment clicks)", Name);
try var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
var elementWaitMs = Configuration.GetValue<int?>("Scraping:Timeouts:ElementWait") ?? 10000;
var stepBtn = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton);
await stepBtn.WaitForAsync(new LocatorWaitForOptions { State = WaitForSelectorState.Visible, Timeout = elementWaitMs });
// Phase 1 — sample one full loop of display times while paused; oldest = frame 0 target
var samples = new List<DateTime>();
for (var i = 0; i < frameCount; i++)
{ {
// Try JavaScript click first to avoid thumb interception var t = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
var clicked = await context.Page.EvaluateAsync<bool>(@"() => { if (t.HasValue)
const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]'); samples.Add(t.Value);
if (segment) {
segment.click(); if (i == frameCount - 1)
return true; break;
var beforeStep = t;
await stepBtn.ClickAsync(new LocatorClickOptions { Force = true });
if (beforeStep.HasValue)
{
await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
context.Page, SelectorService, beforeStep.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken);
}
else
await context.Page.WaitForTimeoutAsync(800);
}
DateTime? frame0Utc = null;
if (samples.Count > 0)
{
frame0Utc = samples.Min();
Logger.LogInformation("Step {Step}: Oldest time in sampled loop (frame 0 target): {Utc:o} UTC", Name, frame0Utc);
}
else
{
Logger.LogWarning("Step {Step}: Could not read any display timestamps; tile timing may be wrong", Name);
}
// Phase 2 — step until the display matches the oldest sample
if (frame0Utc.HasValue)
{
const double matchMinutes = 2.0;
for (var attempt = 0; attempt < frameCount + 4; attempt++)
{
var cur = await _scrubDisplayTime.ReadUtcAsync(context.Page, SelectorService, context, cancellationToken);
if (cur.HasValue && Math.Abs((cur.Value - frame0Utc.Value).TotalMinutes) <= matchMinutes)
{
Logger.LogInformation("Step {Step}: Display matches frame 0 target (≤{MatchMinutes:F0} min)", Name, matchMinutes);
break;
} }
return false;
}"); await stepBtn.ClickAsync(new LocatorClickOptions { Force = true });
if (cur.HasValue)
if (clicked) {
{ await _scrubDisplayTime.WaitForDisplayLabelChangeAsync(
await context.Page.WaitForTimeoutAsync(500); context.Page, SelectorService, cur.Value, context, maxWaitMs: 8000, cancellationToken: cancellationToken);
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name); }
} else
else await context.Page.WaitForTimeoutAsync(800);
{
// Fallback to locator click with force
var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true });
await context.Page.WaitForTimeoutAsync(500);
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name);
} }
} }
catch (Exception ex)
{ await SaveDebugAsync(context, 9, "frame_0_aligned", cancellationToken);
Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name);
} // Best-effort only: segment-based checks are unreliable on the new spatial map layout
await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken);
// Verify scrubber is at position 0
Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name);
try try
{ {
var activeSegment = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment); var onSegmentZero = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment);
if (onSegmentZero)
if (activeSegment) Logger.LogDebug("Step {Step}: Segment check reports frame 0 (informational)", Name);
{
Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name);
}
else
{
Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name);
}
} }
catch (Exception ex) catch (Exception ex)
{ {
Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name); Logger.LogDebug(ex, "Step {Step}: Segment position check skipped", Name);
} }
await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken); await SaveDebugAsync(context, 10, "scrubber_frame0", cancellationToken);
// Wait for frame 0 tiles to fully render
var tileRenderWaitMsConfig = Configuration.GetValue<int?>("Screenshot:TileRenderWaitMs"); var tileRenderWaitMsConfig = Configuration.GetValue<int?>("Screenshot:TileRenderWaitMs");
if (!tileRenderWaitMsConfig.HasValue) if (!tileRenderWaitMsConfig.HasValue)
{ {
throw new InvalidOperationException("Screenshot:TileRenderWaitMs configuration is required. Set it in appsettings.json or via SCREENSHOT__TILERENDERWAITMS environment variable."); throw new InvalidOperationException("Screenshot:TileRenderWaitMs configuration is required. Set it in appsettings.json or via SCREENSHOT__TILERENDERWAITMS environment variable.");
} }
var tileRenderWaitMs = tileRenderWaitMsConfig.Value;
await context.Page.WaitForTimeoutAsync(tileRenderWaitMs); await context.Page.WaitForTimeoutAsync(tileRenderWaitMsConfig.Value);
context.CurrentState = PageState.Frame0Selected; context.CurrentState = PageState.Frame0Selected;
return ScrapingStepResult.Successful(); return ScrapingStepResult.Successful();
} }
@@ -105,4 +140,3 @@ public class ResetToFirstFrameStep : BaseScrapingStep
} }
} }
} }
@@ -1,15 +1,18 @@
using System.Text.Json;
using BomLocalService.Models;
using BomLocalService.Services.Interfaces; using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping; using BomLocalService.Services.Scraping;
using BomLocalService.Utilities;
namespace BomLocalService.Services.Scraping.Steps.Metadata; namespace BomLocalService.Services.Scraping.Steps.Metadata;
public class ExtractMetadataStep : BaseScrapingStep public class ExtractMetadataStep : BaseScrapingStep
{ {
private readonly ITimeParsingService _timeParsingService; private readonly ITimeParsingService _timeParsingService;
public override string Name => "ExtractMetadata"; public override string Name => "ExtractMetadata";
public override string[] Prerequisites => new[] { "ResetToFirstFrame" }; public override string[] Prerequisites => new[] { "PauseRadar" };
public ExtractMetadataStep( public ExtractMetadataStep(
ILogger<ExtractMetadataStep> logger, ILogger<ExtractMetadataStep> logger,
ISelectorService selectorService, ISelectorService selectorService,
@@ -20,41 +23,67 @@ public class ExtractMetadataStep : BaseScrapingStep
{ {
_timeParsingService = timeParsingService; _timeParsingService = timeParsingService;
} }
public override bool CanExecute(ScrapingContext context) public override bool CanExecute(ScrapingContext context)
{ {
return context.IsMapReady; return context.IsMapReady;
} }
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
{ {
try try
{ {
Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name); Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name);
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page); var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page);
context.LastUpdatedInfo = lastUpdatedInfo; context.LastUpdatedInfo = lastUpdatedInfo;
// Extract frame info var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
var defaults = Enumerable.Range(0, frameCount)
.Select(i => (index: i, minutesAgo: 40 - i * 5))
.ToList();
try try
{ {
var frameInfo = await context.Page.EvaluateAsync<object[]>(JavaScriptTemplates.ExtractFrameInfo); var element = await context.Page.EvaluateAsync<JsonElement>(JavaScriptTemplates.ExtractFrameInfo);
var result = new List<(int index, int minutesAgo)>(); if (element.ValueKind == JsonValueKind.Array && element.GetArrayLength() > 0)
for (int i = 0; i < 7; i++)
{ {
var minutesAgo = 40 - (i * 5); var byIndex = new Dictionary<int, int>();
result.Add((i, minutesAgo)); foreach (var item in element.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Object)
continue;
var idx = item.GetProperty("index").GetInt32();
if (item.TryGetProperty("minutesAgo", out var m) && m.ValueKind == JsonValueKind.Number)
byIndex[idx] = m.GetInt32();
}
if (byIndex.Count > 0)
{
context.FrameInfo = Enumerable.Range(0, frameCount)
.Select(i => (i, byIndex.TryGetValue(i, out var mm) ? mm : defaults[i].minutesAgo))
.ToList();
Logger.LogInformation("Step {Step}: Parsed frame offsets from segment aria-labels where available", Name);
}
else
{
context.FrameInfo = defaults;
Logger.LogInformation("Step {Step}: Segment times not parsed; using default ladder", Name);
}
}
else
{
context.FrameInfo = defaults;
Logger.LogInformation("Step {Step}: No segment array from page; using default ladder", Name);
} }
context.FrameInfo = result;
} }
catch (Exception ex) catch (Exception ex)
{ {
Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name); Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name);
context.FrameInfo = Enumerable.Range(0, 7) context.FrameInfo = defaults;
.Select(i => (i, 40 - (i * 5)))
.ToList();
} }
return ScrapingStepResult.Successful(); return ScrapingStepResult.Successful();
} }
catch (Exception ex) catch (Exception ex)
@@ -65,4 +94,3 @@ public class ExtractMetadataStep : BaseScrapingStep
} }
} }
} }
@@ -28,8 +28,8 @@ public class RadarScrapingWorkflow : IRadarScrapingWorkflow
"ClickRadarLink", "ClickRadarLink",
"WaitForMapReady", "WaitForMapReady",
"PauseRadar", "PauseRadar",
"ResetToFirstFrame",
"ExtractMetadata", "ExtractMetadata",
"ResetToFirstFrame",
"CalculateMapBounds", "CalculateMapBounds",
"CaptureFrames" "CaptureFrames"
}; };
+11 -3
View File
@@ -52,9 +52,17 @@ public class SelectorService : ISelectorService
{ {
throw new ArgumentException($"No selectors configured for {config.Name}"); throw new ArgumentException($"No selectors configured for {config.Name}");
} }
// Returns first selector as locator (for cases where we know it exists) // Match any configured selector (BOM occasionally renames test ids; list is tried in order)
return page.Locator(config.Selectors[0]).First; var combined = page.Locator(config.Selectors[0]).First;
for (var i = 1; i < config.Selectors.Length; i++)
{
combined = combined.Or(page.Locator(config.Selectors[i]).First);
}
// Or() unions alternatives; BOM sometimes renders duplicates (e.g. responsive + hidden).
// Pin to a single element so Playwright strict mode does not reject clicks.
return combined.First;
} }
} }
+27 -19
View File
@@ -41,26 +41,30 @@ public class TimeParsingService : ITimeParsingService
{ {
try try
{ {
// Look for the weather metadata section using the correct selector // BOM layouts: legacy map used inner divs; location/spatial pages often use C07_WeatherMetadata with
// HTML structure: <section data-testid="weatherMetadata"> with aria-label="Last updated" // a single section and visible text in the section (no div split). Prefer normalized innerText of the
var metadataSection = page.Locator("section[data-testid='weatherMetadata'], section[aria-label='Last updated']").First; // section so both shapes parse the same.
var metadataSection = page.Locator(
// Extract text from each div separately to preserve structure "section[data-testid='weatherMetadata'], " +
// Structure: <div>Observations: X minutes ago, 9:40 pm AEST</div><div>at Gympie...</div><div>Forecast: ...</div> "section[aria-label='Last updated'], " +
"section[data-component='C07_WeatherMetadata']").First;
var lastUpdatedText = await page.EvaluateAsync<string>(@"() => { var lastUpdatedText = await page.EvaluateAsync<string>(@"() => {
const section = document.querySelector('section[data-testid=""weatherMetadata""]') || const section =
document.querySelector('section[aria-label=""Last updated""]'); document.querySelector('section[data-testid=""weatherMetadata""]') ||
document.querySelector('section[aria-label=""Last updated""]') ||
document.querySelector('section[data-component=""C07_WeatherMetadata""]');
if (!section) return null; if (!section) return null;
const raw = section.innerText || section.textContent || '';
// Get all divs and join with newlines to preserve structure return raw.replace(/\s+/g, ' ').trim();
const divs = section.querySelectorAll('div');
return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
}"); }");
if (string.IsNullOrEmpty(lastUpdatedText)) if (string.IsNullOrEmpty(lastUpdatedText))
{ {
// Fallback: get all text content var fallback = await metadataSection.InnerTextAsync();
lastUpdatedText = await metadataSection.TextContentAsync(); lastUpdatedText = string.IsNullOrEmpty(fallback)
? null
: Regex.Replace(fallback, @"\s+", " ").Trim();
} }
return ParseLastUpdatedText(lastUpdatedText ?? string.Empty); return ParseLastUpdatedText(lastUpdatedText ?? string.Empty);
@@ -82,8 +86,10 @@ public class TimeParsingService : ITimeParsingService
_logger.LogInformation("Parsing last updated text: {Text}", text); _logger.LogInformation("Parsing last updated text: {Text}", text);
// Parse observation time from time string (e.g., "9:40 pm AEST") // Parse observation time from time string (e.g., "9:40 pm AEST")
// Ignore "minutes ago" - client will calculate this dynamically from UTC timestamp // Ignore relative prefix — BOM uses "a minute ago", "6 minutes ago", "an hour ago", "2 hours ago", etc.
var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase); var observationMatch = Regex.Match(text,
@"Observations:\s*(?:a\s+minute\s+ago|\d+\s*minutes?\s+ago|an\s+hour\s+ago|\d+\s*hours?\s+ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)",
RegexOptions.IgnoreCase);
if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success) if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
{ {
var timeStr = observationMatch.Groups[1].Value.Trim(); var timeStr = observationMatch.Groups[1].Value.Trim();
@@ -114,8 +120,10 @@ public class TimeParsingService : ITimeParsingService
} }
// Parse forecast time from time string (e.g., "7:50 pm AEST") // Parse forecast time from time string (e.g., "7:50 pm AEST")
// Ignore "minutes ago" - client will calculate this dynamically from UTC timestamp // Ignore relative prefix (minutes/hours ago) — client derives "ago" from the UTC timestamp
var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:an\s+hour\s+ago|\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase); var forecastMatch = Regex.Match(text,
@"Forecast:\s*(?:\d+\s*hours?\s+ago|an\s+hour\s+ago|\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)",
RegexOptions.IgnoreCase);
if (forecastMatch.Success && forecastMatch.Groups[1].Success) if (forecastMatch.Success && forecastMatch.Groups[1].Success)
{ {
var timeStr = forecastMatch.Groups[1].Value.Trim(); var timeStr = forecastMatch.Groups[1].Value.Trim();
+15 -10
View File
@@ -61,11 +61,9 @@
"Name": "Search Button", "Name": "Search Button",
"Selectors": [ "Selectors": [
"button[data-testid='searchLabel']", "button[data-testid='searchLabel']",
"button[aria-label='Search for a location']",
"button.search-location__trigger-button", "button.search-location__trigger-button",
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))", "button[aria-label='Search for a location'].search-location__trigger-button",
"button:has-text('Search for a location')", "button.search-location__trigger-button.bom-button:has(span.bom-button__label:has-text('Search for a location'))"
"[data-testid='showSearchModal'] button"
], ],
"TimeoutMs": 10000, "TimeoutMs": 10000,
"Required": true, "Required": true,
@@ -137,7 +135,10 @@
}, },
"PlayPauseButton": { "PlayPauseButton": {
"Name": "Play Pause Button", "Name": "Play Pause Button",
"Selectors": ["button[data-testid='bom-time-scrub-play-pause']"], "Selectors": [
"button[data-testid='bom-time-scrub-step-through']",
"button[data-testid='bom-time-scrub-play-pause']"
],
"TimeoutMs": 5000, "TimeoutMs": 5000,
"Required": true "Required": true
}, },
@@ -155,7 +156,11 @@
}, },
"StepForwardButton": { "StepForwardButton": {
"Name": "Step Forward Button", "Name": "Step Forward Button",
"Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"], "Selectors": [
"button[data-testid='bom-time-scrub-step-through']",
"button.bom-scrub-action--step-through",
"button[data-testid='bom-scrub-utils__right__step-forward']"
],
"TimeoutMs": 5000, "TimeoutMs": 5000,
"Required": true "Required": true
}, },
@@ -184,9 +189,9 @@
"ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }", "ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }",
"WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }", "WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }",
"WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }", "WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }",
"CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }", "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); if (segments.length === 0) return true; const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
"WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }", "WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }",
"ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }", "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]+),?\\s+(\\d{1,2}):(\\d{2})\\s*(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
"CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }", "CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }",
"CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }", "CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }",
"GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })", "GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })",
@@ -194,7 +199,7 @@
}, },
"TextPatterns": { "TextPatterns": {
"ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)", "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)", "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]+,?\\s+\\d{1,2}:\\d{2}\\s*(?:am|pm)",
"ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago", "ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago",
"ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago", "ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago",
"ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago", "ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago",
@@ -218,8 +223,8 @@
"ClickRadarLink": { "Enabled": true }, "ClickRadarLink": { "Enabled": true },
"WaitForMapReady": { "Enabled": true }, "WaitForMapReady": { "Enabled": true },
"PauseRadar": { "Enabled": true }, "PauseRadar": { "Enabled": true },
"ResetToFirstFrame": { "Enabled": true },
"ExtractMetadata": { "Enabled": true }, "ExtractMetadata": { "Enabled": true },
"ResetToFirstFrame": { "Enabled": true },
"CalculateMapBounds": { "Enabled": true }, "CalculateMapBounds": { "Enabled": true },
"CaptureFrames": { "CaptureFrames": {
"Enabled": true, "Enabled": true,