Remove minutes ago parsing, use UTC timestamps directly

- Refactor TimeParsingService to parse time strings to UTC only
- Remove minutesAgo from RadarFrame and FrameMetadata models
- Update CaptureFramesStep to extract UTC timestamps directly
- Update CacheService to store/load ObservationTime instead of MinutesAgo
- Update BomRadarService to use AbsoluteObservationTime directly
- Remove all backward compatibility code for old cache folders
- Update documentation and test SPA to calculate minutes ago on client
- Fix step registration to register concrete types for DI resolution
- Fix hosted service registration to use reflection correctly
This commit is contained in:
2025-12-23 21:52:42 +10:00
parent b30d76ddec
commit ffd3636d1c
31 changed files with 598 additions and 372 deletions
+259
View File
@@ -0,0 +1,259 @@
using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Interfaces.Registration;
using BomLocalService.Services.Scraping;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using System.Reflection;
namespace BomLocalService.Extensions;
public static class ServiceCollectionExtensions
{
/// <summary>
/// Configures CORS with settings from configuration.
/// Configuration values come from appsettings.json (defaults) and can be overridden via environment variables.
/// Environment variables use double underscore for nested keys (e.g., CORS__ALLOWEDORIGINS).
/// </summary>
public static IServiceCollection AddCorsConfiguration(this IServiceCollection services, IConfiguration configuration)
{
var corsOrigins = configuration.GetValue<string>("Cors:AllowedOrigins")
?? throw new InvalidOperationException("Cors:AllowedOrigins configuration is required. Set it in appsettings.json or via CORS__ALLOWEDORIGINS environment variable.");
var corsMethods = configuration.GetValue<string>("Cors:AllowedMethods")
?? throw new InvalidOperationException("Cors:AllowedMethods configuration is required. Set it in appsettings.json or via CORS__ALLOWEDMETHODS environment variable.");
var corsHeaders = configuration.GetValue<string>("Cors:AllowedHeaders")
?? throw new InvalidOperationException("Cors:AllowedHeaders configuration is required. Set it in appsettings.json or via CORS__ALLOWEDHEADERS environment variable.");
// For bool, check if the key exists in configuration (GetValue<bool> returns false if not found, which is ambiguous)
var corsAllowCredentialsKey = configuration["Cors:AllowCredentials"];
if (corsAllowCredentialsKey == null)
{
throw new InvalidOperationException("Cors:AllowCredentials configuration is required. Set it in appsettings.json or via CORS__ALLOWCREDENTIALS environment variable.");
}
var corsAllowCredentials = configuration.GetValue<bool>("Cors:AllowCredentials");
services.AddCors(options =>
{
options.AddDefaultPolicy(policy =>
{
if (corsOrigins == "*")
{
policy.AllowAnyOrigin();
}
else
{
// Split comma-separated origins
var origins = corsOrigins.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithOrigins(origins);
}
// Split comma-separated methods
var methods = corsMethods.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithMethods(methods);
// Split comma-separated headers or allow all
if (corsHeaders == "*")
{
policy.AllowAnyHeader();
}
else
{
var headers = corsHeaders.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithHeaders(headers);
}
if (corsAllowCredentials)
{
policy.AllowCredentials();
}
});
});
return services;
}
/// <summary>
/// Scans the assembly for services implementing registration interfaces and registers them automatically.
/// Services are registered as their concrete interface (that inherits from registration interface).
/// Properly handles generic interfaces by using concrete interfaces that hide the generics.
/// </summary>
public static IServiceCollection ScanAndRegisterServices(this IServiceCollection services, Assembly assembly)
{
var registeredTypes = new HashSet<Type>(); // Track registered types to avoid duplicates
// Get all types from the assembly with improved filtering
var types = GetRegisterableTypes(assembly);
// Register Singleton services
RegisterServicesByLifetime<ISingletonService>(
services,
types,
registeredTypes);
// Register Scoped services
RegisterServicesByLifetime<IScopedService>(
services,
types,
registeredTypes);
// Register Transient services
RegisterServicesByLifetime<ITransientService>(
services,
types,
registeredTypes);
// Register Hosted Services
RegisterHostedServices(services, types, registeredTypes);
return services;
}
/// <summary>
/// Gets all types that should be considered for registration.
/// Filters out abstract classes, interfaces, generic type definitions, nested private types,
/// compiler-generated types, and types from excluded namespaces.
/// </summary>
private static List<Type> GetRegisterableTypes(Assembly assembly)
{
return assembly.GetTypes()
.Where(t =>
// Must be a class (not interface, struct, enum, etc.)
t.IsClass
// Must be concrete (not abstract)
&& !t.IsAbstract
// Must not be a generic type definition (but closed generics are OK)
&& !t.IsGenericTypeDefinition
// Must be public (or nested public in a public type)
&& (t.IsPublic || (t.IsNestedPublic && t.DeclaringType?.IsPublic == true))
// Must not be compiler-generated (e.g., async state machines, iterator classes)
&& !t.IsDefined(typeof(System.Runtime.CompilerServices.CompilerGeneratedAttribute), inherit: false)
// Must not be a nested private/internal type
&& !(t.IsNested && !t.IsNestedPublic)
// Exclude test namespaces if any (optional - adjust as needed)
&& !t.Namespace?.StartsWith("BomLocalService.Tests", StringComparison.Ordinal) == true
)
.ToList();
}
private static void RegisterServicesByLifetime<TRegistrationInterface>(
IServiceCollection services,
List<Type> types,
HashSet<Type> registeredTypes)
where TRegistrationInterface : class
{
foreach (var implementationType in types)
{
// Skip if already registered
if (registeredTypes.Contains(implementationType))
continue;
// Get all interfaces implemented by this type
var allInterfaces = implementationType.GetInterfaces().ToList();
// Find interfaces that inherit from TRegistrationInterface (but not the registration interface itself)
var registrationInterfaces = allInterfaces
.Where(i => typeof(TRegistrationInterface).IsAssignableFrom(i)
&& i != typeof(TRegistrationInterface)
&& !IsGenericTypeDefinition(i))
.ToList();
if (registrationInterfaces.Count == 0)
continue;
// For each registration interface, find the most specific concrete interface
// Priority: concrete interfaces (like IRadarScrapingWorkflow) over generic interfaces (like IWorkflow<T>)
foreach (var registrationInterface in registrationInterfaces)
{
// Skip if this is a generic interface definition (we prefer concrete interfaces)
if (registrationInterface.IsGenericTypeDefinition)
continue;
// Check if this interface is already registered to avoid duplicates
var alreadyRegistered = services.Any(s =>
s.ServiceType == registrationInterface &&
s.ImplementationType == implementationType);
if (alreadyRegistered)
continue;
// Determine lifetime based on registration interface
var lifetime = GetLifetime<TRegistrationInterface>();
// Register the service by interface
if (lifetime == ServiceLifetime.Singleton)
{
services.AddSingleton(registrationInterface, implementationType);
// Also register as concrete type for direct resolution (e.g., for ScrapingStepRegistry)
services.AddSingleton(implementationType);
}
else if (lifetime == ServiceLifetime.Scoped)
{
services.AddScoped(registrationInterface, implementationType);
services.AddScoped(implementationType);
}
else if (lifetime == ServiceLifetime.Transient)
{
services.AddTransient(registrationInterface, implementationType);
services.AddTransient(implementationType);
}
registeredTypes.Add(implementationType);
}
}
}
private static bool IsGenericTypeDefinition(Type type)
{
return type.IsGenericTypeDefinition ||
(type.IsGenericType && type.GetGenericTypeDefinition() == type);
}
private static ServiceLifetime GetLifetime<TRegistrationInterface>()
where TRegistrationInterface : class
{
if (typeof(ISingletonService).IsAssignableFrom(typeof(TRegistrationInterface)))
return ServiceLifetime.Singleton;
if (typeof(IScopedService).IsAssignableFrom(typeof(TRegistrationInterface)))
return ServiceLifetime.Scoped;
if (typeof(ITransientService).IsAssignableFrom(typeof(TRegistrationInterface)))
return ServiceLifetime.Transient;
throw new InvalidOperationException($"Unknown registration interface: {typeof(TRegistrationInterface).Name}");
}
private static void RegisterHostedServices(
IServiceCollection services,
List<Type> types,
HashSet<Type> registeredTypes)
{
foreach (var type in types)
{
// Skip if already registered
if (registeredTypes.Contains(type))
continue;
// Check if it's a BackgroundService that implements IHostedServiceRegistration
if (typeof(Microsoft.Extensions.Hosting.BackgroundService).IsAssignableFrom(type) &&
typeof(IHostedServiceRegistration).IsAssignableFrom(type))
{
// Check if already registered
var alreadyRegistered = services.Any(s =>
s.ServiceType == typeof(Microsoft.Extensions.Hosting.IHostedService) &&
s.ImplementationType == type);
if (!alreadyRegistered)
{
// Use AddHostedService via reflection - this is the recommended way
// It properly handles lifecycle management for hosted services
// AddHostedService<T> is generic, so we need to call it via reflection
var addHostedServiceMethod = typeof(Microsoft.Extensions.DependencyInjection.ServiceCollectionHostedServiceExtensions)
.GetMethod(nameof(Microsoft.Extensions.DependencyInjection.ServiceCollectionHostedServiceExtensions.AddHostedService),
new[] { typeof(IServiceCollection) })!
.MakeGenericMethod(type);
addHostedServiceMethod.Invoke(null, new object[] { services });
registeredTypes.Add(type);
}
}
}
}
}
+3 -2
View File
@@ -11,8 +11,9 @@ public class FrameMetadata
public int FrameIndex { get; set; } public int FrameIndex { get; set; }
/// <summary> /// <summary>
/// Number of minutes ago this frame represents (40, 35, 30, 25, 20, 15, 10). /// The absolute UTC observation time for this frame.
/// Parsed directly from the frame display label during capture.
/// </summary> /// </summary>
public int MinutesAgo { get; set; } public DateTime ObservationTime { get; set; }
} }
+2 -10
View File
@@ -2,22 +2,14 @@ namespace BomLocalService.Models;
/// <summary> /// <summary>
/// Represents a single radar frame (historical precipitation data). /// Represents a single radar frame (historical precipitation data).
/// Frame 0 is oldest (40 minutes ago), Frame 6 is newest (10 minutes ago).
/// Radar frames can be joined across cache folders to create extended historical slideshows. /// Radar frames can be joined across cache folders to create extended historical slideshows.
/// </summary> /// </summary>
public class RadarFrame : CachedFrame public class RadarFrame : CachedFrame
{ {
/// <summary>
/// Number of minutes ago this frame represents (40, 35, 30, 25, 20, 15, 10).
/// Frame 0 = 40 minutes ago, Frame 6 = 10 minutes ago.
/// This is relative to the cache folder's observation time.
/// </summary>
public int MinutesAgo { get; set; }
/// <summary> /// <summary>
/// The absolute UTC observation time for this frame. /// The absolute UTC observation time for this frame.
/// Calculated as: ObservationTime - MinutesAgo. /// Parsed directly from the frame display label during capture.
/// This is set when frames are joined across cache folders in timeseries responses. /// Client calculates "minutes ago" dynamically from this timestamp.
/// </summary> /// </summary>
public DateTime? AbsoluteObservationTime { get; set; } public DateTime? AbsoluteObservationTime { get; set; }
+2 -1
View File
@@ -8,7 +8,8 @@ public class RadarResponse
{ {
/// <summary> /// <summary>
/// List of all captured frames (typically 7 frames: 0-6). /// List of all captured frames (typically 7 frames: 0-6).
/// Frame 0 is oldest (40 minutes ago), Frame 6 is newest (10 minutes ago). /// Each frame contains an absoluteObservationTime UTC timestamp.
/// Client should calculate "minutes ago" dynamically from the timestamp.
/// </summary> /// </summary>
public List<RadarFrame> Frames { get; set; } = new(); public List<RadarFrame> Frames { get; set; } = new();
+4 -118
View File
@@ -1,12 +1,6 @@
using BomLocalService.Services; using BomLocalService.Extensions;
using BomLocalService.Services.Interfaces; using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Scraping; using BomLocalService.Services.Scraping;
using BomLocalService.Services.Scraping.Steps.Navigation;
using BomLocalService.Services.Scraping.Steps.Search;
using BomLocalService.Services.Scraping.Steps.Map;
using BomLocalService.Services.Scraping.Steps.Metadata;
using BomLocalService.Services.Scraping.Steps.Capture;
using BomLocalService.Services.Scraping.Workflows;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
@@ -17,100 +11,10 @@ builder.Services.AddEndpointsApiExplorer();
builder.Services.AddHealthChecks(); builder.Services.AddHealthChecks();
// Configure CORS - MUST be added before other services // Configure CORS - MUST be added before other services
// Configuration values come from appsettings.json (defaults) and can be overridden via environment variables builder.Services.AddCorsConfiguration(builder.Configuration);
// Environment variables use double underscore for nested keys (e.g., CORS__ALLOWEDORIGINS)
var corsOrigins = builder.Configuration.GetValue<string>("Cors:AllowedOrigins")
?? throw new InvalidOperationException("Cors:AllowedOrigins configuration is required. Set it in appsettings.json or via CORS__ALLOWEDORIGINS environment variable.");
var corsMethods = builder.Configuration.GetValue<string>("Cors:AllowedMethods")
?? throw new InvalidOperationException("Cors:AllowedMethods configuration is required. Set it in appsettings.json or via CORS__ALLOWEDMETHODS environment variable.");
var corsHeaders = builder.Configuration.GetValue<string>("Cors:AllowedHeaders")
?? throw new InvalidOperationException("Cors:AllowedHeaders configuration is required. Set it in appsettings.json or via CORS__ALLOWEDHEADERS environment variable.");
// For bool, check if the key exists in configuration (GetValue<bool> returns false if not found, which is ambiguous) // Register all services via assembly scanning
var corsAllowCredentialsKey = builder.Configuration["Cors:AllowCredentials"]; builder.Services.ScanAndRegisterServices(typeof(Program).Assembly);
if (corsAllowCredentialsKey == null)
{
throw new InvalidOperationException("Cors:AllowCredentials configuration is required. Set it in appsettings.json or via CORS__ALLOWCREDENTIALS environment variable.");
}
var corsAllowCredentials = builder.Configuration.GetValue<bool>("Cors:AllowCredentials");
builder.Services.AddCors(options =>
{
options.AddDefaultPolicy(policy =>
{
if (corsOrigins == "*")
{
policy.AllowAnyOrigin();
}
else
{
// Split comma-separated origins
var origins = corsOrigins.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithOrigins(origins);
}
// Split comma-separated methods
var methods = corsMethods.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithMethods(methods);
// Split comma-separated headers or allow all
if (corsHeaders == "*")
{
policy.AllowAnyHeader();
}
else
{
var headers = corsHeaders.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
policy.WithHeaders(headers);
}
if (corsAllowCredentials)
{
policy.AllowCredentials();
}
});
});
// Register core services via interfaces (order matters - dependencies must be registered first)
builder.Services.AddSingleton<IDebugService, DebugService>();
builder.Services.AddSingleton<ICacheService, CacheService>();
builder.Services.AddSingleton<ITimeParsingService, TimeParsingService>();
builder.Services.AddSingleton<IBrowserService, BrowserService>();
builder.Services.AddSingleton<ISelectorService, SelectorService>();
// Register scraping step registry
builder.Services.AddSingleton<IScrapingStepRegistry, ScrapingStepRegistry>();
// Register all scraping steps
builder.Services.AddScoped<NavigateHomepageStep>();
builder.Services.AddScoped<ClickSearchButtonStep>();
builder.Services.AddScoped<FillSearchInputStep>();
builder.Services.AddScoped<WaitForSearchResultsStep>();
builder.Services.AddScoped<SelectSearchResultStep>();
builder.Services.AddScoped<ClickRadarLinkStep>();
builder.Services.AddScoped<WaitForMapReadyStep>();
builder.Services.AddScoped<PauseRadarStep>();
builder.Services.AddScoped<ResetToFirstFrameStep>();
builder.Services.AddScoped<ExtractMetadataStep>();
builder.Services.AddScoped<CalculateMapBoundsStep>();
builder.Services.AddScoped<CaptureFramesStep>();
// Register workflows
builder.Services.AddScoped<RadarScrapingWorkflow>();
builder.Services.AddScoped<TemperatureMapWorkflow>();
// Register workflow factory
builder.Services.AddSingleton<IWorkflowFactory, WorkflowFactory>();
// Register scraping service (depends on workflow factory)
builder.Services.AddSingleton<IScrapingService, ScrapingService>();
// Register BOM Radar Service as singleton (orchestrator, depends on all above services)
builder.Services.AddSingleton<IBomRadarService, BomLocalService.Services.BomRadarService>();
// Register background services (order matters - management service needs radar service)
builder.Services.AddHostedService<CacheCleanupService>();
builder.Services.AddHostedService<CacheManagementService>();
var app = builder.Build(); var app = builder.Build();
@@ -142,24 +46,6 @@ app.MapControllers();
// Map health check endpoint for Docker health monitoring // Map health check endpoint for Docker health monitoring
app.MapHealthChecks("/api/health"); app.MapHealthChecks("/api/health");
// Auto-register all scraping steps in the registry
var stepRegistry = app.Services.GetRequiredService<IScrapingStepRegistry>();
var stepTypes = typeof(IScrapingStep).Assembly.GetTypes()
.Where(t => typeof(IScrapingStep).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract && !t.IsGenericType);
foreach (var stepType in stepTypes)
{
try
{
var step = (IScrapingStep)ActivatorUtilities.CreateInstance(app.Services, stepType);
stepRegistry.RegisterStep(step);
}
catch (Exception ex)
{
var logger = app.Services.GetRequiredService<ILogger<Program>>();
logger.LogWarning(ex, "Failed to register step {StepType}", stepType.Name);
}
}
// Cleanup incomplete cache folders from previous crashes/restarts before starting services // Cleanup incomplete cache folders from previous crashes/restarts before starting services
var cacheService = app.Services.GetRequiredService<ICacheService>(); var cacheService = app.Services.GetRequiredService<ICacheService>();
var deletedCount = cacheService.CleanupIncompleteCacheFolders(); var deletedCount = cacheService.CleanupIncompleteCacheFolders();
+7 -6
View File
@@ -523,8 +523,7 @@ GET /api/radar/{suburb}/{state}
{ {
"frameIndex": 0, "frameIndex": 0,
"imageUrl": "/api/radar/Brisbane/QLD/frame/0", "imageUrl": "/api/radar/Brisbane/QLD/frame/0",
"minutesAgo": 0, "absoluteObservationTime": "2025-01-15T10:00:00Z"
"observationTime": "2025-01-15T10:00:00Z"
} }
], ],
"observationTime": "2025-01-15T10:00:00Z", "observationTime": "2025-01-15T10:00:00Z",
@@ -539,7 +538,7 @@ GET /api/radar/{suburb}/{state}
``` ```
**Response Fields:** **Response Fields:**
- `frames`: Array of radar frame objects with image URLs and timing information - `frames`: Array of radar frame objects with image URLs. Each frame contains `absoluteObservationTime` (UTC timestamp). Client should calculate "minutes ago" dynamically from this timestamp.
- `observationTime`: UTC timestamp when the observation was made - `observationTime`: UTC timestamp when the observation was made
- `forecastTime`: UTC timestamp for the forecast - `forecastTime`: UTC timestamp for the forecast
- `weatherStation`: Name of the weather station - `weatherStation`: Name of the weather station
@@ -669,8 +668,7 @@ GET /api/radar/{suburb}/{state}/timeseries?startTime={iso8601}&endTime={iso8601}
{ {
"frameIndex": 0, "frameIndex": 0,
"imageUrl": "/api/radar/Brisbane/QLD/frame/0?cacheFolder=Brisbane_QLD_20250115_100000", "imageUrl": "/api/radar/Brisbane/QLD/frame/0?cacheFolder=Brisbane_QLD_20250115_100000",
"absoluteObservationTime": "2025-01-15T10:00:00Z", "absoluteObservationTime": "2025-01-15T10:00:00Z"
"minutesAgo": 0
} }
] ]
} }
@@ -1008,7 +1006,10 @@ if (radarData.frames && radarData.frames.length > 0) {
// Or loop through all frames for animation // Or loop through all frames for animation
radarData.frames.forEach((frame, index) => { radarData.frames.forEach((frame, index) => {
console.log(`Frame ${index}: ${frame.imageUrl} (${frame.minutesAgo} min ago)`); const minutesAgo = frame.absoluteObservationTime
? Math.round((Date.now() - new Date(frame.absoluteObservationTime).getTime()) / 60000)
: null;
console.log(`Frame ${index}: ${frame.imageUrl}${minutesAgo !== null ? ` (${minutesAgo} min ago)` : ''}`);
}); });
} }
``` ```
+12 -17
View File
@@ -519,32 +519,27 @@ public class BomRadarService : IBomRadarService, IDisposable
if (radarFrames.Count > 0) if (radarFrames.Count > 0)
{ {
// Generate URLs and calculate absolute observation times for each frame // Generate URLs and filter frames by unique absolute observation times
var uniqueFrames = new List<RadarFrame>(); var uniqueFrames = new List<RadarFrame>();
foreach (var frame in radarFrames) foreach (var frame in radarFrames)
{ {
frame.ImageUrl = $"/api/radar/{encodedSuburb}/{encodedState}/frame/{frame.FrameIndex}?cacheFolder={Uri.EscapeDataString(folderInfo.FolderName)}"; frame.ImageUrl = $"/api/radar/{encodedSuburb}/{encodedState}/frame/{frame.FrameIndex}?cacheFolder={Uri.EscapeDataString(folderInfo.FolderName)}";
// Calculate absolute observation time for this frame // Use absolute observation time directly (already set during capture)
// minutesAgo is relative to the cache folder's observation time // Only include frames with unique absolute observation times
// Validate that ObservationTime is reasonable (not default/min value) // Since we process folders newest-first, this ensures we keep frames from the most recent cache folder
if (folderInfo.ObservationTime > DateTime.MinValue.AddYears(1) && frame.MinutesAgo >= 0) if (frame.AbsoluteObservationTime.HasValue &&
frame.AbsoluteObservationTime.Value > DateTime.MinValue.AddYears(1) &&
!seenAbsoluteTimes.Contains(frame.AbsoluteObservationTime.Value))
{ {
frame.AbsoluteObservationTime = folderInfo.ObservationTime.AddMinutes(-frame.MinutesAgo); seenAbsoluteTimes.Add(frame.AbsoluteObservationTime.Value);
uniqueFrames.Add(frame);
// Only include frames with unique absolute observation times
// Since we process folders newest-first, this ensures we keep frames from the most recent cache folder
if (frame.AbsoluteObservationTime.HasValue && !seenAbsoluteTimes.Contains(frame.AbsoluteObservationTime.Value))
{
seenAbsoluteTimes.Add(frame.AbsoluteObservationTime.Value);
uniqueFrames.Add(frame);
}
} }
else else if (!frame.AbsoluteObservationTime.HasValue)
{ {
_logger.LogWarning("Skipping frame {FrameIndex} from folder {FolderName}: Invalid ObservationTime ({ObservationTime}) or MinutesAgo ({MinutesAgo})", _logger.LogWarning("Skipping frame {FrameIndex} from folder {FolderName}: Missing AbsoluteObservationTime",
frame.FrameIndex, folderInfo.FolderName, folderInfo.ObservationTime, frame.MinutesAgo); frame.FrameIndex, folderInfo.FolderName);
} }
} }
+2 -1
View File
@@ -1,9 +1,10 @@
using BomLocalService.Services.Interfaces.Registration;
using BomLocalService.Utilities; using BomLocalService.Utilities;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
namespace BomLocalService.Services; namespace BomLocalService.Services;
public class CacheCleanupService : BackgroundService public class CacheCleanupService : BackgroundService, IHostedServiceRegistration
{ {
private readonly ILogger<CacheCleanupService> _logger; private readonly ILogger<CacheCleanupService> _logger;
private readonly string _cacheDirectory; private readonly string _cacheDirectory;
+2 -1
View File
@@ -1,10 +1,11 @@
using BomLocalService.Services.Interfaces; using BomLocalService.Services.Interfaces;
using BomLocalService.Services.Interfaces.Registration;
using BomLocalService.Utilities; using BomLocalService.Utilities;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
namespace BomLocalService.Services; namespace BomLocalService.Services;
public class CacheManagementService : BackgroundService public class CacheManagementService : BackgroundService, IHostedServiceRegistration
{ {
private readonly ILogger<CacheManagementService> _logger; private readonly ILogger<CacheManagementService> _logger;
private readonly IBomRadarService _bomRadarService; private readonly IBomRadarService _bomRadarService;
+10 -11
View File
@@ -154,18 +154,18 @@ public class CacheService : ICacheService
return null; return null;
} }
// Load frame metadata to get accurate minutesAgo // Load frame metadata to get observation time
var framesMetadata = await LoadFramesMetadataAsync(cacheFolderPath, CachedDataType.Radar, cancellationToken); var framesMetadata = await LoadFramesMetadataAsync(cacheFolderPath, CachedDataType.Radar, cancellationToken);
var frameMetadata = framesMetadata.FirstOrDefault(f => f.FrameIndex == frameIndex); var frameMetadata = framesMetadata.FirstOrDefault(f => f.FrameIndex == frameIndex);
var minutesAgo = frameMetadata != null var observationTime = frameMetadata?.ObservationTime > DateTime.MinValue.AddYears(1)
? frameMetadata.MinutesAgo ? frameMetadata.ObservationTime
: 40 - (frameIndex * 5); : (DateTime?)null;
return new RadarFrame return new RadarFrame
{ {
FrameIndex = frameIndex, FrameIndex = frameIndex,
ImagePath = framePath, ImagePath = framePath,
MinutesAgo = minutesAgo AbsoluteObservationTime = observationTime
}; };
} }
@@ -200,7 +200,7 @@ public class CacheService : ICacheService
var framesMetadata = frames.Select(f => new FrameMetadata var framesMetadata = frames.Select(f => new FrameMetadata
{ {
FrameIndex = f.FrameIndex, FrameIndex = f.FrameIndex,
MinutesAgo = f.MinutesAgo ObservationTime = f.AbsoluteObservationTime ?? throw new InvalidOperationException($"Frame {f.FrameIndex} missing AbsoluteObservationTime")
}).ToList(); }).ToList();
var framesPath = FilePathHelper.GetFramesMetadataFilePath(cacheFolderPath, dataType); var framesPath = FilePathHelper.GetFramesMetadataFilePath(cacheFolderPath, dataType);
@@ -424,7 +424,8 @@ public class CacheService : ICacheService
// Load frame metadata from frames.json if available // Load frame metadata from frames.json if available
var framesMetadata = await LoadFramesMetadataAsync(folderPath, dataType, cancellationToken); var framesMetadata = await LoadFramesMetadataAsync(folderPath, dataType, cancellationToken);
var metadataDict = framesMetadata.ToDictionary(f => f.FrameIndex, f => f.MinutesAgo); var metadataDict = framesMetadata.Where(f => f.ObservationTime > DateTime.MinValue.AddYears(1))
.ToDictionary(f => f.FrameIndex, f => f.ObservationTime);
// Load frames from data type subfolder // Load frames from data type subfolder
for (int i = 0; i < frameCount; i++) for (int i = 0; i < frameCount; i++)
@@ -434,15 +435,13 @@ public class CacheService : ICacheService
{ {
if (dataType == CachedDataType.Radar) if (dataType == CachedDataType.Radar)
{ {
var minutesAgo = metadataDict.ContainsKey(i) var observationTime = metadataDict.ContainsKey(i) ? metadataDict[i] : (DateTime?)null;
? metadataDict[i]
: 40 - (i * 5);
frames.Add(new RadarFrame frames.Add(new RadarFrame
{ {
FrameIndex = i, FrameIndex = i,
ImagePath = framePath, ImagePath = framePath,
MinutesAgo = minutesAgo AbsoluteObservationTime = observationTime
}); });
} }
else else
+2 -1
View File
@@ -1,4 +1,5 @@
using BomLocalService.Models; using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -6,7 +7,7 @@ namespace BomLocalService.Services.Interfaces;
/// Main service interface for BOM radar screenshot operations. /// Main service interface for BOM radar screenshot operations.
/// Orchestrates cache management, browser automation, and web scraping to provide radar screenshots for Australian locations. /// Orchestrates cache management, browser automation, and web scraping to provide radar screenshots for Australian locations.
/// </summary> /// </summary>
public interface IBomRadarService public interface IBomRadarService : ISingletonService
{ {
/// <summary> /// <summary>
/// Gets cached radar data for a location. /// Gets cached radar data for a location.
+2 -1
View File
@@ -1,3 +1,4 @@
using BomLocalService.Services.Interfaces.Registration;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -6,7 +7,7 @@ namespace BomLocalService.Services.Interfaces;
/// Service interface for managing Playwright browser instances and automation. /// Service interface for managing Playwright browser instances and automation.
/// Handles browser lifecycle, context creation, and anti-detection measures for web scraping. /// Handles browser lifecycle, context creation, and anti-detection measures for web scraping.
/// </summary> /// </summary>
public interface IBrowserService : IDisposable public interface IBrowserService : ISingletonService, IDisposable
{ {
/// <summary> /// <summary>
/// Creates a new browser context with proper configuration for BOM website scraping. /// Creates a new browser context with proper configuration for BOM website scraping.
+3 -2
View File
@@ -1,4 +1,5 @@
using BomLocalService.Models; using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -6,7 +7,7 @@ namespace BomLocalService.Services.Interfaces;
/// Service interface for managing cached radar screenshot files and metadata. /// Service interface for managing cached radar screenshot files and metadata.
/// Handles file system operations for storing and retrieving cached BOM radar screenshots. /// Handles file system operations for storing and retrieving cached BOM radar screenshots.
/// </summary> /// </summary>
public interface ICacheService public interface ICacheService : ISingletonService
{ {
/// <summary> /// <summary>
/// Gets the cached screenshot folder path and associated metadata for a location and data type. /// Gets the cached screenshot folder path and associated metadata for a location and data type.
@@ -90,7 +91,7 @@ public interface ICacheService
Task SaveMetadataAsync(string cacheFolderPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default); Task SaveMetadataAsync(string cacheFolderPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default);
/// <summary> /// <summary>
/// Saves frame metadata (frame index and minutes ago) to a frames.json file in the data type subfolder. /// Saves frame metadata (frame index and observation time) to a frames.json file in the data type subfolder.
/// </summary> /// </summary>
/// <param name="cacheFolderPath">Full path to the cache folder</param> /// <param name="cacheFolderPath">Full path to the cache folder</param>
/// <param name="dataType">The type of cached data</param> /// <param name="dataType">The type of cached data</param>
+2 -1
View File
@@ -1,3 +1,4 @@
using BomLocalService.Services.Interfaces.Registration;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -6,7 +7,7 @@ namespace BomLocalService.Services.Interfaces;
/// Service interface for debug file generation during web scraping operations. /// Service interface for debug file generation during web scraping operations.
/// When enabled, saves screenshots, HTML, console logs, and network request information for troubleshooting. /// When enabled, saves screenshots, HTML, console logs, and network request information for troubleshooting.
/// </summary> /// </summary>
public interface IDebugService public interface IDebugService : ISingletonService
{ {
/// <summary> /// <summary>
/// Indicates whether debug mode is enabled. /// Indicates whether debug mode is enabled.
+2 -1
View File
@@ -1,4 +1,5 @@
using BomLocalService.Models; using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -7,7 +8,7 @@ namespace BomLocalService.Services.Interfaces;
/// Service interface for scraping radar screenshots from the BOM website. /// Service interface for scraping radar screenshots from the BOM website.
/// Orchestrates the multi-step process of navigating BOM, finding locations, and capturing radar screenshots. /// Orchestrates the multi-step process of navigating BOM, finding locations, and capturing radar screenshots.
/// </summary> /// </summary>
public interface IScrapingService public interface IScrapingService : ISingletonService
{ {
/// <summary> /// <summary>
/// Scrapes a radar screenshot from the BOM website for a given location. /// Scrapes a radar screenshot from the BOM website for a given location.
+2 -1
View File
@@ -1,4 +1,5 @@
using BomLocalService.Models; using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -6,7 +7,7 @@ namespace BomLocalService.Services.Interfaces;
/// <summary> /// <summary>
/// Service for finding page elements using configured selectors /// Service for finding page elements using configured selectors
/// </summary> /// </summary>
public interface ISelectorService public interface ISelectorService : ISingletonService
{ {
/// <summary> /// <summary>
/// Finds an element using the configured selectors, trying each in order until one is found /// Finds an element using the configured selectors, trying each in order until one is found
+2 -1
View File
@@ -1,4 +1,5 @@
using BomLocalService.Models; using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
using Microsoft.Playwright; using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces; namespace BomLocalService.Services.Interfaces;
@@ -7,7 +8,7 @@ namespace BomLocalService.Services.Interfaces;
/// Service interface for parsing time and metadata information from BOM website content. /// Service interface for parsing time and metadata information from BOM website content.
/// Extracts observation times, forecast times, weather station names, and distances from BOM weather map pages. /// Extracts observation times, forecast times, weather station names, and distances from BOM weather map pages.
/// </summary> /// </summary>
public interface ITimeParsingService public interface ITimeParsingService : ISingletonService
{ {
/// <summary> /// <summary>
/// Extracts last updated information from a BOM weather map page. /// Extracts last updated information from a BOM weather map page.
@@ -0,0 +1,30 @@
namespace BomLocalService.Services.Interfaces.Registration;
/// <summary>
/// Marker interface for services that should be registered as Singleton
/// </summary>
public interface ISingletonService
{
}
/// <summary>
/// Marker interface for services that should be registered as Scoped
/// </summary>
public interface IScopedService
{
}
/// <summary>
/// Marker interface for services that should be registered as Transient
/// </summary>
public interface ITransientService
{
}
/// <summary>
/// Marker interface for hosted services (BackgroundService implementations)
/// </summary>
public interface IHostedServiceRegistration
{
}
+4 -1
View File
@@ -1,9 +1,12 @@
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Scraping; namespace BomLocalService.Services.Scraping;
/// <summary> /// <summary>
/// Interface for a single scraping step /// Interface for a single scraping step
/// Steps are stateless (all request data is in ScrapingContext), so singleton lifetime is appropriate
/// </summary> /// </summary>
public interface IScrapingStep public interface IScrapingStep : ISingletonService
{ {
/// <summary> /// <summary>
/// Unique name of the step /// Unique name of the step
+3 -1
View File
@@ -1,9 +1,11 @@
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Scraping; namespace BomLocalService.Services.Scraping;
/// <summary> /// <summary>
/// Registry for managing scraping steps /// Registry for managing scraping steps
/// </summary> /// </summary>
public interface IScrapingStepRegistry public interface IScrapingStepRegistry : ISingletonService
{ {
void RegisterStep(IScrapingStep step); void RegisterStep(IScrapingStep step);
IScrapingStep? GetStep(string name); IScrapingStep? GetStep(string name);
+3 -1
View File
@@ -1,9 +1,11 @@
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Scraping; namespace BomLocalService.Services.Scraping;
/// <summary> /// <summary>
/// Factory for creating workflows /// Factory for creating workflows
/// </summary> /// </summary>
public interface IWorkflowFactory public interface IWorkflowFactory : ISingletonService
{ {
IWorkflow<TResponse> GetWorkflow<TResponse>(string name); IWorkflow<TResponse> GetWorkflow<TResponse>(string name);
} }
+27 -1
View File
@@ -1,3 +1,5 @@
using System.Reflection;
namespace BomLocalService.Services.Scraping; namespace BomLocalService.Services.Scraping;
public class ScrapingStepRegistry : IScrapingStepRegistry public class ScrapingStepRegistry : IScrapingStepRegistry
@@ -5,9 +7,33 @@ public class ScrapingStepRegistry : IScrapingStepRegistry
private readonly Dictionary<string, IScrapingStep> _steps = new(); private readonly Dictionary<string, IScrapingStep> _steps = new();
private readonly ILogger<ScrapingStepRegistry> _logger; private readonly ILogger<ScrapingStepRegistry> _logger;
public ScrapingStepRegistry(ILogger<ScrapingStepRegistry> logger) public ScrapingStepRegistry(ILogger<ScrapingStepRegistry> logger, IServiceProvider serviceProvider)
{ {
_logger = logger; _logger = logger;
AutoRegisterSteps(serviceProvider);
}
private void AutoRegisterSteps(IServiceProvider serviceProvider)
{
var stepTypes = typeof(IScrapingStep).Assembly.GetTypes()
.Where(t => typeof(IScrapingStep).IsAssignableFrom(t)
&& !t.IsInterface
&& !t.IsAbstract
&& !t.IsGenericType);
foreach (var stepType in stepTypes)
{
try
{
// Use DI to resolve the step (ensures proper dependency injection)
var step = (IScrapingStep)serviceProvider.GetRequiredService(stepType);
RegisterStep(step);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to auto-register step {StepType}", stepType.Name);
}
}
} }
public void RegisterStep(IScrapingStep step) public void RegisterStep(IScrapingStep step)
@@ -78,7 +78,7 @@ public class CaptureFramesStep : BaseScrapingStep
var frames = new List<RadarFrame>(); var frames = new List<RadarFrame>();
var stepForwardButton = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton); var stepForwardButton = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton);
int? previousMinutesAgo = null; DateTime? previousTimestamp = null;
for (int frameIndex = 0; frameIndex < frameCount; frameIndex++) for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
{ {
@@ -90,37 +90,48 @@ public class CaptureFramesStep : BaseScrapingStep
// This is especially important for frame 0 which was just selected in ResetToFirstFrame // This is especially important for frame 0 which was just selected in ResetToFirstFrame
await context.Page.WaitForTimeoutAsync(300); await context.Page.WaitForTimeoutAsync(300);
// Try extracting with a retry in case the label is still updating // Try extracting timestamp with a retry in case the label is still updating
var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); var frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page);
if (minutesAgo == null) if (frameTimestamp == null)
{ {
// Retry once after a short wait in case label was updating // Retry once after a short wait in case label was updating
await context.Page.WaitForTimeoutAsync(200); await context.Page.WaitForTimeoutAsync(200);
minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page);
}
if (minutesAgo == null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
{
var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
minutesAgo = defaultMinutesAgo;
Logger.LogWarning("Step {Step}: Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", Name, frameIndex, minutesAgo);
} }
if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value) // Fallback: calculate expected timestamp from observation time and frame index if we can't parse it
if (frameTimestamp == null && context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
{ {
Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", Name, frameIndex, minutesAgo); var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
await WaitForDisplayLabelToChangeAsync(context.Page, previousMinutesAgo.Value); frameTimestamp = context.LastUpdatedInfo.ObservationTime.AddMinutes(-defaultMinutesAgo);
minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); Logger.LogWarning("Step {Step}: Failed to extract timestamp from display label for frame {FrameIndex}, calculated from observation time: {Timestamp}",
if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value) Name, frameIndex, frameTimestamp);
}
if (frameIndex > 0 && previousTimestamp.HasValue && frameTimestamp == previousTimestamp.Value)
{
Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same timestamp ({Timestamp}) as previous frame. Waiting for display to update...",
Name, frameIndex, frameTimestamp);
await WaitForDisplayLabelToChangeAsync(context.Page, previousTimestamp.Value);
frameTimestamp = await ExtractTimestampFromDisplayAsync(context.Page);
if (frameTimestamp == null || frameTimestamp == previousTimestamp.Value)
{ {
if (context.FrameInfo != null && frameIndex < context.FrameInfo.Count) if (context.LastUpdatedInfo?.ObservationTime != null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
{ {
var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex]; var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
minutesAgo = defaultMinutesAgo; frameTimestamp = context.LastUpdatedInfo.ObservationTime.AddMinutes(-defaultMinutesAgo);
Logger.LogWarning("Step {Step}: Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", Name, frameIndex, minutesAgo); Logger.LogWarning("Step {Step}: Display label did not update for frame {FrameIndex}, calculated from observation time: {Timestamp}",
Name, frameIndex, frameTimestamp);
} }
} }
} }
if (frameTimestamp == null)
{
Logger.LogError("Step {Step}: Could not determine timestamp for frame {FrameIndex}, skipping", Name, frameIndex);
continue;
}
var radarFolder = FilePathHelper.GetDataTypeFolderPath(context.CacheFolderPath, CachedDataType.Radar); var radarFolder = FilePathHelper.GetDataTypeFolderPath(context.CacheFolderPath, CachedDataType.Radar);
if (!Directory.Exists(radarFolder)) if (!Directory.Exists(radarFolder))
{ {
@@ -134,13 +145,13 @@ public class CaptureFramesStep : BaseScrapingStep
{ {
FrameIndex = frameIndex, FrameIndex = frameIndex,
ImagePath = framePath, ImagePath = framePath,
MinutesAgo = minutesAgo ?? 0 AbsoluteObservationTime = frameTimestamp.Value
}); });
previousMinutesAgo = minutesAgo; previousTimestamp = frameTimestamp;
Logger.LogInformation("Step {Step}: Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)", Logger.LogInformation("Step {Step}: Frame {FrameIndex} saved: {Path} (timestamp: {Timestamp} UTC)",
Name, frameIndex, framePath, minutesAgo ?? 0); Name, frameIndex, framePath, frameTimestamp.Value);
_cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount); _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount);
@@ -150,13 +161,13 @@ public class CaptureFramesStep : BaseScrapingStep
{ {
await DismissModalOverlaysAsync(context.Page); await DismissModalOverlaysAsync(context.Page);
var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); var currentTimestamp = await ExtractTimestampFromDisplayAsync(context.Page);
await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true }); await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
if (currentMinutesAgo.HasValue) if (currentTimestamp.HasValue)
{ {
await WaitForDisplayLabelToChangeAsync(context.Page, currentMinutesAgo.Value); await WaitForDisplayLabelToChangeAsync(context.Page, currentTimestamp.Value);
} }
else else
{ {
@@ -187,7 +198,10 @@ public class CaptureFramesStep : BaseScrapingStep
} }
} }
private async Task<int?> ExtractMinutesAgoFromDisplayAsync(IPage page) /// <summary>
/// Extracts the UTC timestamp from the frame display label
/// </summary>
private async Task<DateTime?> ExtractTimestampFromDisplayAsync(IPage page)
{ {
try try
{ {
@@ -200,7 +214,7 @@ public class CaptureFramesStep : BaseScrapingStep
} }
var trimmedLabel = timeLabel.Trim(); var trimmedLabel = timeLabel.Trim();
Logger.LogInformation("Extracting minutes from display label: '{Label}'", trimmedLabel); Logger.LogInformation("Extracting timestamp from display label: '{Label}'", trimmedLabel);
// Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" // Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern); Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern);
@@ -209,19 +223,10 @@ public class CaptureFramesStep : BaseScrapingStep
{ {
var timestampStr = timestampMatch.Groups[0].Value; var timestampStr = timestampMatch.Groups[0].Value;
Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}'", timestampStr, trimmedLabel); Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}'", timestampStr, trimmedLabel);
if (TryParseTimestamp(timestampStr, out var timestamp)) if (TryParseTimestamp(timestampStr, out var frameTimestampUtc))
{ {
var minutesAgo = (int)(DateTime.UtcNow - timestamp).TotalMinutes; Logger.LogInformation("Successfully parsed frame timestamp: {Timestamp} UTC", frameTimestampUtc);
Logger.LogInformation("Parsed timestamp: {Timestamp} UTC, calculated minutes ago: {Minutes}", timestamp, minutesAgo); return frameTimestampUtc;
if (minutesAgo >= 0 && minutesAgo <= 120) // Reasonable range: 0-2 hours
{
Logger.LogInformation("Successfully calculated minutes ago from timestamp: {Minutes}", minutesAgo);
return minutesAgo;
}
else
{
Logger.LogWarning("Calculated minutes ago ({Minutes}) outside reasonable range (0-120)", minutesAgo);
}
} }
else else
{ {
@@ -237,17 +242,27 @@ public class CaptureFramesStep : BaseScrapingStep
} }
catch (Exception ex) catch (Exception ex)
{ {
Logger.LogDebug(ex, "Failed to extract minutes from display label"); Logger.LogDebug(ex, "Failed to extract timestamp from display label");
return null; return null;
} }
} }
private bool TryParseTimestamp(string timestampStr, out DateTime timestamp) private bool TryParseTimestamp(string timestampStr, out DateTime timestampUtc)
{ {
timestamp = DateTime.MinValue; timestampUtc = DateTime.MinValue;
try try
{ {
// Get configured timezone
var timezone = Configuration.GetValue<string>("Timezone");
if (string.IsNullOrEmpty(timezone))
{
Logger.LogWarning("Timezone not configured, cannot parse timestamp correctly");
return false;
}
var timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
// Try common Australian date formats // Try common Australian date formats
// Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" // Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
var formats = new[] var formats = new[]
@@ -261,57 +276,68 @@ public class CaptureFramesStep : BaseScrapingStep
}; };
var culture = new System.Globalization.CultureInfo("en-AU"); var culture = new System.Globalization.CultureInfo("en-AU");
DateTime localTime = default;
bool parsed = false;
foreach (var format in formats) foreach (var format in formats)
{ {
if (DateTime.TryParseExact(timestampStr, format, culture, if (DateTime.TryParseExact(timestampStr, format, culture,
System.Globalization.DateTimeStyles.AssumeLocal, out timestamp)) System.Globalization.DateTimeStyles.None, out localTime))
{ {
// If year is not specified, assume current year parsed = true;
if (timestamp.Year == 1) break;
{
timestamp = new DateTime(DateTime.Now.Year, timestamp.Month, timestamp.Day,
timestamp.Hour, timestamp.Minute, timestamp.Second);
}
// If the parsed time is in the future (likely same day next year), adjust
if (timestamp > DateTime.Now && timestamp < DateTime.Now.AddDays(1))
{
// Already correct
}
else if (timestamp > DateTime.Now)
{
// Likely parsed as next year, adjust to this year
timestamp = timestamp.AddYears(-1);
}
return true;
} }
} }
return false; if (!parsed)
{
return false;
}
// If year is not specified, assume current year
if (localTime.Year == 1)
{
localTime = new DateTime(DateTime.Now.Year, localTime.Month, localTime.Day,
localTime.Hour, localTime.Minute, localTime.Second);
}
// Get current time in the configured timezone to determine the date
var nowInTz = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, timeZoneInfo);
var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay);
// If the time is in the future, it must be from yesterday
if (dateTimeInTz > nowInTz)
{
dateTimeInTz = dateTimeInTz.AddDays(-1);
}
// Convert to UTC
timestampUtc = TimeZoneInfo.ConvertTimeToUtc(dateTimeInTz, timeZoneInfo);
return true;
} }
catch catch (Exception ex)
{ {
Logger.LogWarning(ex, "Failed to parse timestamp: {Timestamp}", timestampStr);
return false; return false;
} }
} }
private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000) private async Task WaitForDisplayLabelToChangeAsync(IPage page, DateTime currentTimestamp, int maxWaitMs = 5000)
{ {
try try
{ {
var startTime = DateTime.UtcNow; var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs) while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
{ {
var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); var newTimestamp = await ExtractTimestampFromDisplayAsync(page);
if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo) if (newTimestamp.HasValue && newTimestamp.Value != currentTimestamp)
{ {
return; return;
} }
await page.WaitForTimeoutAsync(200); await page.WaitForTimeoutAsync(200);
} }
Logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs); Logger.LogDebug("Display label did not change from {CurrentTimestamp} within {MaxWaitMs}ms", currentTimestamp, maxWaitMs);
} }
catch (Exception ex) catch (Exception ex)
{ {
+6 -2
View File
@@ -1,3 +1,5 @@
using BomLocalService.Services.Scraping.Workflows;
namespace BomLocalService.Services.Scraping; namespace BomLocalService.Services.Scraping;
public class WorkflowFactory : IWorkflowFactory public class WorkflowFactory : IWorkflowFactory
@@ -13,8 +15,10 @@ public class WorkflowFactory : IWorkflowFactory
{ {
return name switch return name switch
{ {
"RadarScraping" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.RadarScrapingWorkflow>(), "RadarScraping" => _serviceProvider.GetRequiredService<IRadarScrapingWorkflow>() as IWorkflow<TResponse>
"TemperatureMap" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.TemperatureMapWorkflow>(), ?? throw new InvalidOperationException($"IRadarScrapingWorkflow is not IWorkflow<{typeof(TResponse).Name}>"),
"TemperatureMap" => _serviceProvider.GetRequiredService<ITemperatureMapWorkflow>() as IWorkflow<TResponse>
?? throw new InvalidOperationException($"ITemperatureMapWorkflow is not IWorkflow<{typeof(TResponse).Name}>"),
_ => throw new ArgumentException($"Unknown workflow: {name}") _ => throw new ArgumentException($"Unknown workflow: {name}")
}; };
} }
@@ -0,0 +1,12 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Scraping.Workflows;
/// <summary>
/// Concrete interface for radar scraping workflow (hides generic IWorkflow&lt;RadarResponse&gt;)
/// </summary>
public interface IRadarScrapingWorkflow : IWorkflow<RadarResponse>, IScopedService
{
}
@@ -0,0 +1,12 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces.Registration;
namespace BomLocalService.Services.Scraping.Workflows;
/// <summary>
/// Concrete interface for temperature map workflow (hides generic IWorkflow&lt;RadarResponse&gt;)
/// </summary>
public interface ITemperatureMapWorkflow : IWorkflow<RadarResponse>, IScopedService
{
}
@@ -5,7 +5,7 @@ using BomLocalService.Utilities;
namespace BomLocalService.Services.Scraping.Workflows; namespace BomLocalService.Services.Scraping.Workflows;
public class RadarScrapingWorkflow : IWorkflow<RadarResponse> public class RadarScrapingWorkflow : IRadarScrapingWorkflow
{ {
private readonly ILogger<RadarScrapingWorkflow> _logger; private readonly ILogger<RadarScrapingWorkflow> _logger;
private readonly IScrapingStepRegistry _stepRegistry; private readonly IScrapingStepRegistry _stepRegistry;
@@ -7,7 +7,7 @@ namespace BomLocalService.Services.Scraping.Workflows;
/// <summary> /// <summary>
/// Future workflow for temperature map scraping (not yet implemented) /// Future workflow for temperature map scraping (not yet implemented)
/// </summary> /// </summary>
public class TemperatureMapWorkflow : IWorkflow<RadarResponse> public class TemperatureMapWorkflow : ITemperatureMapWorkflow
{ {
private readonly ILogger<TemperatureMapWorkflow> _logger; private readonly ILogger<TemperatureMapWorkflow> _logger;
+46 -83
View File
@@ -67,113 +67,77 @@ public class TimeParsingService : ITimeParsingService
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogWarning(ex, "Failed to extract last updated information"); _logger.LogError(ex, "Failed to extract last updated information");
return new LastUpdatedInfo throw;
{
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
} }
} }
/// <summary> /// <summary>
/// Parses last updated text to extract observation time, forecast time, weather station, and distance /// Parses last updated text to extract observation time, forecast time, weather station, and distance
/// Parses actual time strings to UTC timestamps - client calculates "minutes ago" dynamically
/// </summary> /// </summary>
public LastUpdatedInfo ParseLastUpdatedText(string text) public LastUpdatedInfo ParseLastUpdatedText(string text)
{ {
var info = new LastUpdatedInfo(); var info = new LastUpdatedInfo();
_logger.LogInformation("Parsing last updated text: {Text}", text); _logger.LogInformation("Parsing last updated text: {Text}", text);
// Parse observation time - use "minutes ago" as primary source for accuracy // Parse observation time from time string (e.g., "9:40 pm AEST")
// "Minutes ago" is reliable and doesn't require date guessing // Ignore "minutes ago" - client will calculate this dynamically from UTC timestamp
// Time string is used only as fallback if "minutes ago" is unavailable var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase); if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
{ {
// Use "minutes ago" as primary source - it's accurate and doesn't require date guessing var timeStr = observationMatch.Groups[1].Value.Trim();
info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo); var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null;
_logger.LogInformation("Calculated observation time from 'minutes ago': {MinutesAgo} minutes ago = {Time} UTC",
minutesAgo, info.ObservationTime); // Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST")
} if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2)
else
{
// Fallback to time string parsing only if "minutes ago" is not available
var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
{ {
var timeStr = observationMatch.Groups[1].Value.Trim(); timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2);
var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null; }
// Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST") if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2) {
{ info.ObservationTime = observationTime;
timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2); _logger.LogInformation("Parsed observation time: {Time} UTC (from '{TimeStr}' {TzStr})",
} observationTime, timeStr, timezoneStr ?? "default timezone");
if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
{
info.ObservationTime = observationTime;
_logger.LogInformation("Used time string fallback for observation time: {Time} UTC (from '{TimeStr}' {TzStr})",
observationTime, timeStr, timezoneStr ?? "default timezone");
}
else
{
_logger.LogWarning("Failed to parse observation time from both 'minutes ago' and time string, using current time");
info.ObservationTime = DateTime.UtcNow;
}
} }
else else
{ {
_logger.LogWarning("No observation time found in text, using current time"); _logger.LogError("Failed to parse observation time from time string '{TimeStr}' with timezone '{TzStr}'", timeStr, timezoneStr ?? "default");
info.ObservationTime = DateTime.UtcNow; throw new InvalidOperationException($"Failed to parse observation time from text: {text}");
} }
} }
// Parse forecast time - use "minutes ago" or "an hour ago" as primary source
// Handle both "X minutes ago" and "an hour ago" formats
var forecastMinutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
var forecastHourAgoMatch = Regex.Match(text, @"Forecast:\s*an\s+hour\s+ago", RegexOptions.IgnoreCase);
if (forecastMinutesAgoMatch.Success && int.TryParse(forecastMinutesAgoMatch.Groups[1].Value, out var forecastMinutesAgo))
{
// Use "minutes ago" as primary source
info.ForecastTime = DateTime.UtcNow.AddMinutes(-forecastMinutesAgo);
_logger.LogInformation("Calculated forecast time from 'minutes ago': {MinutesAgo} minutes ago = {Time} UTC",
forecastMinutesAgo, info.ForecastTime);
}
else if (forecastHourAgoMatch.Success)
{
// Handle "an hour ago" format
info.ForecastTime = DateTime.UtcNow.AddHours(-1);
_logger.LogInformation("Calculated forecast time from 'an hour ago': {Time} UTC", info.ForecastTime);
}
else else
{ {
// Fallback to time string parsing only if "minutes ago" or "an hour ago" is not available _logger.LogError("No observation time found in text: {Text}", text);
var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:an\s+hour\s+ago|\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase); throw new InvalidOperationException($"Could not find observation time in text: {text}");
if (forecastMatch.Success && forecastMatch.Groups[1].Success) }
// Parse forecast time from time string (e.g., "7:50 pm AEST")
// Ignore "minutes ago" - client will calculate this dynamically from UTC timestamp
var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:an\s+hour\s+ago|\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase);
if (forecastMatch.Success && forecastMatch.Groups[1].Success)
{
var timeStr = forecastMatch.Groups[1].Value.Trim();
var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null;
if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
{ {
var timeStr = forecastMatch.Groups[1].Value.Trim(); info.ForecastTime = forecastTime;
var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null; _logger.LogInformation("Parsed forecast time: {Time} UTC (from '{TimeStr}' {TzStr})",
forecastTime, timeStr, timezoneStr ?? "default timezone");
if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
{
info.ForecastTime = forecastTime;
_logger.LogInformation("Used time string fallback for forecast time: {Time} UTC (from '{TimeStr}' {TzStr})",
forecastTime, timeStr, timezoneStr ?? "default timezone");
}
else
{
_logger.LogWarning("Failed to parse forecast time from both 'minutes ago' and time string, using current time");
info.ForecastTime = DateTime.UtcNow;
}
} }
else else
{ {
_logger.LogWarning("No forecast time found in text, using current time"); _logger.LogError("Failed to parse forecast time from time string '{TimeStr}' with timezone '{TzStr}'", timeStr, timezoneStr ?? "default");
info.ForecastTime = DateTime.UtcNow; throw new InvalidOperationException($"Failed to parse forecast time from text: {text}");
} }
} }
else
{
_logger.LogError("No forecast time found in text: {Text}", text);
throw new InvalidOperationException($"Could not find forecast time in text: {text}");
}
// Extract weather station name // Extract weather station name
var stationMatch = Regex.Match(text, @"at\s+([^,]+)\s+weather\s+station", RegexOptions.IgnoreCase); var stationMatch = Regex.Match(text, @"at\s+([^,]+)\s+weather\s+station", RegexOptions.IgnoreCase);
@@ -256,8 +220,7 @@ public class TimeParsingService : ITimeParsingService
var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay); var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay);
// Only adjust to yesterday if the time is clearly in the future // Only adjust to yesterday if the time is clearly in the future
// Don't use the 12-hour threshold - it's too aggressive for recent observations // This ensures we get the correct date for the observation/forecast time
// This method is now only used as fallback when "minutes ago" is unavailable
if (dateTimeInTz > nowInTz) if (dateTimeInTz > nowInTz)
{ {
// Time is in the future, so it must be from yesterday // Time is in the future, so it must be from yesterday
+1 -12
View File
@@ -63,19 +63,8 @@ public static class ResponseBuilder
} }
} }
// Calculate AbsoluteObservationTime for all frames if metadata is available // AbsoluteObservationTime is already set during capture, no calculation needed
// This ensures consistency with the time series endpoint // This ensures consistency with the time series endpoint
if (metadata != null && metadata.ObservationTime > DateTime.MinValue.AddYears(1))
{
foreach (var frame in frames)
{
// Only calculate if not already set and MinutesAgo is valid
if (!frame.AbsoluteObservationTime.HasValue && frame.MinutesAgo >= 0)
{
frame.AbsoluteObservationTime = metadata.ObservationTime.AddMinutes(-frame.MinutesAgo);
}
}
}
// Calculate NextUpdateTime based on cache status: // Calculate NextUpdateTime based on cache status:
// - If cache is valid: NextUpdateTime = max(CacheExpiresAt, next background service check) // - If cache is valid: NextUpdateTime = max(CacheExpiresAt, next background service check)
+39 -25
View File
@@ -1222,32 +1222,37 @@
}); });
}); });
// Filter out frames without absoluteObservationTime (backend should have filtered these, so this catches bugs)
const invalidFrames = allFrames.filter((frame, idx) => !frame.absoluteObservationTime);
if (invalidFrames.length > 0) {
console.error('Backend returned frames missing absoluteObservationTime (should have been filtered):', {
count: invalidFrames.length,
frames: invalidFrames.map((f, idx) => ({
index: idx,
frameIndex: f.frameIndex,
cacheFolder: f.cacheFolderName,
imageUrl: f.imageUrl
}))
});
}
const validFrames = allFrames.filter(frame => frame.absoluteObservationTime);
// Validate chronological order to detect backend issues // Validate chronological order to detect backend issues
let previousTime = null; let previousTime = null;
const outOfOrderFrames = []; const outOfOrderFrames = [];
allFrames.forEach((frame, idx) => { validFrames.forEach((frame, idx) => {
if (frame.absoluteObservationTime) { const currentTime = new Date(frame.absoluteObservationTime).getTime();
const currentTime = new Date(frame.absoluteObservationTime).getTime(); if (previousTime !== null && currentTime < previousTime) {
if (previousTime !== null && currentTime < previousTime) { outOfOrderFrames.push({
outOfOrderFrames.push({ index: idx,
index: idx,
frameIndex: frame.frameIndex,
cacheFolder: frame.cacheFolderName,
absoluteTime: frame.absoluteObservationTime,
previousTime: new Date(previousTime).toISOString(),
timeDiffMinutes: (currentTime - previousTime) / 1000 / 60
});
}
previousTime = currentTime;
} else {
// Missing absoluteObservationTime is also an issue
console.warn(`Frame at index ${idx} missing absoluteObservationTime`, {
frameIndex: frame.frameIndex, frameIndex: frame.frameIndex,
cacheFolder: frame.cacheFolderName, cacheFolder: frame.cacheFolderName,
minutesAgo: frame.minutesAgo, absoluteTime: frame.absoluteObservationTime,
observationTime: frame.observationTime previousTime: new Date(previousTime).toISOString(),
timeDiffMinutes: (currentTime - previousTime) / 1000 / 60
}); });
} }
previousTime = currentTime;
}); });
if (outOfOrderFrames.length > 0) { if (outOfOrderFrames.length > 0) {
@@ -1257,6 +1262,10 @@
}); });
} }
// Use only valid frames going forward
allFrames.length = 0;
allFrames.push(...validFrames);
// Re-index frames sequentially for display // Re-index frames sequentially for display
allFrames.forEach((frame, idx) => { allFrames.forEach((frame, idx) => {
frame.sequentialIndex = idx; frame.sequentialIndex = idx;
@@ -1988,10 +1997,9 @@
// Update time display // Update time display
const timeDisplay = document.getElementById('current-frame-time'); const timeDisplay = document.getElementById('current-frame-time');
if (timeDisplay) { if (timeDisplay) {
if (isExtendedMode && frame.absoluteObservationTime) { if (frame.absoluteObservationTime) {
timeDisplay.textContent = formatDate(frame.absoluteObservationTime); const minutesAgo = Math.round((Date.now() - new Date(frame.absoluteObservationTime).getTime()) / 60000);
} else if (frame.minutesAgo !== undefined) { timeDisplay.textContent = `${minutesAgo} min ago`;
timeDisplay.textContent = `${frame.minutesAgo} min ago`;
} else { } else {
timeDisplay.textContent = '-'; timeDisplay.textContent = '-';
} }
@@ -2075,7 +2083,10 @@
: (frame.cacheTimestamp ? formatDate(frame.cacheTimestamp) : ''); : (frame.cacheTimestamp ? formatDate(frame.cacheTimestamp) : '');
altText = `Radar frame ${frameNum}${timeInfo ? ' (' + timeInfo + ')' : ''}`; altText = `Radar frame ${frameNum}${timeInfo ? ' (' + timeInfo + ')' : ''}`;
} else { } else {
altText = `Radar frame ${frame.frameIndex} (${frame.minutesAgo} minutes ago)`; const minutesAgo = frame.absoluteObservationTime
? Math.round((Date.now() - new Date(frame.absoluteObservationTime).getTime()) / 60000)
: null;
altText = `Radar frame ${frame.frameIndex}${minutesAgo !== null ? ` (${minutesAgo} minutes ago)` : ''}`;
} }
// Check if image is already loaded (from preload) // Check if image is already loaded (from preload)
@@ -2103,8 +2114,11 @@
document.getElementById('frame-info').textContent = document.getElementById('frame-info').textContent =
`Frame ${frameNum} of ${frames.length - 1}${timeInfo ? ' • ' + timeInfo : ''}`; `Frame ${frameNum} of ${frames.length - 1}${timeInfo ? ' • ' + timeInfo : ''}`;
} else { } else {
const minutesAgo = frame.absoluteObservationTime
? Math.round((Date.now() - new Date(frame.absoluteObservationTime).getTime()) / 60000)
: null;
document.getElementById('frame-info').textContent = document.getElementById('frame-info').textContent =
`Frame ${frame.frameIndex} of ${frames.length - 1} • ${frame.minutesAgo} minutes ago`; `Frame ${frame.frameIndex} of ${frames.length - 1}${minutesAgo !== null ? ` • ${minutesAgo} minutes ago` : ''}`;
} }
} }