Refactor scraping system to workflow-based architecture

Major architectural changes:
- Replace monolithic ScrapingService with workflow-based system
- Extract scraping logic into discrete, testable steps
- Add configuration-driven selectors, JavaScript templates, and text patterns
- Implement step registry and workflow factory for extensibility
- Add PageState tracking and step dependencies
- Make IWorkflow generic to support different response types

New components:
- Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture)
- BaseScrapingStep base class with common dependencies
- ScrapingContext for shared state between steps
- RadarScrapingWorkflow orchestrating step execution
- SelectorService for configurable element selection
- Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig)

Performance and monitoring:
- Add step-level timing metrics with historical tracking
- Log step durations and compare to averages
- Warn on slow steps/workflows (50%+ slower than average)
- Integrate metrics with CacheService

Configuration improvements:
- Externalize all selectors to appsettings.json
- Externalize JavaScript templates to appsettings.json
- Externalize text patterns to appsettings.json
- Support Docker configuration via mounted appsettings.json
- Remove hardcoded waits, use configurable timeouts

Bug fixes:
- Fix DebugService collection modification exception
- Fix ResetToFirstFrameStep click interception with JavaScript fallback
- Remove minutes ago fallback, use timestamp parsing only
- Update timestamp parsing for new BOM website format

Documentation:
- Update README with new architecture details
- Add configuration section with Docker guidance
- Add development guidelines for extending scraping system
- Add performance monitoring documentation

Controllers/CacheController.cs

@@ -35,14 +35,21 @@ public class CacheController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             var result = await _bomRadarService.GetCacheRangeAsync(suburb, state, cancellationToken);
             
             if (result.TotalCacheFolders == 0)
             {
-                return NotFound(new { error = "No cached data found for this location." });
+                var errorResponse = ApiErrorResponseBuilder.CacheNotFound(suburb, state);
+                errorResponse.Message = "No cached data found for this location.";
+                errorResponse.Suggestions = new Dictionary<string, object>
+                {
+                    { "action", "refresh_cache" },
+                    { "refreshEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/refresh" }
+                };
+                return NotFound(errorResponse);
             }
             
             return Ok(result);
@@ -50,7 +57,11 @@ public class CacheController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error getting cache range for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while getting cache range", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while getting cache range", 
+                ex);
+            errorResponse.Details!["location"] = new { suburb, state };
+            return StatusCode(500, errorResponse);
         }
     }
 
@@ -65,7 +76,7 @@ public class CacheController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
@@ -74,7 +85,12 @@ public class CacheController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.CacheUpdateFailed(
+                suburb, 
+                state, 
+                "Failed to trigger cache update", 
+                ex);
+            return StatusCode(500, errorResponse);
         }
     }
 
@@ -89,7 +105,7 @@ public class CacheController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
@@ -100,13 +116,22 @@ public class CacheController : ControllerBase
             }
             else
             {
-                return NotFound(new { error = $"No cached data found for {suburb}, {state}" });
+                var errorResponse = ApiErrorResponseBuilder.NotFound(
+                    "Cache", 
+                    $"{suburb}, {state}",
+                    "No cached data exists for this location.");
+                errorResponse.Details!["location"] = new { suburb, state };
+                return NotFound(errorResponse);
             }
         }
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while deleting cached location", 
+                ex);
+            errorResponse.Details!["location"] = new { suburb, state };
+            return StatusCode(500, errorResponse);
         }
     }
 }

Controllers/RadarController.cs

@@ -37,7 +37,7 @@ public class RadarController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             // Check if cache needs updating and trigger if needed (non-blocking)
@@ -70,17 +70,37 @@ public class RadarController : ControllerBase
                     cacheManagementCheckIntervalMinutes,
                     cancellationToken);
                 
-                return NotFound(new { 
-                    error = "Screenshots not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
-                    retryAfter = 30, // seconds
-                    refreshEndpoint = $"/api/cache/{suburb}/{state}/refresh",
-                    updateTriggered = cacheStatus.UpdateTriggered,
-                    cacheExists = cacheStatus.CacheExists,
-                    cacheIsValid = cacheStatus.CacheIsValid,
-                    cacheExpiresAt = cacheStatus.CacheExpiresAt,
-                    nextUpdateTime = cacheStatus.NextUpdateTime,
-                    message = cacheStatus.Message
-                });
+                // Determine if update was triggered (check if it's already in progress or was just triggered)
+                var updateTriggered = cacheStatus.UpdateTriggered || 
+                                     (cacheStatus.Message?.Contains("in progress") ?? false) ||
+                                     (!cacheStatus.CacheExists && !cacheStatus.CacheIsValid);
+                
+                var errorResponse = ApiErrorResponseBuilder.CacheNotFound(suburb, state, cacheStatus, updateTriggered);
+                
+                // Enhance message based on cache status
+                if (cacheStatus.UpdateFailed)
+                {
+                    errorResponse.Message = $"No cached data found. Previous update attempt failed: {cacheStatus.Error ?? "Unknown error"}. Retrying...";
+                    errorResponse.Details!["previousUpdateFailed"] = true;
+                    if (cacheStatus.Error != null)
+                    {
+                        errorResponse.Details!["previousError"] = cacheStatus.Error;
+                    }
+                    if (cacheStatus.ErrorCode != null)
+                    {
+                        errorResponse.Details!["previousErrorCode"] = cacheStatus.ErrorCode;
+                    }
+                }
+                else if (cacheStatus.Message?.Contains("in progress") ?? false)
+                {
+                    errorResponse.Message = "No cached data found. Cache update is currently in progress. Please retry shortly.";
+                }
+                else if (!cacheStatus.CacheExists)
+                {
+                    errorResponse.Message = "No cached data found for this location (fresh start). Cache update has been triggered in background. Please retry in a few moments.";
+                }
+                
+                return NotFound(errorResponse);
             }
             
             // Generate URLs for frames if not already set
@@ -97,7 +117,11 @@ public class RadarController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error getting cached radar screenshots for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while getting the radar screenshots", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while getting the radar screenshots", 
+                ex);
+            errorResponse.Details!["location"] = new { suburb, state };
+            return StatusCode(500, errorResponse);
         }
     }
 
@@ -112,7 +136,15 @@ public class RadarController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
+            }
+            
+            // Validate frame index
+            if (frameIndex < 0 || frameIndex > 6)
+            {
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(
+                    $"Frame index must be between 0 and 6, got {frameIndex}", 
+                    "frameIndex"));
             }
             
             RadarFrame? frame;
@@ -130,7 +162,17 @@ public class RadarController : ControllerBase
             
             if (frame == null || !System.IO.File.Exists(frame.ImagePath))
             {
-                return NotFound(new { error = $"Frame {frameIndex} not found for {suburb}, {state}" });
+                var errorResponse = ApiErrorResponseBuilder.NotFound(
+                    "Frame", 
+                    $"Frame {frameIndex} for {suburb}, {state}",
+                    "The frame may not exist yet. Try refreshing the cache or checking if cache update is in progress.");
+                errorResponse.Details!["frameIndex"] = frameIndex;
+                errorResponse.Details!["location"] = new { suburb, state };
+                if (!string.IsNullOrEmpty(cacheFolder))
+                {
+                    errorResponse.Details["cacheFolder"] = cacheFolder;
+                }
+                return NotFound(errorResponse);
             }
             
             var imageBytes = await System.IO.File.ReadAllBytesAsync(frame.ImagePath, cancellationToken);
@@ -139,7 +181,12 @@ public class RadarController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error getting frame {FrameIndex} for suburb: {Suburb}, state: {State}", frameIndex, suburb, state);
-            return StatusCode(500, new { error = "An error occurred while getting the frame", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while getting the frame", 
+                ex);
+            errorResponse.Details!["frameIndex"] = frameIndex;
+            errorResponse.Details!["location"] = new { suburb, state };
+            return StatusCode(500, errorResponse);
         }
     }
 
@@ -154,14 +201,24 @@ public class RadarController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             var result = await _bomRadarService.GetLastUpdatedInfoAsync(suburb, state, cancellationToken);
             
             if (result == null)
             {
-                return NotFound(new { error = "No cached data found. Use POST /api/cache/{suburb}/{state}/refresh to trigger cache update." });
+                var errorResponse = ApiErrorResponseBuilder.NotFound(
+                    "Metadata", 
+                    $"{suburb}, {state}",
+                    "Use POST /api/cache/{suburb}/{state}/refresh to trigger cache update.");
+                errorResponse.Details!["location"] = new { suburb, state };
+                errorResponse.Suggestions = new Dictionary<string, object>
+                {
+                    { "action", "refresh_cache" },
+                    { "refreshEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/refresh" }
+                };
+                return NotFound(errorResponse);
             }
             
             return Ok(result);
@@ -169,7 +226,11 @@ public class RadarController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error getting metadata for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while getting metadata", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while getting metadata", 
+                ex);
+            errorResponse.Details!["location"] = new { suburb, state };
+            return StatusCode(500, errorResponse);
         }
     }
 
@@ -190,7 +251,7 @@ public class RadarController : ControllerBase
             var validationError = ValidationHelper.ValidateLocation(suburb, state);
             if (validationError != null)
             {
-                return BadRequest(new { error = validationError });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(validationError));
             }
 
             DateTime? start = null;
@@ -200,7 +261,9 @@ public class RadarController : ControllerBase
             {
                 if (!DateTime.TryParse(startTime, null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsedStart))
                 {
-                    return BadRequest(new { error = "Invalid startTime format. Use ISO 8601 format (e.g., 2025-12-07T00:00:00Z)" });
+                    return BadRequest(ApiErrorResponseBuilder.ValidationError(
+                        "Invalid startTime format. Use ISO 8601 format (e.g., 2025-12-07T00:00:00Z)", 
+                        "startTime"));
                 }
                 start = parsedStart.ToUniversalTime();
             }
@@ -209,21 +272,111 @@ public class RadarController : ControllerBase
             {
                 if (!DateTime.TryParse(endTime, null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsedEnd))
                 {
-                    return BadRequest(new { error = "Invalid endTime format. Use ISO 8601 format (e.g., 2025-12-07T12:00:00Z)" });
+                    return BadRequest(ApiErrorResponseBuilder.ValidationError(
+                        "Invalid endTime format. Use ISO 8601 format (e.g., 2025-12-07T12:00:00Z)", 
+                        "endTime"));
                 }
                 end = parsedEnd.ToUniversalTime();
             }
             
             if (start.HasValue && end.HasValue && start.Value > end.Value)
             {
-                return BadRequest(new { error = "startTime must be before or equal to endTime" });
+                return BadRequest(ApiErrorResponseBuilder.ValidationError(
+                    "startTime must be before or equal to endTime",
+                    "timeRange"));
             }
 
+            // Validate time range size to prevent excessive data loading
+            // Default: base limit on cache retention, but allow override via config
+            var cacheRetentionHours = _configuration.GetValue<int>("CacheRetentionHours", 24);
+            var configuredMaxHours = _configuration.GetValue<int?>("TimeSeries:MaxTimeRangeHours");
+            
+            // Use configured value if set, otherwise use cache retention (with minimum of 24 hours)
+            var maxTimeRangeHours = configuredMaxHours ?? Math.Max(cacheRetentionHours, 24);
+            
+            if (start.HasValue && end.HasValue)
+            {
+                var timeRange = end.Value - start.Value;
+                if (timeRange.TotalHours > maxTimeRangeHours)
+                {
+                    var reason = configuredMaxHours.HasValue 
+                        ? $"configured limit: {configuredMaxHours} hours"
+                        : $"cache retention: {cacheRetentionHours} hours";
+                    
+                    return BadRequest(ApiErrorResponseBuilder.TimeRangeError(
+                        $"Time range exceeds maximum allowed duration of {maxTimeRangeHours} hours (based on {reason}). Please specify a smaller range.",
+                        null,
+                        new { start = start.Value, end = end.Value, requestedHours = timeRange.TotalHours }));
+                }
+            }
+
+            // Check if location has any cache at all (to distinguish from "no data in range")
+            var cacheRange = await _bomRadarService.GetCacheRangeAsync(suburb, state, cancellationToken);
+            var locationHasCache = cacheRange.TotalCacheFolders > 0;
+
+            // If no cache exists for this location, trigger background update and return detailed 404
+            if (!locationHasCache)
+            {
+                // Trigger background cache update (non-blocking)
+                _ = Task.Run(async () =>
+                {
+                    try
+                    {
+                        await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, CancellationToken.None);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogWarning(ex, "Background cache update failed for {Suburb}, {State}", suburb, state);
+                    }
+                });
+
+                // Get cache status to include in 404 response
+                var cacheExpirationMinutes = (int)_configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
+                var cacheManagementCheckIntervalMinutes = _configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
+                var cacheStatus = await _cacheService.GetCacheStatusAsync(
+                    suburb, 
+                    state, 
+                    CachedDataType.Radar,
+                    cacheExpirationMinutes,
+                    cacheManagementCheckIntervalMinutes,
+                    cancellationToken);
+                
+                var updateTriggered = cacheStatus.UpdateTriggered || 
+                                     (cacheStatus.Message?.Contains("in progress") ?? false) ||
+                                     (!cacheStatus.CacheExists && !cacheStatus.CacheIsValid);
+                
+                var errorResponse = ApiErrorResponseBuilder.CacheNotFound(suburb, state, cacheStatus, updateTriggered);
+                errorResponse.Message = "No cached data found for this location. Cache update has been triggered in background. Please retry in a few moments.";
+                errorResponse.Details!["requestedTimeRange"] = new { start, end };
+                
+                return NotFound(errorResponse);
+            }
+
+            // Location has cache, check if there's data in the requested time range
             var result = await _bomRadarService.GetRadarTimeSeriesAsync(suburb, state, start, end, cancellationToken);
             
             if (result.CacheFolders.Count == 0)
             {
-                return NotFound(new { error = "No historical data found for the specified time range." });
+                // Location has cache, but no data in the requested time range
+                var oldestCache = cacheRange.OldestCache?.CacheTimestamp;
+                var newestCache = cacheRange.NewestCache?.CacheTimestamp;
+                
+                var availableRange = new {
+                    oldest = oldestCache,
+                    newest = newestCache,
+                    totalCacheFolders = cacheRange.TotalCacheFolders,
+                    timeSpanMinutes = cacheRange.TimeSpanMinutes
+                };
+                
+                var requestedRange = new {
+                    start = start,
+                    end = end
+                };
+                
+                return NotFound(ApiErrorResponseBuilder.TimeRangeError(
+                    "No historical data found for the specified time range.",
+                    availableRange,
+                    requestedRange));
             }
             
             return Ok(result);
@@ -231,7 +384,12 @@ public class RadarController : ControllerBase
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error getting radar time series for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while getting radar time series", message = ex.Message });
+            var errorResponse = ApiErrorResponseBuilder.InternalError(
+                "An error occurred while getting radar time series", 
+                ex);
+            errorResponse.Details!["location"] = new { suburb, state };
+            errorResponse.Details!["requestedTimeRange"] = new { startTime, endTime };
+            return StatusCode(500, errorResponse);
         }
     }
 }

Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
     chmod +x dotnet-install.sh && \
     ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \
     rm dotnet-install.sh && \
-    apt-get install -y xvfb x11vnc fluxbox && \
+    apt-get install -y xvfb x11vnc fluxbox curl && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -34,12 +34,9 @@ COPY --from=publish /app/publish .
 # Create cache directory for screenshots
 RUN mkdir -p /app/cache && chmod 777 /app/cache
 
-# Create startup script to run Xvfb and then the app
-RUN echo '#!/bin/bash\n\
-Xvfb :99 -screen 0 1920x1080x24 > /dev/null 2>&1 &\n\
-export DISPLAY=:99\n\
-exec dotnet BomLocalService.dll' > /app/start.sh && \
-    chmod +x /app/start.sh
+# Copy startup script
+COPY scripts/docker-start.sh /app/start.sh
+RUN chmod +x /app/start.sh
 
 # OCI labels for GitHub Container Registry metadata
 LABEL org.opencontainers.image.source="https://github.com/alexhopeoconnor/bom-local-service"
@@ -48,4 +45,7 @@ LABEL org.opencontainers.image.licenses=MIT
 
 EXPOSE 8080
 
+HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
+  CMD curl --fail http://localhost:8080/api/health || exit 1
+
 ENTRYPOINT ["/app/start.sh"]

Models/ApiErrorResponse.cs

@@ -0,0 +1,340 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Standardized error response model for API endpoints.
+/// Provides consistent error structure with error codes, types, and detailed information.
+/// </summary>
+public class ApiErrorResponse
+{
+    /// <summary>
+    /// Machine-readable error code for programmatic handling.
+    /// Examples: "CACHE_NOT_FOUND", "VALIDATION_ERROR", "INTERNAL_ERROR", "CACHE_UPDATE_FAILED"
+    /// </summary>
+    public string ErrorCode { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Human-readable error message describing what went wrong.
+    /// </summary>
+    public string Message { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Error type/category for grouping similar errors.
+    /// Examples: "CacheError", "ValidationError", "ServiceError", "NotFoundError"
+    /// </summary>
+    public string ErrorType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Additional context about the error (e.g., field name for validation errors).
+    /// </summary>
+    public Dictionary<string, object>? Details { get; set; }
+
+    /// <summary>
+    /// Suggested action for the client (e.g., "retry_after_seconds", "refresh_cache").
+    /// </summary>
+    public Dictionary<string, object>? Suggestions { get; set; }
+
+    /// <summary>
+    /// Timestamp when the error occurred (UTC).
+    /// </summary>
+    public DateTime Timestamp { get; set; } = DateTime.UtcNow;
+}
+
+/// <summary>
+/// Helper class for creating standardized error responses.
+/// </summary>
+public static class ApiErrorResponseBuilder
+{
+    /// <summary>
+    /// Creates an error response for when cache is not found (fresh start scenario).
+    /// </summary>
+    public static ApiErrorResponse CacheNotFound(
+        string suburb, 
+        string state, 
+        CacheUpdateStatus? cacheStatus = null,
+        bool updateTriggered = false)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "CACHE_NOT_FOUND",
+            ErrorType = "CacheError",
+            Message = "No cached data found for this location. Cache update has been triggered in background.",
+            Details = new Dictionary<string, object>
+            {
+                { "location", new { suburb, state } },
+                { "cacheExists", cacheStatus?.CacheExists ?? false },
+                { "cacheIsValid", cacheStatus?.CacheIsValid ?? false },
+                { "updateTriggered", updateTriggered || (cacheStatus?.UpdateTriggered ?? false) }
+            },
+            Suggestions = new Dictionary<string, object>
+            {
+                { "action", "retry_after_seconds" },
+                { "refreshEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/refresh" },
+                { "statusEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/range" }
+            }
+        };
+
+        // Determine meaningful retryAfter based on cache status
+        int retryAfter;
+        bool isUpdateInProgress = cacheStatus?.Message?.Contains("in progress") ?? false;
+        
+        if (cacheStatus != null)
+        {
+            if (cacheStatus.CacheExpiresAt.HasValue)
+            {
+                response.Details!["cacheExpiresAt"] = cacheStatus.CacheExpiresAt.Value;
+            }
+            if (cacheStatus.NextUpdateTime.HasValue)
+            {
+                response.Details!["nextUpdateTime"] = cacheStatus.NextUpdateTime.Value;
+                
+                // Calculate retryAfter from NextUpdateTime if available
+                var secondsUntilUpdate = (int)(cacheStatus.NextUpdateTime.Value - DateTime.UtcNow).TotalSeconds;
+                
+                if (isUpdateInProgress)
+                {
+                    // Update is in progress - retry when it completes (typically ~2 minutes)
+                    // Use NextUpdateTime if it's reasonable (1-3 minutes), otherwise default to 2 minutes
+                    retryAfter = secondsUntilUpdate > 0 && secondsUntilUpdate <= 180 
+                        ? secondsUntilUpdate 
+                        : 120; // Default to 2 minutes for in-progress updates
+                }
+                else if (updateTriggered || cacheStatus.UpdateTriggered)
+                {
+                    // Update was just triggered - allow time for it to complete (60-90 seconds)
+                    // Use NextUpdateTime if reasonable, otherwise default to 90 seconds
+                    retryAfter = secondsUntilUpdate > 0 && secondsUntilUpdate <= 120 
+                        ? secondsUntilUpdate 
+                        : 90; // Default to 90 seconds for newly triggered updates
+                }
+                else
+                {
+                    // No update triggered yet - suggest waiting for background service check
+                    // Cap at reasonable maximum (5 minutes)
+                    retryAfter = secondsUntilUpdate > 0 && secondsUntilUpdate <= 300 
+                        ? secondsUntilUpdate 
+                        : 60; // Default to 60 seconds if NextUpdateTime is too far or in past
+                }
+            }
+            else
+            {
+                // No NextUpdateTime available - use defaults based on scenario
+                if (isUpdateInProgress)
+                {
+                    retryAfter = 120; // 2 minutes for in-progress updates
+                }
+                else if (updateTriggered || cacheStatus.UpdateTriggered)
+                {
+                    retryAfter = 90; // 90 seconds for newly triggered updates
+                }
+                else
+                {
+                    retryAfter = 60; // 60 seconds default
+                }
+            }
+            
+            response.Details!["statusMessage"] = cacheStatus.Message ?? "Unknown status";
+            
+            // Add update failure information if available
+            if (cacheStatus.UpdateFailed)
+            {
+                response.Details!["previousUpdateFailed"] = true;
+                if (cacheStatus.Error != null)
+                {
+                    response.Details!["previousError"] = cacheStatus.Error;
+                }
+                if (cacheStatus.ErrorCode != null)
+                {
+                    response.Details!["previousErrorCode"] = cacheStatus.ErrorCode;
+                }
+                // Suggest manual refresh if previous update failed
+                response.Suggestions!["action"] = "manual_refresh_recommended";
+            }
+        }
+        else
+        {
+            // No cache status - default to 90 seconds for fresh start
+            retryAfter = 90;
+        }
+        
+        // Ensure retryAfter is within reasonable bounds (30 seconds to 5 minutes)
+        retryAfter = Math.Max(30, Math.Min(retryAfter, 300));
+        response.Suggestions!["retryAfter"] = retryAfter;
+
+        return response;
+    }
+
+    /// <summary>
+    /// Creates an error response for validation errors.
+    /// </summary>
+    public static ApiErrorResponse ValidationError(string message, string? field = null)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "VALIDATION_ERROR",
+            ErrorType = "ValidationError",
+            Message = message
+        };
+
+        if (!string.IsNullOrEmpty(field))
+        {
+            response.Details = new Dictionary<string, object> { { "field", field } };
+        }
+
+        return response;
+    }
+
+    /// <summary>
+    /// Creates an error response for when cache update fails.
+    /// </summary>
+    public static ApiErrorResponse CacheUpdateFailed(string suburb, string state, string reason, Exception? exception = null)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "CACHE_UPDATE_FAILED",
+            ErrorType = "ServiceError",
+            Message = $"Failed to update cache for {suburb}, {state}: {reason}",
+            Details = new Dictionary<string, object>
+            {
+                { "location", new { suburb, state } },
+                { "reason", reason }
+            },
+            Suggestions = new Dictionary<string, object>
+            {
+                { "action", "retry_after_seconds" },
+                { "retryAfter", 60 }, // Wait 1 minute before retrying failed update
+                { "refreshEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/refresh" },
+                { "statusEndpoint", $"/api/cache/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/range" }
+            }
+        };
+
+        if (exception != null)
+        {
+            response.Details["exceptionType"] = exception.GetType().Name;
+            response.Details["exceptionMessage"] = exception.Message;
+            
+            // Adjust retry suggestion based on exception type
+            if (exception is TimeoutException || exception.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase))
+            {
+                response.Suggestions!["retryAfter"] = 30; // Shorter wait for timeouts
+                response.Suggestions!["action"] = "retry_soon";
+            }
+            else if (exception.Message.Contains("network", StringComparison.OrdinalIgnoreCase) || 
+                     exception.Message.Contains("connection", StringComparison.OrdinalIgnoreCase))
+            {
+                response.Suggestions!["retryAfter"] = 120; // Longer wait for network issues
+                response.Suggestions!["action"] = "check_network_and_retry";
+            }
+        }
+
+        return response;
+    }
+
+    /// <summary>
+    /// Creates an error response for internal server errors.
+    /// </summary>
+    public static ApiErrorResponse InternalError(string message, Exception? exception = null)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "INTERNAL_ERROR",
+            ErrorType = "ServiceError",
+            Message = message
+        };
+
+        if (exception != null)
+        {
+            response.Details = new Dictionary<string, object>
+            {
+                { "exceptionType", exception.GetType().Name },
+                { "exceptionMessage", exception.Message }
+            };
+        }
+
+        return response;
+    }
+
+    /// <summary>
+    /// Creates an error response for when a specific resource is not found.
+    /// </summary>
+    public static ApiErrorResponse NotFound(string resourceType, string identifier, string? suggestion = null)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "NOT_FOUND",
+            ErrorType = "NotFoundError",
+            Message = $"{resourceType} not found: {identifier}",
+            Details = new Dictionary<string, object>
+            {
+                { "resourceType", resourceType },
+                { "identifier", identifier }
+            }
+        };
+
+        if (!string.IsNullOrEmpty(suggestion))
+        {
+            response.Suggestions = new Dictionary<string, object> { { "suggestion", suggestion } };
+        }
+
+        return response;
+    }
+
+    /// <summary>
+    /// Creates an error response for time range validation errors.
+    /// </summary>
+    public static ApiErrorResponse TimeRangeError(string message, object? availableRange = null, object? requestedRange = null)
+    {
+        var response = new ApiErrorResponse
+        {
+            ErrorCode = "TIME_RANGE_ERROR",
+            ErrorType = "ValidationError",
+            Message = message,
+            Details = new Dictionary<string, object>(),
+            Suggestions = new Dictionary<string, object>
+            {
+                { "action", "adjust_time_range" }
+            }
+        };
+
+        if (availableRange != null)
+        {
+            response.Details["availableRange"] = availableRange;
+            
+            // Add suggestion to use available range if provided
+            // Try to extract oldest/newest from the availableRange object using reflection
+            try
+            {
+                var rangeType = availableRange.GetType();
+                var oldestProp = rangeType.GetProperty("oldest");
+                var newestProp = rangeType.GetProperty("newest");
+                
+                if (oldestProp != null && newestProp != null)
+                {
+                    var oldest = oldestProp.GetValue(availableRange);
+                    var newest = newestProp.GetValue(availableRange);
+                    
+                    if (oldest != null && newest != null)
+                    {
+                        response.Suggestions!["suggestedRange"] = new
+                        {
+                            start = oldest,
+                            end = newest
+                        };
+                        response.Suggestions!["suggestion"] = $"Try querying data between {oldest} and {newest}";
+                    }
+                }
+            }
+            catch
+            {
+                // Ignore if we can't extract range info
+            }
+        }
+
+        if (requestedRange != null)
+        {
+            response.Details["requestedRange"] = requestedRange;
+        }
+
+        return response;
+    }
+}
+

Models/CacheUpdateStatus.cs

@@ -1,5 +1,15 @@
 namespace BomLocalService.Models;
 
+/// <summary>
+/// Represents the current phase of a cache update operation.
+/// </summary>
+public enum CacheUpdatePhase
+{
+    Initializing,      // Browser setup, navigation (0-20% of time)
+    CapturingFrames,   // Frame capture loop (20-95% of time) 
+    Saving            // Metadata, cleanup (95-100% of time)
+}
+
 /// <summary>
 /// Status information about a cache update operation for a location.
 /// Returned when manually triggering a cache refresh via the refresh endpoint.
@@ -51,7 +61,34 @@ public class CacheUpdateStatus
     /// - "Cache is stale, update triggered" - Cache exists but expired, update initiated
     /// - "No cache exists, update triggered" - No cache found, update initiated
     /// - "Cache update already in progress" - An update is currently running
+    /// - "Cache update failed" - An update was attempted but failed (check Error property)
     /// </summary>
     public string? Message { get; set; }
+
+    /// <summary>
+    /// Indicates whether the last cache update attempt failed.
+    /// True if an update was triggered but encountered an error.
+    /// </summary>
+    public bool UpdateFailed { get; set; }
+
+    /// <summary>
+    /// Error information if the cache update failed.
+    /// Contains error code, message, and details about what went wrong.
+    /// Null if no error occurred or if update hasn't been attempted yet.
+    /// </summary>
+    public string? Error { get; set; }
+
+    /// <summary>
+    /// Error code for programmatic handling of update failures.
+    /// Examples: "SCRAPING_ERROR", "BROWSER_ERROR", "STORAGE_ERROR", "TIMEOUT_ERROR"
+    /// Null if no error occurred.
+    /// </summary>
+    public string? ErrorCode { get; set; }
+
+    /// <summary>
+    /// Timestamp when the last update attempt was made (UTC).
+    /// Null if no update has been attempted yet.
+    /// </summary>
+    public DateTime? LastUpdateAttempt { get; set; }
 }
 

Models/JavaScriptTemplatesConfig.cs

@@ -0,0 +1,186 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// JavaScript code templates used for page evaluation
+/// </summary>
+public class JavaScriptTemplatesConfig
+{
+    public string WaitForSearchResults { get; set; } = @"() => {
+        const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
+        return results.length > 0 && results.some(r => r.offsetParent !== null);
+    }";
+
+    public string ExtractSearchResults { get; set; } = @"() => {
+        const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
+        if (!resultsList) {
+            console.log('Location results list not found');
+            return [];
+        }
+        const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
+        console.log('Found', results.length, 'location results');
+        return results.map((r) => {
+            const nameEl = r.querySelector('[data-testid=""location-name""]');
+            const descEl = r.querySelector('.bom-linklist-item__desc');
+            const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
+            const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
+            const fullText = (r.textContent || r.innerText || '').trim();
+            console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
+            return [name, desc, fullText];
+        });
+    }";
+
+    public string ExtractSearchResultsFallback { get; set; } = @"() => { 
+        const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); 
+        return results.map(r => r.textContent || ''); 
+    }";
+
+    public string WaitForMapCanvas { get; set; } = @"() => {
+        const canvas = document.querySelector('.esri-view-surface canvas');
+        return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
+    }";
+
+    public string WaitForEsriView { get; set; } = @"() => {
+        try {
+            const elements = document.querySelectorAll('.esri-view');
+            for (let el of elements) {
+                if (el.__view && el.__view.ready) {
+                    return true;
+                }
+            }
+        } catch(e) {}
+        return false;
+    }";
+
+    public string CheckActiveFrameSegment { get; set; } = @"() => {
+        const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
+        const activeSegment = segments.find(s => {
+            const style = window.getComputedStyle(s);
+            return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
+        });
+        return activeSegment && activeSegment.getAttribute('data-id') === '0';
+    }";
+
+    public string WaitForMapContainer { get; set; } = @"() => {
+        const container = document.querySelector('.esri-view-surface');
+        return container && container.offsetWidth > 0 && container.offsetHeight > 0;
+    }";
+
+    public string ExtractFrameInfo { get; set; } = @"() => {
+        const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
+        const now = new Date();
+        return segments.map((seg, index) => {
+            const ariaLabel = seg.getAttribute('aria-label') || '';
+            let minutes = null;
+            
+            // Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm""
+            const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i);
+            if (timestampMatch) {
+                try {
+                    const day = parseInt(timestampMatch[1]);
+                    const monthStr = timestampMatch[2];
+                    const hour12 = parseInt(timestampMatch[3]);
+                    const minute = parseInt(timestampMatch[4]);
+                    const ampm = timestampMatch[5].toLowerCase();
+                    
+                    const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+                    const month = monthNames.indexOf(monthStr.toLowerCase());
+                    
+                    if (month >= 0) {
+                        let hour24 = hour12;
+                        if (ampm === 'pm' && hour12 !== 12) hour24 += 12;
+                        if (ampm === 'am' && hour12 === 12) hour24 = 0;
+                        
+                        const year = now.getFullYear();
+                        const frameTime = new Date(year, month, day, hour24, minute);
+                        
+                        // If parsed time is in future, assume it's from last year
+                        if (frameTime > now) {
+                            frameTime.setFullYear(year - 1);
+                        }
+                        
+                        const diffMs = now - frameTime;
+                        minutes = Math.round(diffMs / (1000 * 60));
+                        
+                        // Validate reasonable range (0-2 hours)
+                        if (minutes < 0 || minutes > 120) {
+                            minutes = null;
+                        }
+                    }
+                } catch(e) {
+                    // Parse error, leave minutes as null
+                }
+            }
+            
+            return { index: index, minutesAgo: minutes };
+        });
+    }";
+
+    public string CheckModalOverlay { get; set; } = @"() => {
+        const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
+        if (bomOverlay && bomOverlay.style.display !== 'none') {
+            return true;
+        }
+        
+        const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container'];
+        for (const selector of recaptchaSelectors) {
+            const el = document.querySelector(selector);
+            if (el) {
+                const style = window.getComputedStyle(el);
+                const rect = el.getBoundingClientRect();
+                if (style.display !== 'none' && style.visibility !== 'hidden' && 
+                    rect.width > 200 && rect.height > 200) {
+                    return true;
+                }
+            }
+        }
+        
+        const allForms = document.querySelectorAll('form');
+        for (const form of allForms) {
+            const action = form.getAttribute('action') || '';
+            const id = form.getAttribute('id') || '';
+            if (action.includes('feedback') || id.includes('feedback')) {
+                const style = window.getComputedStyle(form);
+                const rect = form.getBoundingClientRect();
+                if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
+                    const text = form.textContent || '';
+                    if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) {
+                        return true;
+                    }
+                }
+            }
+        }
+        
+        return false;
+    }";
+
+    public string CheckModalStillVisible { get; set; } = @"() => {
+        const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
+        if (bomOverlay && bomOverlay.style.display !== 'none') return true;
+        
+        const allForms = document.querySelectorAll('form');
+        for (const form of allForms) {
+            const action = form.getAttribute('action') || '';
+            const id = form.getAttribute('id') || '';
+            if (action.includes('feedback') || id.includes('feedback')) {
+                const style = window.getComputedStyle(form);
+                const rect = form.getBoundingClientRect();
+                if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }";
+
+    public string GetViewportSize { get; set; } = @"() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })";
+
+    public string ExtractWeatherMetadata { get; set; } = @"() => {
+        const section = document.querySelector('section[data-testid=""weatherMetadata""]') || 
+                       document.querySelector('section[aria-label=""Last updated""]');
+        if (!section) return null;
+        
+        const divs = section.querySelectorAll('div');
+        return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
+    }";
+}
+

Models/ScrapingSelectorsConfig.cs

@@ -0,0 +1,26 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Configuration for all selectors used in scraping
+/// </summary>
+public class ScrapingSelectorsConfig
+{
+    public SelectorConfig SearchButton { get; set; } = new();
+    public SelectorConfig SearchInput { get; set; } = new();
+    public SelectorConfig SearchResultsList { get; set; } = new();
+    public SelectorConfig SearchResultItem { get; set; } = new();
+    public SelectorConfig LocationName { get; set; } = new();
+    public SelectorConfig LocationDescription { get; set; } = new();
+    public SelectorConfig ResultsTitle { get; set; } = new();
+    public SelectorConfig RadarLink { get; set; } = new();
+    public SelectorConfig MapCanvas { get; set; } = new();
+    public SelectorConfig MapContainer { get; set; } = new();
+    public SelectorConfig PlayPauseButton { get; set; } = new();
+    public SelectorConfig PlayPauseLabel { get; set; } = new();
+    public SelectorConfig FrameSegment { get; set; } = new();
+    public SelectorConfig StepForwardButton { get; set; } = new();
+    public SelectorConfig TimeDisplayLabel { get; set; } = new();
+    public SelectorConfig ModalOverlay { get; set; } = new();
+    public SelectorConfig WeatherMetadata { get; set; } = new();
+}
+

Models/ScrapingStepConfig.cs

@@ -0,0 +1,18 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Configuration for a single scraping step
+/// </summary>
+public class ScrapingStepConfig
+{
+    /// <summary>
+    /// Whether this step is enabled (can be disabled without removing from workflow)
+    /// </summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    /// Optional parameters specific to this step
+    /// </summary>
+    public Dictionary<string, object>? Parameters { get; set; }
+}
+

Models/ScrapingWorkflowConfig.cs

@@ -0,0 +1,11 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Configuration for a scraping workflow
+/// </summary>
+public class ScrapingWorkflowConfig
+{
+    public string Description { get; set; } = string.Empty;
+    public Dictionary<string, ScrapingStepConfig> Steps { get; set; } = new();
+}
+

Models/SelectorConfig.cs

@@ -0,0 +1,33 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Configuration for a single selector with multiple fallback options
+/// </summary>
+public class SelectorConfig
+{
+    /// <summary>
+    /// Human-readable name for this selector (for logging/debugging)
+    /// </summary>
+    public string Name { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Array of CSS selectors to try in order (first match wins)
+    /// </summary>
+    public string[] Selectors { get; set; } = Array.Empty<string>();
+
+    /// <summary>
+    /// Timeout in milliseconds when waiting for this element
+    /// </summary>
+    public int TimeoutMs { get; set; } = 5000;
+
+    /// <summary>
+    /// Whether this element is required (throws if not found)
+    /// </summary>
+    public bool Required { get; set; } = true;
+
+    /// <summary>
+    /// Custom error message if element is not found (when Required=true)
+    /// </summary>
+    public string? ErrorMessage { get; set; }
+}
+

Models/TextPatternsConfig.cs

@@ -0,0 +1,23 @@
+namespace BomLocalService.Models;
+
+/// <summary>
+/// Text patterns and regex configurations for parsing page content
+/// </summary>
+public class TextPatternsConfig
+{
+    public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)";
+    public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)";
+    public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago";
+    public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago";
+    public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago";
+    public string WeatherStationPattern { get; set; } = @"at\s+([^,]+)\s+weather\s+station";
+    public string DistancePattern { get; set; } = @"(\d+)\s*km\s+from";
+    
+    public Dictionary<string, string> ExpectedTexts { get; set; } = new()
+    {
+        ["PlayButtonLabel"] = "Play",
+        ["PauseButtonLabel"] = "Pause",
+        ["RadarLinkText"] = "Rain radar and weather map"
+    };
+}
+

Program.cs

@@ -1,5 +1,12 @@
 using BomLocalService.Services;
 using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using BomLocalService.Services.Scraping.Steps.Navigation;
+using BomLocalService.Services.Scraping.Steps.Search;
+using BomLocalService.Services.Scraping.Steps.Map;
+using BomLocalService.Services.Scraping.Steps.Metadata;
+using BomLocalService.Services.Scraping.Steps.Capture;
+using BomLocalService.Services.Scraping.Workflows;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -8,12 +15,108 @@ var builder = WebApplication.CreateBuilder(args);
 builder.Services.AddControllersWithViews();
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddOpenApi();
+builder.Services.AddHealthChecks();
+
+// Configure CORS - MUST be added before other services
+var corsOrigins = builder.Configuration.GetValue<string>("Cors:AllowedOrigins", "*");
+var corsMethods = builder.Configuration.GetValue<string>("Cors:AllowedMethods", "GET,POST,OPTIONS");
+var corsHeaders = builder.Configuration.GetValue<string>("Cors:AllowedHeaders", "*");
+var corsAllowCredentials = builder.Configuration.GetValue<bool>("Cors:AllowCredentials", false);
+
+// Support environment variable override (comma-separated for multiple origins)
+var corsOriginsEnv = Environment.GetEnvironmentVariable("CORS__ALLOWEDORIGINS");
+if (!string.IsNullOrEmpty(corsOriginsEnv))
+{
+    corsOrigins = corsOriginsEnv;
+}
+
+var corsMethodsEnv = Environment.GetEnvironmentVariable("CORS__ALLOWEDMETHODS");
+if (!string.IsNullOrEmpty(corsMethodsEnv))
+{
+    corsMethods = corsMethodsEnv;
+}
+
+var corsHeadersEnv = Environment.GetEnvironmentVariable("CORS__ALLOWEDHEADERS");
+if (!string.IsNullOrEmpty(corsHeadersEnv))
+{
+    corsHeaders = corsHeadersEnv;
+}
+
+var corsAllowCredentialsEnv = Environment.GetEnvironmentVariable("CORS__ALLOWCREDENTIALS");
+if (!string.IsNullOrEmpty(corsAllowCredentialsEnv) && bool.TryParse(corsAllowCredentialsEnv, out var parsedCredentials))
+{
+    corsAllowCredentials = parsedCredentials;
+}
+
+builder.Services.AddCors(options =>
+{
+    options.AddDefaultPolicy(policy =>
+    {
+        if (corsOrigins == "*")
+        {
+            policy.AllowAnyOrigin();
+        }
+        else
+        {
+            // Split comma-separated origins
+            var origins = corsOrigins.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+            policy.WithOrigins(origins);
+        }
+
+        // Split comma-separated methods
+        var methods = corsMethods.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        policy.WithMethods(methods);
+
+        // Split comma-separated headers or allow all
+        if (corsHeaders == "*")
+        {
+            policy.AllowAnyHeader();
+        }
+        else
+        {
+            var headers = corsHeaders.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+            policy.WithHeaders(headers);
+        }
+
+        if (corsAllowCredentials)
+        {
+            policy.AllowCredentials();
+        }
+    });
+});
 
 // Register core services via interfaces (order matters - dependencies must be registered first)
 builder.Services.AddSingleton<IDebugService, DebugService>();
 builder.Services.AddSingleton<ICacheService, CacheService>();
 builder.Services.AddSingleton<ITimeParsingService, TimeParsingService>();
 builder.Services.AddSingleton<IBrowserService, BrowserService>();
+builder.Services.AddSingleton<ISelectorService, SelectorService>();
+
+// Register scraping step registry
+builder.Services.AddSingleton<IScrapingStepRegistry, ScrapingStepRegistry>();
+
+// Register all scraping steps
+builder.Services.AddScoped<NavigateHomepageStep>();
+builder.Services.AddScoped<ClickSearchButtonStep>();
+builder.Services.AddScoped<FillSearchInputStep>();
+builder.Services.AddScoped<WaitForSearchResultsStep>();
+builder.Services.AddScoped<SelectSearchResultStep>();
+builder.Services.AddScoped<ClickRadarLinkStep>();
+builder.Services.AddScoped<WaitForMapReadyStep>();
+builder.Services.AddScoped<PauseRadarStep>();
+builder.Services.AddScoped<ResetToFirstFrameStep>();
+builder.Services.AddScoped<ExtractMetadataStep>();
+builder.Services.AddScoped<CalculateMapBoundsStep>();
+builder.Services.AddScoped<CaptureFramesStep>();
+
+// Register workflows
+builder.Services.AddScoped<RadarScrapingWorkflow>();
+builder.Services.AddScoped<TemperatureMapWorkflow>();
+
+// Register workflow factory
+builder.Services.AddSingleton<IWorkflowFactory, WorkflowFactory>();
+
+// Register scraping service (depends on workflow factory)
 builder.Services.AddSingleton<IScrapingService, ScrapingService>();
 
 // Register BOM Radar Service as singleton (orchestrator, depends on all above services)
@@ -45,6 +148,12 @@ if (enableHttpsRedirection)
     app.UseHttpsRedirection();
 }
 
+// Enable routing (required for CORS to work with controllers)
+app.UseRouting();
+
+// CORS middleware - MUST be after UseRouting but before MapControllers
+app.UseCors();
+
 // No authorization required - service is designed to run behind a reverse proxy if auth is needed
 
 // Map MVC routes first (before API routes to avoid conflicts)
@@ -55,6 +164,27 @@ app.MapControllerRoute(
 // Map API controllers (with /api prefix)
 app.MapControllers();
 
+// Map health check endpoint for Docker health monitoring
+app.MapHealthChecks("/api/health");
+
+// Auto-register all scraping steps in the registry
+var stepRegistry = app.Services.GetRequiredService<IScrapingStepRegistry>();
+var stepTypes = typeof(IScrapingStep).Assembly.GetTypes()
+    .Where(t => typeof(IScrapingStep).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract && !t.IsGenericType);
+foreach (var stepType in stepTypes)
+{
+    try
+    {
+        var step = (IScrapingStep)ActivatorUtilities.CreateInstance(app.Services, stepType);
+        stepRegistry.RegisterStep(step);
+    }
+    catch (Exception ex)
+    {
+        var logger = app.Services.GetRequiredService<ILogger<Program>>();
+        logger.LogWarning(ex, "Failed to register step {StepType}", stepType.Name);
+    }
+}
+
 // Cleanup incomplete cache folders from previous crashes/restarts before starting services
 var cacheService = app.Services.GetRequiredService<ICacheService>();
 var deletedCount = cacheService.CleanupIncompleteCacheFolders();

README.md

@@ -1,3 +1,11 @@
+<p align="center">
+  <img
+    src="https://github.com/user-attachments/assets/684b73cd-a374-49e6-aaf1-ccc4c5697b47"
+    width="500"
+    alt="f67325fe-33ae-4fd8-9e57-38b46899d09c"
+  />
+</p>
+
 # BOM Local Service
 
 A local caching service for Australian Bureau of Meteorology (BOM) radar data, designed to provide a reliable API for Home Assistant integrations and other local services.
@@ -33,10 +41,25 @@ Built on ASP.NET Core 9.0, the service uses a service-oriented architecture with
 - **BomRadarService**: Main orchestrator that coordinates cache operations, browser automation, and data retrieval
 - **CacheService**: Manages file-based storage of radar screenshots and metadata in organized directory structures
 - **BrowserService**: Handles Playwright browser automation for headless browser sessions
-- **ScrapingService**: Performs web scraping operations using the browser service to navigate and capture radar data from the BOM website
+- **ScrapingService**: Coordinates web scraping workflows (simplified orchestrator)
+- **SelectorService**: Finds page elements using configurable CSS selectors with fallback support
 - **TimeParsingService**: Parses and converts time formats from BOM data
 - **DebugService**: Provides debug functionality for troubleshooting
 
+### Scraping Architecture
+The scraping system uses a **workflow-based architecture** with configurable steps:
+
+- **Workflows**: Define fixed sequences of steps for different data types (e.g., `RadarScrapingWorkflow`, `TemperatureMapWorkflow`). Each workflow specifies its response type via generics (`IWorkflow<TResponse>`)
+- **Steps**: Individual, testable units that perform specific actions (navigation, search, map interaction, capture). Steps declare prerequisites and validate page state before execution
+- **Step Registry**: Manages and discovers available scraping steps
+- **Workflow Factory**: Creates typed workflow instances based on configuration
+
+**Configuration-Driven Design:**
+- **Selectors**: All CSS selectors are configurable via `appsettings.json` with fallback options, allowing adaptation to website changes without code modifications
+- **JavaScript Templates**: JavaScript code for page evaluation is externalized in configuration, making it easy to update logic as the website evolves
+- **Text Patterns**: Regex patterns for parsing page content are configurable, enabling quick adjustments to parsing logic
+- **Workflow Steps**: Individual steps within workflows can be enabled/disabled via configuration, providing flexibility for testing and troubleshooting
+
 ### Background Services
 - **CacheManagementService**: Periodically checks cache validity for all cached locations and triggers updates when data expires
 - **CacheCleanupService**: Removes cache files older than the configured retention period
@@ -155,11 +178,23 @@ If you want to build locally, keep the `build:` section in `docker-compose.yml`
 docker-compose up -d
 ```
 
+**Mounting custom appsettings.json:**
+
+To use a custom configuration file, add it to the volumes section in `docker-compose.yml`:
+```yaml
+services:
+  bom-local-service:
+    volumes:
+      - ./cache:/app/cache
+      - ./appsettings.json:/app/appsettings.json:ro  # Custom config
+```
+
 This will:
 - Pull/build the image as configured
 - Start the service on port 8082 (configurable via `HOST_PORT`)
 - Mount the `./cache` directory for persistent storage
-- Apply all environment variable configurations
+- Mount custom `appsettings.json` if specified
+- Apply all environment variable configurations (which override appsettings.json)
 
 To view logs:
 ```bash
@@ -190,7 +225,7 @@ All configuration can be done via environment variables, which override the defa
 | Variable | Description | Default | Example |
 |----------|-------------|---------|---------|
 | `CACHEDIRECTORY` | Directory path for cache storage | `/app/cache` | `/data/bom-cache` |
-| `CACHERETENTIONHOURS` | Hours to retain cached data before cleanup | `24` | `48` |
+| `CACHERETENTIONHOURS` | Hours to retain cached data before cleanup (can be any positive integer) | `24` | `48`, `72`, `168` (1 week) |
 | `CACHEEXPIRATIONMINUTES` | Minutes before cache is considered expired | `12.5` | `15` |
 | `TIMEZONE` | Timezone for time parsing (IANA format) | `Australia/Brisbane` | `Australia/Sydney` |
 
@@ -213,8 +248,50 @@ All configuration can be done via environment variables, which override the defa
 
 | Variable | Description | Default | Example |
 |----------|-------------|---------|---------|
-| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `2000` | `3000` |
-| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `5000` | `7000` |
+| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `1500` | `2000` |
+| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `3000` | `5000` |
+| `SCREENSHOT__CROP__X` | X offset in pixels for screenshot cropping | `250` | `300` |
+| `SCREENSHOT__CROP__Y` | Y offset in pixels for screenshot cropping | `0` | `50` |
+| `SCREENSHOT__CROP__RIGHTOFFSET` | Right offset in pixels for screenshot cropping | `250` | `300` |
+| `SCREENSHOT__CROP__HEIGHT` | Height in pixels for screenshot cropping (null = full height) | `null` | `800` |
+
+#### Scraping Configuration
+
+The scraping system is highly configurable through `appsettings.json`. Most scraping settings (selectors, JavaScript templates, text patterns, workflow steps) are configured in `appsettings.json`, but can be overridden in Docker deployments.
+
+**Option 1: Mount Custom appsettings.json (Recommended for Docker)**
+
+Mount a custom `appsettings.json` file as a volume:
+
+```bash
+docker run -d \
+  --name bom-local-service \
+  -p 8082:8080 \
+  -v $(pwd)/cache:/app/cache \
+  -v $(pwd)/appsettings.json:/app/appsettings.json:ro \
+  --shm-size=1gb \
+  --ipc=host \
+  ghcr.io/alexhopeoconnor/bom-local-service:latest
+```
+
+Or in `docker-compose.yml`:
+```yaml
+services:
+  bom-local-service:
+    volumes:
+      - ./cache:/app/cache
+      - ./appsettings.json:/app/appsettings.json:ro  # Add this line
+```
+
+**Option 2: Environment Variables (Simple Overrides)**
+
+For quick overrides of commonly needed settings:
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `SCRAPING__BASEURL` | Base URL for the BOM website | `https://www.bom.gov.au/` | `https://www.bom.gov.au/` |
+
+**Note**: Complex configurations (selectors, JavaScript templates, text patterns, workflow steps) are best managed via a mounted `appsettings.json` file. See the [Configuration File](#configuration-file) section below for the complete structure.
 
 #### Debug Configuration
 
@@ -223,6 +300,13 @@ All configuration can be done via environment variables, which override the defa
 | `DEBUG__ENABLED` | Enable debug mode (saves debug screenshots) | `false` | `true` |
 | `DEBUG__WAITMS` | Additional wait time in debug mode | `2000` | `5000` |
 
+#### Time Series Configuration
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `TIMESERIES__WARNINGFOLDERCOUNT` | Number of cache folders that triggers a warning log when processing time series requests | `200` | `300` |
+| `TIMESERIES__MAXTIMERANGEHOURS` | Maximum time range allowed for time series queries (null = use CacheRetentionHours) | `null` (uses CacheRetentionHours) | `72` |
+
 #### Docker Compose Port Configuration
 
 | Variable | Description | Default |
@@ -291,6 +375,126 @@ Then run:
 docker-compose up -d
 ```
 
+#### Custom appsettings.json (For Selector/Scraping Configuration)
+
+If you need to customize selectors, JavaScript templates, or workflow steps:
+
+1. **Copy the default appsettings.json** from the repository
+2. **Edit the sections you need** (e.g., `Scraping:Selectors`)
+3. **Mount it as a volume**:
+
+```bash
+docker run -d \
+  --name bom-local-service \
+  -p 8082:8080 \
+  -v $(pwd)/cache:/app/cache \
+  -v $(pwd)/appsettings.json:/app/appsettings.json:ro \
+  --shm-size=1gb \
+  --ipc=host \
+  ghcr.io/alexhopeoconnor/bom-local-service:latest
+```
+
+Or in `docker-compose.yml`:
+```yaml
+services:
+  bom-local-service:
+    volumes:
+      - ./cache:/app/cache
+      - ./appsettings.json:/app/appsettings.json:ro
+```
+
+**Note**: Environment variables still override values in the mounted `appsettings.json`, so you can use env vars for simple overrides and the mounted file for complex configurations.
+
+### Configuration File
+
+For advanced scraping configuration, edit `appsettings.json` directly. When using Docker, you can mount a custom `appsettings.json` file as a volume (see [Scraping Configuration](#scraping-configuration) above).
+
+The scraping system supports extensive configuration:
+
+#### Scraping Selectors
+
+All CSS selectors used to find page elements are configurable with fallback options:
+
+```json
+{
+  "Scraping": {
+    "Selectors": {
+      "SearchButton": {
+        "Name": "Search Button",
+        "Selectors": [
+          "button[data-testid='searchLabel']",
+          "button[aria-label='Search for a location']",
+          "button.search-location__trigger-button"
+        ],
+        "TimeoutMs": 10000,
+        "Required": true,
+        "ErrorMessage": "Could not find search button"
+      }
+    }
+  }
+}
+```
+
+#### JavaScript Templates
+
+JavaScript code used for page evaluation is externalized and configurable:
+
+```json
+{
+  "Scraping": {
+    "JavaScriptTemplates": {
+      "WaitForSearchResults": "() => { /* template code */ }",
+      "ExtractSearchResults": "() => { /* template code */ }"
+    }
+  }
+}
+```
+
+#### Text Patterns
+
+Regex patterns for parsing page content are configurable:
+
+```json
+{
+  "Scraping": {
+    "TextPatterns": {
+      "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
+      "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
+      "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)"
+    }
+  }
+}
+```
+
+#### Workflow Steps
+
+Individual workflow steps can be enabled/disabled and configured:
+
+```json
+{
+  "Scraping": {
+    "Workflows": {
+      "RadarScraping": {
+        "Description": "Scrapes radar images for a location",
+        "Steps": {
+          "NavigateHomepage": { "Enabled": true },
+          "ClickSearchButton": { "Enabled": true },
+          "CaptureFrames": {
+            "Enabled": true,
+            "Parameters": {
+              "FrameCount": 7,
+              "WaitBetweenFrames": 5000
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+**Note**: Step order is fixed within workflows due to dependencies. Steps can be disabled but not reordered. See `appsettings.json` for the complete configuration structure.
+
 ## API Documentation
 
 The service provides RESTful API endpoints for accessing radar data and managing the cache.
@@ -335,10 +539,56 @@ GET /api/radar/{suburb}/{state}
 }
 ```
 
+**Response Fields:**
+- `frames`: Array of radar frame objects with image URLs and timing information
+- `observationTime`: UTC timestamp when the observation was made
+- `forecastTime`: UTC timestamp for the forecast
+- `weatherStation`: Name of the weather station
+- `distance`: Distance from location to weather station
+- `cacheIsValid`: Whether the cache is still valid (not expired)
+- `cacheExpiresAt`: UTC timestamp when the cache expires
+- `isUpdating`: Whether a cache update is currently in progress
+- `nextUpdateTime`: **Estimated** UTC timestamp for when the cache will be updated or when an in-progress update will complete. This value is calculated using:
+  - **Metrics-based estimation** (preferred): When historical data is available, uses median durations from previous cache updates to provide hardware-adaptive estimates
+  - **Calculated estimation** (fallback): When no metrics are available yet (e.g., first update), calculates based on configured wait times and frame count
+  - **Progress-aware**: During active updates, estimates improve as progress is tracked through phases (Initializing → CapturingFrames → Saving)
+
 **Status Codes:**
 - `200 OK`: Radar data available
 - `404 Not Found`: Cache is being generated (check response for retry information)
+  ```json
+  {
+    "errorCode": "CACHE_NOT_FOUND",
+    "errorType": "CacheError",
+    "message": "No cached data found for this location (fresh start). Cache update has been triggered in background.",
+    "details": {
+      "location": { "suburb": "Brisbane", "state": "QLD" },
+      "cacheExists": false,
+      "cacheIsValid": false,
+      "updateTriggered": true,
+      "nextUpdateTime": "2025-01-15T10:12:30Z"
+    },
+    "suggestions": {
+      "action": "retry_after_seconds",
+      "retryAfter": 30,
+      "refreshEndpoint": "/api/cache/Brisbane/QLD/refresh"
+    },
+    "note": "The retryAfter value is dynamically calculated based on the estimated cache update duration. On first startup with no cache, it uses a calculated estimate. After metrics are collected from completed updates, it uses hardware-adaptive estimates based on actual performance."
+    "timestamp": "2025-01-15T10:00:00Z"
+  }
+  ```
 - `400 Bad Request`: Invalid location parameters
+  ```json
+  {
+    "errorCode": "VALIDATION_ERROR",
+    "errorType": "ValidationError",
+    "message": "Invalid state abbreviation. Use: NSW, VIC, QLD, SA, WA, TAS, NT, ACT",
+    "details": {
+      "field": "state"
+    },
+    "timestamp": "2025-01-15T10:00:00Z"
+  }
+  ```
 
 #### Get Frame Image
 
@@ -357,6 +607,23 @@ GET /api/radar/{suburb}/{state}/frame/{frameIndex}
 **Response:**
 - `200 OK`: PNG image
 - `404 Not Found`: Frame not found
+  ```json
+  {
+    "errorCode": "NOT_FOUND",
+    "errorType": "NotFoundError",
+    "message": "Frame 3 not found for Brisbane, QLD",
+    "details": {
+      "resourceType": "Frame",
+      "identifier": "Frame 3 for Brisbane, QLD",
+      "frameIndex": 3,
+      "location": { "suburb": "Brisbane", "state": "QLD" }
+    },
+    "suggestions": {
+      "suggestion": "The frame may not exist yet. Try refreshing the cache or checking if cache update is in progress."
+    },
+    "timestamp": "2025-01-15T10:00:00Z"
+  }
+  ```
 
 #### Get Metadata
 
@@ -391,7 +658,7 @@ GET /api/radar/{suburb}/{state}/timeseries?startTime={iso8601}&endTime={iso8601}
 - `startTime` (query, optional): ISO 8601 start time (e.g., `2025-01-15T00:00:00Z`)
 - `endTime` (query, optional): ISO 8601 end time (defaults to now)
 
-**Response:**
+**Response (200 OK):**
 ```json
 {
   "cacheFolders": [
@@ -409,11 +676,147 @@ GET /api/radar/{suburb}/{state}/timeseries?startTime={iso8601}&endTime={iso8601}
       ]
     }
   ],
-  "totalFrames": 7,
-  "timeSpanMinutes": 60
+  "startTime": "2025-01-15T07:00:00Z",
+  "endTime": "2025-01-15T10:00:00Z",
+  "totalFrames": 7
 }
 ```
 
+**Status Codes:**
+- `200 OK`: Historical data available
+- `400 Bad Request`: Invalid request (e.g., time range exceeds maximum allowed duration, invalid time format, startTime after endTime)
+  - **Invalid time format**:
+    ```json
+    {
+      "errorCode": "VALIDATION_ERROR",
+      "errorType": "ValidationError",
+      "message": "Invalid startTime format. Use ISO 8601 format (e.g., 2025-12-07T00:00:00Z)",
+      "details": {
+        "field": "startTime"
+      },
+      "timestamp": "2025-01-15T10:00:00Z"
+    }
+    ```
+  - **Time range exceeds maximum**:
+    ```json
+    {
+      "errorCode": "TIME_RANGE_ERROR",
+      "errorType": "ValidationError",
+      "message": "Time range exceeds maximum allowed duration of 24 hours (based on cache retention: 24 hours). Please specify a smaller range.",
+      "details": {
+        "requestedRange": {
+          "start": "2025-01-15T00:00:00Z",
+          "end": "2025-01-15T25:00:00Z",
+          "requestedHours": 25.0
+        }
+      },
+      "suggestions": {
+        "action": "adjust_time_range"
+      },
+      "timestamp": "2025-01-15T10:00:00Z"
+    }
+    ```
+- `404 Not Found`: 
+  - **Location not cached**: No cache exists for this location. Cache update is triggered in background:
+    ```json
+    {
+      "errorCode": "CACHE_NOT_FOUND",
+      "errorType": "CacheError",
+      "message": "No cached data found for this location. Cache update has been triggered in background.",
+      "details": {
+        "location": { "suburb": "Brisbane", "state": "QLD" },
+        "cacheExists": false,
+        "cacheIsValid": false,
+        "updateTriggered": true,
+        "cacheExpiresAt": null,
+        "nextUpdateTime": "2025-01-15T10:12:30Z",
+        "statusMessage": "No cache exists, update triggered"
+      },
+      "suggestions": {
+        "action": "retry_after_seconds",
+        "retryAfter": 30,
+        "refreshEndpoint": "/api/cache/Brisbane/QLD/refresh",
+        "statusEndpoint": "/api/cache/Brisbane/QLD/range"
+      },
+      "timestamp": "2025-01-15T10:00:00Z"
+    }
+    ```
+  - **No data in range**: Cache exists but no data in the requested time range:
+    ```json
+    {
+      "errorCode": "TIME_RANGE_ERROR",
+      "errorType": "ValidationError",
+      "message": "No historical data found for the specified time range.",
+      "details": {
+        "availableRange": {
+          "oldest": "2025-01-15T08:00:00Z",
+          "newest": "2025-01-15T10:00:00Z",
+          "totalCacheFolders": 10,
+          "timeSpanMinutes": 120
+        },
+        "requestedRange": {
+          "start": "2025-01-15T00:00:00Z",
+          "end": "2025-01-15T10:00:00Z"
+        }
+      },
+      "suggestions": {
+        "action": "adjust_time_range",
+        "suggestedRange": {
+          "start": "2025-01-15T08:00:00Z",
+          "end": "2025-01-15T10:00:00Z"
+        },
+        "suggestion": "Try querying data between 2025-01-15T08:00:00Z and 2025-01-15T10:00:00Z"
+      },
+      "timestamp": "2025-01-15T10:00:00Z"
+    }
+    ```
+
+**Time Range Limits:**
+- Maximum time range is configurable via `TimeSeries:MaxTimeRangeHours` (defaults to `CacheRetentionHours` or minimum 24 hours)
+- If `TimeSeries:MaxTimeRangeHours` is not set, the limit automatically matches your `CacheRetentionHours` setting
+- This ensures you can always query all available cached data (e.g., if retention is 72 hours, you can query up to 72 hours)
+
+### Error Response Format
+
+All API endpoints return standardized error responses using the `ApiErrorResponse` format:
+
+```json
+{
+  "errorCode": "CACHE_NOT_FOUND",
+  "errorType": "CacheError",
+  "message": "Human-readable error message",
+  "details": {
+    "location": { "suburb": "Brisbane", "state": "QLD" },
+    "cacheExists": false,
+    "cacheIsValid": false
+  },
+  "suggestions": {
+    "action": "retry_after_seconds",
+    "retryAfter": 30,
+    "refreshEndpoint": "/api/cache/Brisbane/QLD/refresh"
+  },
+  "timestamp": "2025-01-15T10:00:00Z"
+}
+```
+
+**Error Response Fields:**
+- `errorCode`: Machine-readable error code (e.g., `CACHE_NOT_FOUND`, `VALIDATION_ERROR`, `TIME_RANGE_ERROR`)
+- `errorType`: Error category (`CacheError`, `ValidationError`, `ServiceError`, `NotFoundError`)
+- `message`: Human-readable error description
+- `details`: Additional context (varies by error type)
+- `suggestions`: Actionable guidance (retry times, endpoints, etc.)
+- `timestamp`: UTC timestamp when error occurred
+
+**Common Error Codes:**
+- `CACHE_NOT_FOUND`: No cached data exists for the location (fresh start scenario)
+- `VALIDATION_ERROR`: Invalid request parameters
+- `TIME_RANGE_ERROR`: Time range validation failed or no data in range
+- `NOT_FOUND`: Specific resource not found (e.g., frame, metadata)
+- `CACHE_UPDATE_FAILED`: Cache update operation failed
+- `INTERNAL_ERROR`: Server-side error occurred
+- If no time range is specified, returns all available historical data
+- **Note**: `CacheRetentionHours` can be set to any positive integer value (24, 48, 72, 168, etc.)
+
 #### Get Cache Range
 
 Get information about available historical cache data.
@@ -457,6 +860,11 @@ POST /api/cache/{suburb}/{state}/refresh
 }
 ```
 
+**Note on `nextUpdateTime`**: The estimated completion time is calculated using:
+- **Metrics-based estimation**: Uses historical median durations from previous cache updates (more accurate, hardware-adaptive)
+- **Calculated estimation**: Falls back to calculated estimates based on configuration when no metrics are available yet
+- Estimates improve in real-time as the update progresses through phases (Initializing → CapturingFrames → Saving)
+
 #### Delete Cache
 
 Delete cached data for a location.
@@ -479,6 +887,50 @@ When running in Development mode, OpenAPI documentation is available at:
 http://localhost:8082/openapi/v1.json
 ```
 
+## Cache Update Estimation
+
+The service uses a **metrics-based estimation system** to provide accurate estimates of cache update completion times. This ensures clients receive meaningful `nextUpdateTime` values that adapt to the actual hardware performance.
+
+### How It Works
+
+1. **Progress Tracking**: During cache updates, the service tracks progress through three phases:
+   - **Initializing**: Browser setup, navigation, and page loading (~0-20% of total time)
+   - **CapturingFrames**: Frame capture loop (~20-95% of total time)
+   - **Saving**: Metadata and cleanup operations (~95-100% of total time)
+
+2. **Metrics Collection**: After each successful cache update, the service records:
+   - Total duration of the update
+   - Duration of each phase (Initializing, CapturingFrames, Saving)
+   - Duration of each individual scraping step (NavigateHomepage, ClickSearchButton, etc.)
+   - Frame-level progress during capture
+
+3. **Estimation Strategy**:
+   - **Metrics-based** (preferred): Uses median durations from the last 20 completed updates to provide hardware-adaptive estimates
+   - **Progress-aware**: During active updates, estimates improve in real-time based on current phase and frame progress
+   - **Calculated fallback**: When no metrics are available (e.g., first update), falls back to calculated estimates based on configuration values
+
+4. **Benefits**:
+   - **Hardware-adaptive**: Estimates automatically adjust to slower/faster hardware
+   - **Improves over time**: More accurate estimates as more updates complete
+   - **Real-time refinement**: Estimates become more precise as updates progress
+   - **Works from clean start**: Provides reasonable estimates even on first run
+
+### Example Scenarios
+
+**First Update (No Metrics)**:
+- Uses calculated estimate based on `Screenshot:DynamicContentWaitMs`, `Screenshot:TileRenderWaitMs`, and frame count
+- Example: ~100 seconds for 7 frames with default settings (optimized wait times)
+
+**Subsequent Updates (With Metrics)**:
+- Uses median duration from historical data
+- Example: If previous updates averaged 95 seconds, estimates will use ~95 seconds (with buffer)
+
+**In-Progress Update**:
+- If capturing frame 3 of 7, estimates remaining time based on:
+  - Average frame duration from historical data
+  - Remaining frames (4 frames × avg frame duration)
+  - Plus estimated time for saving phase
+
 ## Demo SPA
 
 The service includes a built-in Single Page Application (SPA) for testing and demonstration purposes. This provides a visual interface to:
@@ -505,11 +957,11 @@ http://localhost:8082/radar/Brisbane/QLD
 
 - **Slideshow Playback**: Play, pause, and navigate through radar frames
 - **Frame Navigation**: Use slider, buttons, or keyboard shortcuts (arrow keys, spacebar)
-- **Extended Timespans**: View historical data from 1 hour to 24 hours
+- **Extended Timespans**: View historical data with configurable time ranges (based on cache retention settings)
 - **Custom Time Ranges**: Select specific start and end times for historical viewing
-- **Auto-Refresh**: Automatically checks for new data at configurable intervals
+- **Auto-Refresh**: Automatically checks for new data at configurable intervals (minimum 5 seconds, no maximum)
 - **Cache Status**: Real-time display of cache validity, expiration, and update status
-- **Settings Panel**: Configure frame intervals, refresh rates, and playback options
+- **Settings Panel**: Configure frame intervals (minimum 0.1 seconds, no maximum), refresh rates, and playback options
 
 ### Keyboard Shortcuts
 
@@ -531,15 +983,23 @@ async function getRadarData(suburb, state) {
     if (response.status === 404) {
         // Cache is being generated - trigger refresh and show message
         const error = await response.json();
-        if (error.refreshEndpoint) {
+        
+        // Use standardized error response format
+        if (error.errorCode === 'CACHE_NOT_FOUND' && error.suggestions?.refreshEndpoint) {
             // Trigger cache update in background
-            fetch(error.refreshEndpoint, { method: 'POST' }).catch(() => {});
+            fetch(error.suggestions.refreshEndpoint, { method: 'POST' }).catch(() => {});
         }
-        return { frames: [], message: `Cache being generated. Retry in ${error.retryAfter} seconds.` };
+        
+        const retryAfter = error.suggestions?.retryAfter || 30;
+        return { 
+            frames: [], 
+            message: error.message || `Cache being generated. Retry in ${retryAfter} seconds.` 
+        };
     }
     
     if (!response.ok) {
-        throw new Error(`HTTP ${response.status}`);
+        const error = await response.json().catch(() => ({ message: `HTTP ${response.status}` }));
+        throw new Error(error.message || `HTTP ${response.status}`);
     }
     
     return await response.json();
@@ -572,6 +1032,36 @@ async function getHistoricalRadar(suburb, state, hoursBack = 3) {
         `/api/radar/${suburb}/${state}/timeseries?startTime=${startTime.toISOString()}&endTime=${endTime.toISOString()}`
     );
     
+    if (response.status === 400) {
+        const error = await response.json();
+        // Use standardized error format
+        throw new Error(error.message || 'Invalid time range request');
+    }
+    
+    if (response.status === 404) {
+        const error = await response.json();
+        
+        // Check error code to determine type
+        if (error.errorCode === 'CACHE_NOT_FOUND') {
+            // Location doesn't exist - trigger cache update if endpoint provided
+            if (error.suggestions?.refreshEndpoint) {
+                fetch(error.suggestions.refreshEndpoint, { method: 'POST' }).catch(() => {});
+            }
+            throw new Error(error.message || 'Cache update triggered, please retry in a few moments.');
+        }
+        
+        // Cache exists but no data in range
+        if (error.errorCode === 'TIME_RANGE_ERROR' && error.details?.availableRange) {
+            const range = error.details.availableRange;
+            const rangeMsg = range.oldest && range.newest 
+                ? ` Available data: ${new Date(range.oldest).toLocaleString()} to ${new Date(range.newest).toLocaleString()}.`
+                : '';
+            throw new Error(error.message + rangeMsg);
+        }
+        
+        throw new Error(error.message || 'No historical data found');
+    }
+    
     if (!response.ok) {
         throw new Error(`HTTP ${response.status}`);
     }
@@ -648,7 +1138,29 @@ curl -X POST http://localhost:8082/api/cache/Brisbane/QLD/refresh
 
 **Get historical data (last 3 hours):**
 ```bash
-curl "http://localhost:8082/api/radar/Brisbane/QLD/timeseries?startTime=2025-01-15T07:00:00Z"
+curl "http://localhost:8082/api/radar/Brisbane/QLD/timeseries?startTime=2025-01-15T07:00:00Z&endTime=2025-01-15T10:00:00Z"
+```
+
+**Get historical data with error handling:**
+```bash
+# Check response status
+response=$(curl -s -w "\n%{http_code}" "http://localhost:8082/api/radar/Brisbane/QLD/timeseries?startTime=2025-01-15T07:00:00Z&endTime=2025-01-15T10:00:00Z")
+http_code=$(echo "$response" | tail -n1)
+body=$(echo "$response" | sed '$d')
+
+if [ "$http_code" = "200" ]; then
+    echo "Success: $body"
+elif [ "$http_code" = "400" ]; then
+    echo "Bad Request: $body"
+elif [ "$http_code" = "404" ]; then
+    echo "Not Found: $body"
+    # Check if cache update was triggered
+    if echo "$body" | grep -q "updateTriggered"; then
+        echo "Cache update triggered, retry in a few moments"
+    fi
+else
+    echo "Error ($http_code): $body"
+fi
 ```
 
 ## Troubleshooting
@@ -676,6 +1188,25 @@ curl "http://localhost:8082/api/radar/Brisbane/QLD/timeseries?startTime=2025-01-
 - **Check cache status**: Use `/api/cache/{suburb}/{state}/range` to verify cache exists
 - **Browser automation issues**: Check logs for Playwright errors
 
+### Scraping Failures
+
+If scraping fails (e.g., "Could not find element"), the BOM website structure may have changed:
+
+- **Check debug screenshots**: Enable `DEBUG__ENABLED=true` to see what the browser sees at each step
+- **Update selectors**: 
+  - **Docker**: Mount a custom `appsettings.json` with updated selectors (see [Scraping Configuration](#scraping-configuration))
+  - **Local**: Edit `appsettings.json` under `Scraping:Selectors` to add new CSS selectors as fallbacks
+- **Check step logs**: Each step logs its execution - look for which step failed
+- **Selector fallbacks**: The system tries multiple selectors in order, so add new selectors to the existing arrays
+- **Workflow steps**: Individual steps can be disabled via `Scraping:Workflows:RadarScraping:Steps:{StepName}:Enabled: false` if needed temporarily
+
+**Quick Fix for Docker Users:**
+
+1. Copy the default `appsettings.json` from the repository
+2. Edit the selectors that are failing
+3. Mount it as a volume: `-v $(pwd)/appsettings.json:/app/appsettings.json:ro`
+4. Restart the container
+
 ### Playwright Resource Usage
 
 Playwright browsers (Chromium) can consume significant CPU and memory:
@@ -709,6 +1240,25 @@ docker run --cpus="1.0" ...
 - **Increase cleanup frequency**: Lower `CACHECLEANUP__INTERVALHOURS` to clean up more often
 - **Limit locations**: The service automatically manages all cached locations; reduce the number of locations being cached to lower resource usage
 
+### Performance Monitoring
+
+The service logs detailed performance metrics for each scraping workflow:
+
+**Step-Level Timing**:
+- Each step logs its duration and compares it to historical averages
+- Example: `Step WaitForMapReady completed in 45.30s (avg: 43.76s)`
+- Steps that are >50% slower than average trigger warnings: `⚠️ Step WaitForMapReady took significantly longer than average: 75.45s (avg: 50.30s, +25.15s, +50.0% slower)`
+
+**Workflow-Level Timing**:
+- Complete workflow duration is logged with step breakdown
+- Example: `Workflow RadarScraping completed in 144.40s. Step breakdown: NavigateHomepage=4.96s, ClickSearchButton=2.70s, ...`
+- Workflows that are >30% slower than average trigger warnings: `⚠️ Workflow RadarScraping took significantly longer than average: 189.45s (avg: 145.67s, +43.78s, +30.0% slower)`
+
+**Metrics Storage**:
+- Step and phase durations are stored in memory (last 20 samples)
+- Used for performance estimation and identifying bottlenecks
+- Metrics improve over time as more updates complete
+
 ## Development
 
 ### Building Locally
@@ -727,7 +1277,45 @@ Enable debug mode to save intermediate screenshots during data capture:
 docker run -e DEBUG__ENABLED=true -e DEBUG__WAITMS=5000 ...
 ```
 
-Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`.
+Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`. Each scraping step saves a screenshot, HTML snapshot, and logs, making it easy to diagnose issues.
+
+### Extending the Scraping System
+
+The workflow-based architecture makes it easy to extend the scraping system:
+
+**Adding a New Workflow**:
+
+1. Create a new workflow class in `Services/Scraping/Workflows/` implementing `IWorkflow<TResponse>` where `TResponse` is your response type
+2. Define the step sequence (can reuse existing steps)
+3. Register the workflow in `WorkflowFactory`
+4. Add workflow configuration to `appsettings.json`
+
+**Adding a New Step**:
+
+1. Create a step class inheriting from `BaseScrapingStep`
+2. Implement `Name`, `Prerequisites`, `CanExecute`, and `ExecuteAsync`
+3. The step will be auto-registered on startup
+4. Add the step to a workflow's `StepNames` array
+
+**Updating Selectors**:
+
+1. Edit `appsettings.json` under `Scraping:Selectors`
+2. Add new CSS selectors to the `Selectors` array (tried in order)
+3. Adjust `TimeoutMs` if needed
+4. No code changes required
+
+**Updating JavaScript Templates**:
+
+1. Edit `appsettings.json` under `Scraping:JavaScriptTemplates`
+2. Update the template code as needed
+3. No code changes required
+
+**Updating Text Patterns**:
+
+1. Edit `appsettings.json` under `Scraping:TextPatterns`
+2. Update regex patterns as needed (e.g., `TimestampPattern`)
+3. The `TimestampPattern` supports parsing timestamps like "Wednesday 17 Dec, 11:05 pm" when the BOM website changes format
+4. No code changes required
 
 ## License
 

Services/BomRadarService.cs

@@ -15,6 +15,8 @@ public class BomRadarService : IBomRadarService, IDisposable
     private readonly IConfiguration _configuration;
     private readonly double _cacheExpirationMinutes;
     private readonly int _cacheManagementCheckIntervalMinutes;
+    private readonly int _timeSeriesWarningFolderCount;
+    private readonly int _estimatedUpdateDurationSeconds;
 
     public BomRadarService(
         ILogger<BomRadarService> logger,
@@ -30,8 +32,30 @@ public class BomRadarService : IBomRadarService, IDisposable
         _scrapingService = scrapingService;
         _debugService = debugService;
         _configuration = configuration;
-        _cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
+        _cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 12.5);
         _cacheManagementCheckIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
+        _timeSeriesWarningFolderCount = configuration.GetValue<int>("TimeSeries:WarningFolderCount", 200);
+        
+        // Calculate estimated cache update duration
+        _estimatedUpdateDurationSeconds = CacheHelper.GetEstimatedUpdateDurationSeconds(configuration, CachedDataType.Radar);
+        _logger.LogInformation("Estimated cache update duration: {Seconds} seconds", _estimatedUpdateDurationSeconds);
+        
+        if (_cacheExpirationMinutes <= 0)
+        {
+            throw new ArgumentException("CacheExpirationMinutes must be greater than 0", nameof(configuration));
+        }
+        if (_cacheManagementCheckIntervalMinutes <= 0 || _cacheManagementCheckIntervalMinutes > 60)
+        {
+            throw new ArgumentException("CacheManagement:CheckIntervalMinutes must be between 1 and 60", nameof(configuration));
+        }
+        if (_timeSeriesWarningFolderCount <= 0)
+        {
+            throw new ArgumentException("TimeSeries:WarningFolderCount must be greater than 0", nameof(configuration));
+        }
+        if (_estimatedUpdateDurationSeconds <= 0)
+        {
+            throw new ArgumentException("Calculated estimated update duration must be greater than 0. Check Screenshot:TileRenderWaitMs, Screenshot:DynamicContentWaitMs, and CachedDataTypes:Radar:FrameCount configuration values.", nameof(configuration));
+        }
     }
 
     public async Task<RadarResponse?> GetCachedRadarAsync(string suburb, string state, CancellationToken cancellationToken = default)
@@ -56,7 +80,11 @@ public class BomRadarService : IBomRadarService, IDisposable
         var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null;
         var isUpdating = _cacheService.IsLocationUpdating(locationKey);
         
-        return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating, _cacheManagementCheckIntervalMinutes);
+        // Use metrics-based estimate if available, otherwise fallback
+        var estimatedDuration = _cacheService.GetEstimatedRemainingSeconds(locationKey);
+        var durationToUse = estimatedDuration > 0 ? estimatedDuration : _estimatedUpdateDurationSeconds;
+        
+        return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, _cacheManagementCheckIntervalMinutes, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating, durationToUse);
     }
     
     public async Task<List<RadarFrame>?> GetCachedFramesAsync(string suburb, string state, CancellationToken cancellationToken = default)
@@ -104,7 +132,19 @@ public class BomRadarService : IBomRadarService, IDisposable
             
             status.UpdateTriggered = false;
             status.Message = "Cache update already in progress";
-            status.NextUpdateTime = status.CacheExpiresAt ?? DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes);
+            
+            // Use metrics-based estimate if available, otherwise fallback
+            var remainingSeconds = _cacheService.GetEstimatedRemainingSeconds(locationKey);
+            if (remainingSeconds > 0)
+            {
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(remainingSeconds);
+            }
+            else
+            {
+                // Fallback to calculated estimate
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(_estimatedUpdateDurationSeconds);
+            }
+            
             return status;
         }
         
@@ -144,14 +184,16 @@ public class BomRadarService : IBomRadarService, IDisposable
                 ? "Cache is stale, update triggered" 
                 : "No cache exists, update triggered";
             
-            // Calculate next update time
-            if (status.CacheExpiresAt.HasValue && status.CacheExpiresAt.Value > DateTime.UtcNow)
+            // Try metrics-based estimate first, then fallback to calculated
+            var remainingSeconds = _cacheService.GetEstimatedRemainingSeconds(locationKey);
+            if (remainingSeconds > 0)
             {
-                status.NextUpdateTime = status.CacheExpiresAt.Value;
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(remainingSeconds);
             }
             else
             {
-                status.NextUpdateTime = DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes);
+                // Fallback to calculated estimate
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(_estimatedUpdateDurationSeconds);
             }
         }
         else
@@ -183,7 +225,12 @@ public class BomRadarService : IBomRadarService, IDisposable
                 var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
                 var isValid = cachedMetadata != null && _cacheService.IsCacheValid(cachedMetadata);
                 var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null;
-                return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: true, _cacheManagementCheckIntervalMinutes);
+                
+                // Use metrics-based estimate if available, otherwise fallback
+                var estimatedDuration = _cacheService.GetEstimatedRemainingSeconds(locationKey);
+                var durationToUse = estimatedDuration > 0 ? estimatedDuration : _estimatedUpdateDurationSeconds;
+                
+                return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, _cacheManagementCheckIntervalMinutes, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: true, durationToUse);
             }
             else
             {
@@ -200,7 +247,7 @@ public class BomRadarService : IBomRadarService, IDisposable
                 _logger.LogInformation("Returning valid cached screenshots for {Suburb}, {State} (no semaphore needed)", suburb, state);
                 var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
                 var cacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
-                return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: false, _cacheManagementCheckIntervalMinutes);
+                return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, _cacheManagementCheckIntervalMinutes, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: false, _estimatedUpdateDurationSeconds);
             }
             else
             {
@@ -250,7 +297,7 @@ public class BomRadarService : IBomRadarService, IDisposable
                 _logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshots");
                 var recheckFrames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
                 var recheckCacheExpiresAt = recheckCachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
-                return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state, cacheIsValid: true, recheckCacheExpiresAt, isUpdating: false, _cacheManagementCheckIntervalMinutes);
+                return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, _cacheManagementCheckIntervalMinutes, recheckCachedMetadata, suburb, state, cacheIsValid: true, recheckCacheExpiresAt, isUpdating: false);
             }
             
             // Create debug folder only now
@@ -275,11 +322,25 @@ public class BomRadarService : IBomRadarService, IDisposable
                 // Remove from active tracking once complete
                 _cacheService.ClearActiveCacheFolder(locationKey);
                 
-                // Update result with cache state
-                result.IsUpdating = false;
-                result.CacheIsValid = true;
-                result.CacheExpiresAt = result.ObservationTime.AddMinutes(_cacheExpirationMinutes);
-                result.NextUpdateTime = result.CacheExpiresAt;
+                // Update result with cache state - rebuild response to get correct NextUpdateTime calculation
+                var cacheExpiresAt = result.ObservationTime.AddMinutes(_cacheExpirationMinutes);
+                var metadata = new LastUpdatedInfo
+                {
+                    ObservationTime = result.ObservationTime,
+                    ForecastTime = result.ForecastTime,
+                    WeatherStation = result.WeatherStation,
+                    Distance = result.Distance
+                };
+                result = ResponseBuilder.CreateRadarResponse(
+                    cacheFolderPath: newCacheFolderPath,
+                    frames: result.Frames,
+                    metadata: metadata,
+                    suburb: suburb,
+                    state: state,
+                    cacheIsValid: true,
+                    cacheExpiresAt: cacheExpiresAt,
+                    isUpdating: false,
+                    cacheManagementCheckIntervalMinutes: _cacheManagementCheckIntervalMinutes);
                 
                 return result;
             }
@@ -406,6 +467,16 @@ public class BomRadarService : IBomRadarService, IDisposable
             filteredFolders = filteredFolders.Where(f => f.CacheTimestamp <= endTime.Value);
         }
         
+        var filteredFoldersList = filteredFolders.ToList();
+        
+        // Warn if processing a large number of folders (configurable threshold)
+        if (filteredFoldersList.Count > _timeSeriesWarningFolderCount)
+        {
+            _logger.LogWarning(
+                "Processing large number of cache folders ({Count}) for time series request. Suburb: {Suburb}, State: {State}, StartTime: {StartTime}, EndTime: {EndTime}",
+                filteredFoldersList.Count, suburb, state, startTime, endTime);
+        }
+        
         var result = new List<RadarCacheFolderFrames>();
         var encodedSuburb = Uri.EscapeDataString(suburb);
         var encodedState = Uri.EscapeDataString(state);
@@ -414,7 +485,7 @@ public class BomRadarService : IBomRadarService, IDisposable
         // Use a HashSet to track unique absolute observation times we've already seen
         var seenAbsoluteTimes = new HashSet<DateTime>();
         
-        foreach (var folderInfo in filteredFolders.OrderByDescending(f => f.CacheTimestamp))
+        foreach (var folderInfo in filteredFoldersList.OrderByDescending(f => f.CacheTimestamp))
         {
             // Load frames from this folder's radar subfolder
             var cachedFrames = await _cacheService.GetFramesFromCacheFolderAsync(

Services/CacheManagementService.cs

@@ -25,6 +25,12 @@ public class CacheManagementService : BackgroundService
         _browserService = browserService;
         _configuration = configuration;
         var checkIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
+        
+        if (checkIntervalMinutes <= 0 || checkIntervalMinutes > 60)
+        {
+            throw new ArgumentException("CacheManagement:CheckIntervalMinutes must be between 1 and 60", nameof(configuration));
+        }
+        
         _checkInterval = TimeSpan.FromMinutes(checkIntervalMinutes);
     }
 

Services/CacheService.cs

@@ -13,13 +13,26 @@ public class CacheService : ICacheService
     private readonly double _cacheExpirationMinutes;
     private readonly IConfiguration _configuration;
     private readonly ConcurrentDictionary<string, string> _activeCacheFolders = new(); // locationKey -> cacheFolderPath
+    
+    // Progress tracking for cache updates
+    private readonly ConcurrentDictionary<string, (DateTime startTime, CacheUpdatePhase phase, int? currentFrame, int? totalFrames)> _updateProgress = new();
+    private readonly ConcurrentQueue<double> _recentTotalDurations = new(); // Overall durations in seconds
+    private readonly ConcurrentDictionary<CacheUpdatePhase, ConcurrentQueue<double>> _phaseDurations = new(); // Phase -> durations
+    private readonly ConcurrentDictionary<string, ConcurrentQueue<double>> _stepDurations = new(); // Step name -> durations
+    private readonly object _metricsLock = new();
+    private const int MaxSamples = 20;
 
     public CacheService(ILogger<CacheService> logger, IConfiguration configuration)
     {
         _logger = logger;
         _configuration = configuration;
         _cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
-        _cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
+        _cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 12.5);
+        
+        if (_cacheExpirationMinutes <= 0)
+        {
+            throw new ArgumentException("CacheExpirationMinutes must be greater than 0", nameof(configuration));
+        }
         
         // Ensure cache directory exists
         Directory.CreateDirectory(_cacheDirectory);
@@ -90,7 +103,10 @@ public class CacheService : ICacheService
         string state, 
         CancellationToken cancellationToken = default)
     {
-        var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, CachedDataType.Radar, null, cancellationToken);
+        // Exclude active cache folder (currently being written to) to avoid reading incomplete data
+        var locationKey = LocationHelper.GetLocationKey(suburb, state);
+        var excludeFolder = GetActiveCacheFolder(locationKey);
+        var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, CachedDataType.Radar, excludeFolder, cancellationToken);
         
         if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
         {
@@ -116,7 +132,10 @@ public class CacheService : ICacheService
             return null;
         }
         
-        var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, CachedDataType.Radar, null, cancellationToken);
+        // Exclude active cache folder (currently being written to) to avoid reading incomplete data
+        var locationKey = LocationHelper.GetLocationKey(suburb, state);
+        var excludeFolder = GetActiveCacheFolder(locationKey);
+        var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, CachedDataType.Radar, excludeFolder, cancellationToken);
         
         if (string.IsNullOrEmpty(cacheFolderPath))
         {
@@ -319,6 +338,10 @@ public class CacheService : ICacheService
         string state, 
         CancellationToken cancellationToken = default)
     {
+        // Exclude active cache folder (currently being written to) to avoid reading incomplete data
+        var locationKey = LocationHelper.GetLocationKey(suburb, state);
+        var excludeFolder = GetActiveCacheFolder(locationKey);
+        
         var pattern = FilePathHelper.GetCacheFolderPattern(suburb, state);
         var folders = Directory.GetDirectories(_cacheDirectory, pattern)
             .Select(f =>
@@ -339,6 +362,13 @@ public class CacheService : ICacheService
             if (!Directory.Exists(folder.Folder))
                 continue;
             
+            // Skip the folder if it's being excluded (currently being written to)
+            if (!string.IsNullOrEmpty(excludeFolder) && Path.GetFullPath(folder.Folder).Equals(Path.GetFullPath(excludeFolder), StringComparison.OrdinalIgnoreCase))
+            {
+                _logger.LogDebug("Skipping excluded cache folder (being written to): {Folder}", folder.Folder);
+                continue;
+            }
+            
             var metadata = await LoadMetadataAsync(folder.Folder, cancellationToken);
             if (metadata == null)
                 continue;
@@ -497,6 +527,7 @@ public class CacheService : ICacheService
     public void SetActiveCacheFolder(string locationKey, string cacheFolderPath)
     {
         _activeCacheFolders[locationKey] = cacheFolderPath;
+        _updateProgress[locationKey] = (DateTime.UtcNow, CacheUpdatePhase.Initializing, null, null);
         _logger.LogDebug("Tracking active cache folder: {Folder} for location: {Location}", cacheFolderPath, locationKey);
     }
     
@@ -504,10 +535,259 @@ public class CacheService : ICacheService
     {
         if (_activeCacheFolders.TryRemove(locationKey, out var folder))
         {
+            RecordUpdateComplete(locationKey);
+            _updateProgress.TryRemove(locationKey, out _);
             _logger.LogDebug("Cleared active cache folder tracking: {Folder} for location: {Location}", folder, locationKey);
         }
     }
     
+    /// <summary>
+    /// Records progress update for a cache update operation.
+    /// </summary>
+    public void RecordUpdateProgress(string locationKey, CacheUpdatePhase phase, int? currentFrame = null, int? totalFrames = null)
+    {
+        if (_updateProgress.TryGetValue(locationKey, out var existing))
+        {
+            // Record duration for previous phase if it changed
+            var previousPhase = existing.phase;
+            if (previousPhase != phase)
+            {
+                var phaseDuration = (DateTime.UtcNow - existing.startTime).TotalSeconds;
+                
+                lock (_metricsLock)
+                {
+                    var durations = _phaseDurations.GetOrAdd(previousPhase, _ => new ConcurrentQueue<double>());
+                    durations.Enqueue(phaseDuration);
+                    
+                    while (durations.Count > MaxSamples)
+                    {
+                        durations.TryDequeue(out _);
+                    }
+                }
+            }
+        }
+        
+        // Update progress
+        var startTime = _updateProgress.TryGetValue(locationKey, out var current) ? current.startTime : DateTime.UtcNow;
+        _updateProgress[locationKey] = (startTime, phase, currentFrame, totalFrames);
+    }
+    
+    /// <summary>
+    /// Records completion of a cache update and stores metrics.
+    /// </summary>
+    private void RecordUpdateComplete(string locationKey)
+    {
+        if (_updateProgress.TryRemove(locationKey, out var progress))
+        {
+            var totalDuration = (DateTime.UtcNow - progress.startTime).TotalSeconds;
+            
+            lock (_metricsLock)
+            {
+                _recentTotalDurations.Enqueue(totalDuration);
+                while (_recentTotalDurations.Count > MaxSamples)
+                {
+                    _recentTotalDurations.TryDequeue(out _);
+                }
+            }
+            
+            _logger.LogInformation("Cache update completed in {Duration:F1} seconds for {Location}", totalDuration, locationKey);
+        }
+    }
+    
+    /// <summary>
+    /// Gets the estimated remaining seconds for an in-progress cache update.
+    /// Returns 0 if not updating or no metrics available.
+    /// </summary>
+    public int GetEstimatedRemainingSeconds(string locationKey)
+    {
+        if (!_updateProgress.TryGetValue(locationKey, out var progress))
+        {
+            return 0; // Not updating
+        }
+        
+        var elapsed = (DateTime.UtcNow - progress.startTime).TotalSeconds;
+        
+        // If we have historical data, use it
+        var avgTotal = GetAverageTotalDuration();
+        if (avgTotal > 0)
+        {
+            // Estimate based on phase and progress
+            double estimatedRemaining = 0;
+            
+            switch (progress.phase)
+            {
+                case CacheUpdatePhase.Initializing:
+                    // Estimate: avg total - elapsed (with some buffer)
+                    estimatedRemaining = Math.Max(0, avgTotal * 1.1 - elapsed);
+                    break;
+                    
+                case CacheUpdatePhase.CapturingFrames:
+                    if (progress.currentFrame.HasValue && progress.totalFrames.HasValue)
+                    {
+                        // We know exactly where we are: frame X of Y
+                        var framesRemaining = progress.totalFrames.Value - progress.currentFrame.Value - 1;
+                        var avgFrameDuration = GetAverageFrameDuration();
+                        if (avgFrameDuration > 0)
+                        {
+                            // Time for remaining frames + saving phase
+                            estimatedRemaining = (framesRemaining * avgFrameDuration) + GetAveragePhaseDuration(CacheUpdatePhase.Saving);
+                        }
+                        else
+                        {
+                            // Fallback: estimate based on progress through total
+                            var progressFraction = (progress.currentFrame.Value + 1.0) / progress.totalFrames.Value;
+                            estimatedRemaining = Math.Max(0, (avgTotal / progressFraction) - elapsed);
+                        }
+                    }
+                    else
+                    {
+                        // No frame info - estimate based on elapsed time
+                        estimatedRemaining = Math.Max(0, avgTotal - elapsed);
+                    }
+                    break;
+                    
+                case CacheUpdatePhase.Saving:
+                    // Almost done - just saving phase remaining
+                    estimatedRemaining = GetAveragePhaseDuration(CacheUpdatePhase.Saving);
+                    if (estimatedRemaining == 0)
+                    {
+                        estimatedRemaining = 5; // Default 5 seconds for saving
+                    }
+                    break;
+            }
+            
+            return (int)Math.Ceiling(Math.Max(0, estimatedRemaining));
+        }
+        
+        // No historical data - return 0 to signal fallback
+        return 0;
+    }
+    
+    /// <summary>
+    /// Gets the average total duration of cache updates from recent metrics.
+    /// </summary>
+    public double GetAverageTotalDuration()
+    {
+        lock (_metricsLock)
+        {
+            if (_recentTotalDurations.Count == 0) return 0;
+            var durations = _recentTotalDurations.ToArray();
+            Array.Sort(durations);
+            // Use median for robustness
+            var median = durations.Length % 2 == 0
+                ? (durations[durations.Length / 2 - 1] + durations[durations.Length / 2]) / 2.0
+                : durations[durations.Length / 2];
+            return median;
+        }
+    }
+    
+    /// <summary>
+    /// Gets the average duration per frame based on historical CapturingFrames phase data.
+    /// </summary>
+    private double GetAverageFrameDuration()
+    {
+        // Estimate frame duration from CapturingFrames phase duration / frame count
+        var avgCapturingDuration = GetAveragePhaseDuration(CacheUpdatePhase.CapturingFrames);
+        var frameCount = CacheHelper.GetFrameCountForDataType(_configuration, CachedDataType.Radar);
+        return avgCapturingDuration > 0 && frameCount > 0 ? avgCapturingDuration / frameCount : 0;
+    }
+    
+    /// <summary>
+    /// Gets the average duration for a specific phase from historical data.
+    /// </summary>
+    private double GetAveragePhaseDuration(CacheUpdatePhase phase)
+    {
+        lock (_metricsLock)
+        {
+            if (!_phaseDurations.TryGetValue(phase, out var durations) || durations.Count == 0)
+            {
+                return 0;
+            }
+            var durationsArray = durations.ToArray();
+            return durationsArray.Average();
+        }
+    }
+    
+    /// <summary>
+    /// Records step completion timing for metrics tracking.
+    /// </summary>
+    public void RecordStepCompletion(string stepName, double durationSeconds)
+    {
+        lock (_metricsLock)
+        {
+            var durations = _stepDurations.GetOrAdd(stepName, _ => new ConcurrentQueue<double>());
+            durations.Enqueue(durationSeconds);
+            
+            while (durations.Count > MaxSamples)
+            {
+                durations.TryDequeue(out _);
+            }
+        }
+    }
+    
+    /// <summary>
+    /// Gets the average duration for a specific step from historical data.
+    /// </summary>
+    public double GetAverageStepDuration(string stepName)
+    {
+        lock (_metricsLock)
+        {
+            if (!_stepDurations.TryGetValue(stepName, out var durations) || durations.Count == 0)
+            {
+                return 0;
+            }
+            var durationsArray = durations.ToArray();
+            return durationsArray.Average();
+        }
+    }
+    
+    /// <summary>
+    /// Gets step performance metrics for debugging/logging.
+    /// </summary>
+    public Dictionary<string, double> GetStepMetrics()
+    {
+        lock (_metricsLock)
+        {
+            var metrics = new Dictionary<string, double>();
+            foreach (var kvp in _stepDurations)
+            {
+                if (kvp.Value.Count > 0)
+                {
+                    var durations = kvp.Value.ToArray();
+                    metrics[kvp.Key] = durations.Average();
+                }
+            }
+            return metrics;
+        }
+    }
+    
+    /// <summary>
+    /// Gets the locationKey from a cacheFolderPath by parsing the folder name.
+    /// </summary>
+    private string? GetLocationKeyFromCacheFolder(string cacheFolderPath)
+    {
+        var folderName = Path.GetFileName(cacheFolderPath);
+        var location = LocationHelper.ParseLocationFromFilename(folderName);
+        if (location.HasValue)
+        {
+            return LocationHelper.GetLocationKey(location.Value.suburb, location.Value.state);
+        }
+        return null;
+    }
+    
+    /// <summary>
+    /// Records progress update for a cache update operation using cacheFolderPath.
+    /// This is useful when locationKey is not directly available (e.g., in ScrapingService).
+    /// </summary>
+    public void RecordUpdateProgressByFolder(string cacheFolderPath, CacheUpdatePhase phase, int? currentFrame = null, int? totalFrames = null)
+    {
+        var locationKey = GetLocationKeyFromCacheFolder(cacheFolderPath);
+        if (locationKey != null)
+        {
+            RecordUpdateProgress(locationKey, phase, currentFrame, totalFrames);
+        }
+    }
+    
     public string CreateCacheFolder(string suburb, string state, string timestamp)
     {
         var cacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheDirectory, suburb, state, timestamp);
@@ -605,8 +885,18 @@ public class CacheService : ICacheService
         if (isUpdating)
         {
             status.Message = "Cache update already in progress";
-            // Update in progress - estimate completion in ~2 minutes
-            status.NextUpdateTime = DateTime.UtcNow.AddMinutes(2);
+            // Try metrics-based estimate first
+            var remainingSeconds = GetEstimatedRemainingSeconds(locationKey);
+            if (remainingSeconds > 0)
+            {
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(remainingSeconds);
+            }
+            else
+            {
+                // Fallback to calculated estimate
+                var estimatedDurationSeconds = CacheHelper.GetEstimatedUpdateDurationSeconds(_configuration, dataType);
+                status.NextUpdateTime = DateTime.UtcNow.AddSeconds(estimatedDurationSeconds);
+            }
         }
         else if (status.CacheIsValid && status.CacheExpiresAt.HasValue)
         {
@@ -618,9 +908,7 @@ public class CacheService : ICacheService
         {
             // Cache is invalid/missing - next update would be after background service check
             status.Message = status.CacheExists ? "Cache is stale" : "No cache exists";
-            var now = DateTime.UtcNow;
-            var minutesUntilNextCheck = cacheManagementCheckIntervalMinutes - (now.Minute % cacheManagementCheckIntervalMinutes);
-            status.NextUpdateTime = now.AddMinutes(minutesUntilNextCheck);
+            status.NextUpdateTime = ResponseBuilder.CalculateNextServiceCheck(cacheManagementCheckIntervalMinutes);
         }
         
         return status;

Services/DebugService.cs

@@ -90,18 +90,22 @@ namespace BomLocalService.Services;
             await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
 
             // Save console messages if provided
+            // Create a snapshot to avoid collection modification during enumeration
             if (consoleMessages != null && consoleMessages.Count > 0)
             {
                 var consolePath = Path.Combine(stepFolder, "console.log");
-                var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
+                var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
+                var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
                 await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
             }
 
             // Save network requests summary if provided
+            // Create a snapshot to avoid collection modification during enumeration
             if (networkRequests != null && networkRequests.Count > 0)
             {
                 var networkPath = Path.Combine(stepFolder, "network.log");
-                var networkText = string.Join("\n", networkRequests.Select(req => 
+                var networkSnapshot = networkRequests.ToList(); // Create snapshot
+                var networkText = string.Join("\n", networkSnapshot.Select(req => 
                     $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
                 ));
                 await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
@@ -148,18 +152,22 @@ namespace BomLocalService.Services;
             }
 
             // Save console messages if provided
+            // Create a snapshot to avoid collection modification during enumeration
             if (consoleMessages != null && consoleMessages.Count > 0)
             {
                 var consolePath = Path.Combine(errorFolder, "console.log");
-                var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
+                var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
+                var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
                 await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
             }
 
             // Save network requests summary if provided
+            // Create a snapshot to avoid collection modification during enumeration
             if (networkRequests != null && networkRequests.Count > 0)
             {
                 var networkPath = Path.Combine(errorFolder, "network.log");
-                var networkText = string.Join("\n", networkRequests.Select(req => 
+                var networkSnapshot = networkRequests.ToList(); // Create snapshot
+                var networkText = string.Join("\n", networkSnapshot.Select(req => 
                     $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
                 ));
                 await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);

Services/Interfaces/ICacheService.cs

@@ -195,5 +195,50 @@ public interface ICacheService
         int cacheExpirationMinutes,
         int cacheManagementCheckIntervalMinutes,
         CancellationToken cancellationToken = default);
+    
+    /// <summary>
+    /// Records progress update for a cache update operation using cacheFolderPath.
+    /// This is useful when locationKey is not directly available (e.g., in ScrapingService).
+    /// </summary>
+    /// <param name="cacheFolderPath">The cache folder path being updated</param>
+    /// <param name="phase">The current phase of the update</param>
+    /// <param name="currentFrame">The current frame being captured (if in CapturingFrames phase)</param>
+    /// <param name="totalFrames">The total number of frames to capture</param>
+    void RecordUpdateProgressByFolder(string cacheFolderPath, CacheUpdatePhase phase, int? currentFrame = null, int? totalFrames = null);
+    
+    /// <summary>
+    /// Gets the estimated remaining seconds for an in-progress cache update.
+    /// Returns 0 if not updating or no metrics available.
+    /// </summary>
+    /// <param name="locationKey">The location key (suburb_state)</param>
+    /// <returns>Estimated remaining seconds, or 0 if not updating or no metrics</returns>
+    int GetEstimatedRemainingSeconds(string locationKey);
+    
+    /// <summary>
+    /// Records step completion timing for metrics tracking.
+    /// </summary>
+    /// <param name="stepName">The name of the step</param>
+    /// <param name="durationSeconds">The duration of the step in seconds</param>
+    void RecordStepCompletion(string stepName, double durationSeconds);
+    
+    /// <summary>
+    /// Gets the average duration for a specific step from historical data.
+    /// </summary>
+    /// <param name="stepName">The name of the step</param>
+    /// <returns>Average duration in seconds, or 0 if no data available</returns>
+    double GetAverageStepDuration(string stepName);
+    
+    /// <summary>
+    /// Gets step performance metrics for debugging/logging.
+    /// </summary>
+    /// <returns>Dictionary mapping step names to their average durations</returns>
+    Dictionary<string, double> GetStepMetrics();
+    
+    /// <summary>
+    /// Gets the average total duration of cache updates from recent metrics.
+    /// Uses median for robustness against outliers.
+    /// </summary>
+    /// <returns>Average total duration in seconds, or 0 if no data available</returns>
+    double GetAverageTotalDuration();
 }
 

Services/Interfaces/ISelectorService.cs

@@ -0,0 +1,21 @@
+using BomLocalService.Models;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Interfaces;
+
+/// <summary>
+/// Service for finding page elements using configured selectors
+/// </summary>
+public interface ISelectorService
+{
+    /// <summary>
+    /// Finds an element using the configured selectors, trying each in order until one is found
+    /// </summary>
+    Task<ILocator?> FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default);
+    
+    /// <summary>
+    /// Gets a locator for the first selector (assumes element exists)
+    /// </summary>
+    ILocator GetLocator(IPage page, SelectorConfig config);
+}
+

Services/Scraping/IScrapingStep.cs

@@ -0,0 +1,28 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Interface for a single scraping step
+/// </summary>
+public interface IScrapingStep
+{
+    /// <summary>
+    /// Unique name of the step
+    /// </summary>
+    string Name { get; }
+    
+    /// <summary>
+    /// Names of steps that must complete before this step can execute
+    /// </summary>
+    string[] Prerequisites { get; }
+    
+    /// <summary>
+    /// Checks if the step can execute in the current page state
+    /// </summary>
+    bool CanExecute(ScrapingContext context);
+    
+    /// <summary>
+    /// Executes the step
+    /// </summary>
+    Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
+}
+

Services/Scraping/IScrapingStepRegistry.cs

@@ -0,0 +1,12 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Registry for managing scraping steps
+/// </summary>
+public interface IScrapingStepRegistry
+{
+    void RegisterStep(IScrapingStep step);
+    IScrapingStep? GetStep(string name);
+    IEnumerable<IScrapingStep> GetAllSteps();
+}
+

Services/Scraping/IWorkflow.cs

@@ -0,0 +1,14 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Interface for a scraping workflow
+/// </summary>
+/// <typeparam name="TResponse">The response type returned by this workflow</typeparam>
+public interface IWorkflow<TResponse>
+{
+    string Name { get; }
+    string Description { get; }
+    string[] StepNames { get; } // Fixed order - cannot be changed
+    Task<TResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
+}
+

Services/Scraping/IWorkflowFactory.cs

@@ -0,0 +1,10 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Factory for creating workflows
+/// </summary>
+public interface IWorkflowFactory
+{
+    IWorkflow<TResponse> GetWorkflow<TResponse>(string name);
+}
+

Services/Scraping/PageState.cs

@@ -0,0 +1,18 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Represents the current state of the page during scraping
+/// </summary>
+public enum PageState
+{
+    Initial,
+    HomepageLoaded,
+    SearchModalOpen,
+    SearchResultsVisible,
+    ForecastPageLoaded,
+    RadarPageLoaded,
+    MapReady,
+    SlideshowPaused,
+    Frame0Selected
+}
+

Services/Scraping/ScrapingContext.cs

@@ -0,0 +1,42 @@
+using BomLocalService.Models;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Context shared between scraping steps
+/// </summary>
+public class ScrapingContext
+{
+    public IPage Page { get; set; } = null!;
+    public string Suburb { get; set; } = string.Empty;
+    public string State { get; set; } = string.Empty;
+    public string CacheFolderPath { get; set; } = string.Empty;
+    public string DebugFolder { get; set; } = string.Empty;
+    public List<(string type, string text, DateTime timestamp)> ConsoleMessages { get; set; } = new();
+    public List<(string method, string url, int? status, string resourceType, DateTime timestamp)> NetworkRequests { get; set; } = new();
+    
+    // Page state tracking
+    public PageState CurrentState { get; set; } = PageState.Initial;
+    public HashSet<string> CompletedSteps { get; set; } = new();
+    
+    // Shared state between steps
+    public ILocator? SearchButton { get; set; }
+    public ILocator? SearchInput { get; set; }
+    public List<(string name, string desc, string fullText)>? SearchResults { get; set; }
+    public int? SelectedResultIndex { get; set; }
+    public ILocator? RadarLink { get; set; }
+    public ILocator? MapContainer { get; set; }
+    public Clip? MapBoundingBox { get; set; }
+    public LastUpdatedInfo? LastUpdatedInfo { get; set; }
+    public List<(int index, int minutesAgo)>? FrameInfo { get; set; }
+    public List<RadarFrame> Frames { get; set; } = new();
+    
+    // State validation helpers
+    public bool IsHomepageLoaded => CurrentState >= PageState.HomepageLoaded;
+    public bool IsSearchModalOpen => CurrentState >= PageState.SearchModalOpen;
+    public bool IsForecastPageLoaded => CurrentState >= PageState.ForecastPageLoaded;
+    public bool IsRadarPageLoaded => CurrentState >= PageState.RadarPageLoaded;
+    public bool IsMapReady => CurrentState >= PageState.MapReady;
+}
+

Services/Scraping/ScrapingStepRegistry.cs

@@ -0,0 +1,33 @@
+namespace BomLocalService.Services.Scraping;
+
+public class ScrapingStepRegistry : IScrapingStepRegistry
+{
+    private readonly Dictionary<string, IScrapingStep> _steps = new();
+    private readonly ILogger<ScrapingStepRegistry> _logger;
+    
+    public ScrapingStepRegistry(ILogger<ScrapingStepRegistry> logger)
+    {
+        _logger = logger;
+    }
+    
+    public void RegisterStep(IScrapingStep step)
+    {
+        if (_steps.ContainsKey(step.Name))
+        {
+            _logger.LogWarning("Step {Name} is already registered, overwriting", step.Name);
+        }
+        _steps[step.Name] = step;
+        _logger.LogDebug("Registered step: {Name}", step.Name);
+    }
+    
+    public IScrapingStep? GetStep(string name)
+    {
+        return _steps.TryGetValue(name, out var step) ? step : null;
+    }
+    
+    public IEnumerable<IScrapingStep> GetAllSteps()
+    {
+        return _steps.Values;
+    }
+}
+

Services/Scraping/ScrapingStepResult.cs

@@ -0,0 +1,15 @@
+namespace BomLocalService.Services.Scraping;
+
+/// <summary>
+/// Result of executing a scraping step
+/// </summary>
+public class ScrapingStepResult
+{
+    public bool Success { get; set; }
+    public string? ErrorMessage { get; set; }
+    public Dictionary<string, object>? Data { get; set; }
+    
+    public static ScrapingStepResult Successful() => new() { Success = true };
+    public static ScrapingStepResult Failed(string errorMessage) => new() { Success = false, ErrorMessage = errorMessage };
+}
+

Services/Scraping/Steps/BaseScrapingStep.cs

@@ -0,0 +1,64 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps;
+
+/// <summary>
+/// Base class for scraping steps with common functionality
+/// </summary>
+public abstract class BaseScrapingStep : IScrapingStep
+{
+    protected readonly ILogger Logger;
+    protected readonly ISelectorService SelectorService;
+    protected readonly IDebugService DebugService;
+    protected readonly IConfiguration Configuration;
+    protected readonly ScrapingSelectorsConfig Selectors;
+    protected readonly JavaScriptTemplatesConfig JavaScriptTemplates;
+    protected readonly TextPatternsConfig TextPatterns;
+    
+    protected BaseScrapingStep(
+        ILogger logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+    {
+        Logger = logger;
+        SelectorService = selectorService;
+        DebugService = debugService;
+        Configuration = configuration;
+        
+        Selectors = configuration.GetSection("Scraping:Selectors").Get<ScrapingSelectorsConfig>() ?? new();
+        JavaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get<JavaScriptTemplatesConfig>() ?? new();
+        TextPatterns = configuration.GetSection("Scraping:TextPatterns").Get<TextPatternsConfig>() ?? new();
+    }
+    
+    public abstract string Name { get; }
+    public abstract string[] Prerequisites { get; }
+    public abstract bool CanExecute(ScrapingContext context);
+    public abstract Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
+    
+    protected async Task SaveDebugAsync(ScrapingContext context, int stepNumber, string stepName, CancellationToken cancellationToken)
+    {
+        await DebugService.SaveStepDebugAsync(
+            context.DebugFolder, 
+            stepNumber, 
+            stepName, 
+            context.Page, 
+            context.ConsoleMessages, 
+            context.NetworkRequests, 
+            cancellationToken);
+    }
+    
+    protected async Task SaveErrorDebugAsync(ScrapingContext context, string errorMessage, CancellationToken cancellationToken)
+    {
+        await DebugService.SaveErrorDebugAsync(
+            context.DebugFolder, 
+            errorMessage, 
+            context.Page, 
+            context.ConsoleMessages, 
+            context.NetworkRequests, 
+            cancellationToken);
+    }
+}
+

Services/Scraping/Steps/Capture/CaptureFramesStep.cs

@@ -0,0 +1,489 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using BomLocalService.Utilities;
+using Microsoft.Playwright;
+using System.Text.RegularExpressions;
+
+namespace BomLocalService.Services.Scraping.Steps.Capture;
+
+public class CaptureFramesStep : BaseScrapingStep
+{
+    private readonly ICacheService _cacheService;
+    private readonly int _tileRenderWaitMs;
+    private readonly ScreenshotCropConfig _cropConfig;
+    
+    public override string Name => "CaptureFrames";
+    public override string[] Prerequisites => new[] { "ExtractMetadata", "CalculateMapBounds" };
+    
+    public CaptureFramesStep(
+        ILogger<CaptureFramesStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration,
+        ICacheService cacheService)
+        : base(logger, selectorService, debugService, configuration)
+    {
+        _cacheService = cacheService;
+        _tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
+        
+        var cropSection = configuration.GetSection("Screenshot:Crop");
+        _cropConfig = new ScreenshotCropConfig
+        {
+            X = cropSection.GetValue<int>("X", 0),
+            Y = cropSection.GetValue<int>("Y", 0),
+            RightOffset = cropSection.GetValue<int>("RightOffset", 0),
+            Height = cropSection.GetValue<int?>("Height")
+        };
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.MapBoundingBox != null && context.MapContainer != null;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Directory.CreateDirectory(context.CacheFolderPath);
+            Logger.LogInformation("Step {Step}: Using cache folder: {Path}", Name, context.CacheFolderPath);
+            
+            var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
+            
+            _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, 0, frameCount);
+            
+            var frames = new List<RadarFrame>();
+            var stepForwardButton = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton);
+            int? previousMinutesAgo = null;
+            
+            for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
+            {
+                Logger.LogInformation("Step {Step}: Capturing frame {FrameIndex} of {FrameCount}", Name, frameIndex, frameCount);
+                
+                await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs);
+                
+                // Small wait to ensure display label is stable after frame change/selection
+                // This is especially important for frame 0 which was just selected in ResetToFirstFrame
+                await context.Page.WaitForTimeoutAsync(300);
+                
+                // Try extracting with a retry in case the label is still updating
+                var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
+                if (minutesAgo == null)
+                {
+                    // Retry once after a short wait in case label was updating
+                    await context.Page.WaitForTimeoutAsync(200);
+                    minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
+                }
+                if (minutesAgo == null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
+                {
+                    var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
+                    minutesAgo = defaultMinutesAgo;
+                    Logger.LogWarning("Step {Step}: Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", Name, frameIndex, minutesAgo);
+                }
+                
+                if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value)
+                {
+                    Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", Name, frameIndex, minutesAgo);
+                    await WaitForDisplayLabelToChangeAsync(context.Page, previousMinutesAgo.Value);
+                    minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
+                    if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value)
+                    {
+                        if (context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
+                        {
+                            var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
+                            minutesAgo = defaultMinutesAgo;
+                            Logger.LogWarning("Step {Step}: Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", Name, frameIndex, minutesAgo);
+                        }
+                    }
+                }
+                
+                var radarFolder = FilePathHelper.GetDataTypeFolderPath(context.CacheFolderPath, CachedDataType.Radar);
+                if (!Directory.Exists(radarFolder))
+                {
+                    Directory.CreateDirectory(radarFolder);
+                }
+                
+                var framePath = FilePathHelper.GetFrameFilePath(context.CacheFolderPath, CachedDataType.Radar, frameIndex);
+                await CaptureMapScreenshotAsync(context.Page, context.MapContainer!, framePath, context.MapBoundingBox!);
+                
+                frames.Add(new RadarFrame
+                {
+                    FrameIndex = frameIndex,
+                    ImagePath = framePath,
+                    MinutesAgo = minutesAgo ?? 0
+                });
+                
+                previousMinutesAgo = minutesAgo;
+                
+                Logger.LogInformation("Step {Step}: Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)", 
+                    Name, frameIndex, framePath, minutesAgo ?? 0);
+                
+                _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount);
+                
+                await SaveDebugAsync(context, 15 + frameIndex, $"frame_{frameIndex}_captured", cancellationToken);
+                
+                if (frameIndex < frameCount - 1)
+                {
+                    await DismissModalOverlaysAsync(context.Page);
+                    
+                    var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
+                    
+                    await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
+                    
+                    if (currentMinutesAgo.HasValue)
+                    {
+                        await WaitForDisplayLabelToChangeAsync(context.Page, currentMinutesAgo.Value);
+                    }
+                    else
+                    {
+                        await context.Page.WaitForTimeoutAsync(500);
+                    }
+                }
+            }
+            
+            Logger.LogInformation("Step {Step}: All {FrameCount} frames captured successfully", Name, frameCount);
+            
+            _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Saving);
+            
+            if (context.LastUpdatedInfo != null)
+            {
+                await _cacheService.SaveMetadataAsync(context.CacheFolderPath, context.LastUpdatedInfo, cancellationToken);
+            }
+            await _cacheService.SaveFramesMetadataAsync(context.CacheFolderPath, CachedDataType.Radar, frames, cancellationToken);
+            
+            context.Frames = frames;
+            
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to capture frames: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to capture frames: {ex.Message}");
+        }
+    }
+    
+    private async Task<int?> ExtractMinutesAgoFromDisplayAsync(IPage page)
+    {
+        try
+        {
+            var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel);
+            var timeLabel = await timeLabelLocator.TextContentAsync();
+            if (string.IsNullOrEmpty(timeLabel))
+            {
+                Logger.LogDebug("Time display label is empty");
+                return null;
+            }
+            
+            var trimmedLabel = timeLabel.Trim();
+            Logger.LogInformation("Extracting minutes from display label: '{Label}'", trimmedLabel);
+            
+            // Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
+            Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern);
+            var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase);
+            if (timestampMatch.Success)
+            {
+                var timestampStr = timestampMatch.Groups[0].Value;
+                Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}'", timestampStr, trimmedLabel);
+                if (TryParseTimestamp(timestampStr, out var timestamp))
+                {
+                    var minutesAgo = (int)(DateTime.UtcNow - timestamp).TotalMinutes;
+                    Logger.LogInformation("Parsed timestamp: {Timestamp} UTC, calculated minutes ago: {Minutes}", timestamp, minutesAgo);
+                    if (minutesAgo >= 0 && minutesAgo <= 120) // Reasonable range: 0-2 hours
+                    {
+                        Logger.LogInformation("Successfully calculated minutes ago from timestamp: {Minutes}", minutesAgo);
+                        return minutesAgo;
+                    }
+                    else
+                    {
+                        Logger.LogWarning("Calculated minutes ago ({Minutes}) outside reasonable range (0-120)", minutesAgo);
+                    }
+                }
+                else
+                {
+                    Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
+                }
+            }
+            else
+            {
+                Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern);
+            }
+            
+            return null;
+        }
+        catch (Exception ex)
+        {
+            Logger.LogDebug(ex, "Failed to extract minutes from display label");
+            return null;
+        }
+    }
+    
+    private bool TryParseTimestamp(string timestampStr, out DateTime timestamp)
+    {
+        timestamp = DateTime.MinValue;
+        
+        try
+        {
+            // Try common Australian date formats
+            // Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
+            var formats = new[]
+            {
+                "dddd d MMM, h:mm tt",      // Wednesday 17 Dec, 11:05 pm
+                "d MMM, h:mm tt",           // 17 Dec, 11:05 pm
+                "dddd d MMM, hh:mm tt",     // Wednesday 17 Dec, 11:05 pm (with leading zero)
+                "d MMM, hh:mm tt",          // 17 Dec, 11:05 pm (with leading zero)
+                "dddd dd MMM, h:mm tt",     // Wednesday 17 Dec, 11:05 pm (with leading zero day)
+                "dd MMM, h:mm tt"           // 17 Dec, 11:05 pm (with leading zero day)
+            };
+            
+            var culture = new System.Globalization.CultureInfo("en-AU");
+            
+            foreach (var format in formats)
+            {
+                if (DateTime.TryParseExact(timestampStr, format, culture, 
+                    System.Globalization.DateTimeStyles.AssumeLocal, out timestamp))
+                {
+                    // If year is not specified, assume current year
+                    if (timestamp.Year == 1)
+                    {
+                        timestamp = new DateTime(DateTime.Now.Year, timestamp.Month, timestamp.Day, 
+                            timestamp.Hour, timestamp.Minute, timestamp.Second);
+                    }
+                    
+                    // If the parsed time is in the future (likely same day next year), adjust
+                    if (timestamp > DateTime.Now && timestamp < DateTime.Now.AddDays(1))
+                    {
+                        // Already correct
+                    }
+                    else if (timestamp > DateTime.Now)
+                    {
+                        // Likely parsed as next year, adjust to this year
+                        timestamp = timestamp.AddYears(-1);
+                    }
+                    
+                    return true;
+                }
+            }
+            
+            return false;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+    
+    private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000)
+    {
+        try
+        {
+            var startTime = DateTime.UtcNow;
+            while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
+            {
+                var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
+                if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo)
+                {
+                    return;
+                }
+                await page.WaitForTimeoutAsync(200);
+            }
+            Logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs);
+        }
+        catch (Exception ex)
+        {
+            Logger.LogDebug(ex, "Error waiting for display label to change");
+        }
+    }
+    
+    private async Task DismissModalOverlaysAsync(IPage page)
+    {
+        try
+        {
+            var hasModal = await page.EvaluateAsync<bool>(JavaScriptTemplates.CheckModalOverlay);
+            
+            if (!hasModal)
+            {
+                return;
+            }
+            
+            Logger.LogDebug("Modal overlay detected, dismissing");
+            
+            await page.Keyboard.PressAsync("Escape");
+            await page.WaitForTimeoutAsync(200);
+            
+            var stillVisible = await page.EvaluateAsync<bool>(JavaScriptTemplates.CheckModalStillVisible);
+            
+            if (stillVisible)
+            {
+                try
+                {
+                    var mapContainer = SelectorService.GetLocator(page, Selectors.MapContainer);
+                    await mapContainer.ClickAsync(new LocatorClickOptions { Force = true });
+                    await page.WaitForTimeoutAsync(100);
+                }
+                catch
+                {
+                    // Ignore if click fails
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            Logger.LogDebug(ex, "Error dismissing modal overlay, continuing");
+        }
+    }
+    
+    private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip)
+    {
+        if (containerClip == null || containerClip.Width <= 0 || containerClip.Height <= 0)
+        {
+            Logger.LogError("Invalid container bounds: X={X}, Y={Y}, Width={Width}, Height={Height}", 
+                containerClip?.X ?? 0, containerClip?.Y ?? 0, containerClip?.Width ?? 0, containerClip?.Height ?? 0);
+            throw new Exception($"Invalid container bounds: {containerClip?.Width ?? 0}x{containerClip?.Height ?? 0}");
+        }
+        
+        Clip cropArea;
+        try
+        {
+            cropArea = CalculateCropArea(containerClip);
+        }
+        catch (Exception ex)
+        {
+            Logger.LogWarning(ex, "Failed to calculate crop area, using full container bounds");
+            cropArea = containerClip;
+        }
+        
+        var viewportSize = page.ViewportSize;
+        int? viewportWidth = viewportSize?.Width;
+        int? viewportHeight = viewportSize?.Height;
+        
+        if (viewportWidth == null || viewportHeight == null)
+        {
+            try
+            {
+                var viewportJson = await page.EvaluateAsync<string>(JavaScriptTemplates.GetViewportSize);
+                if (!string.IsNullOrEmpty(viewportJson))
+                {
+                    using var doc = System.Text.Json.JsonDocument.Parse(viewportJson);
+                    var root = doc.RootElement;
+                    if (root.TryGetProperty("width", out var widthProp) && root.TryGetProperty("height", out var heightProp))
+                    {
+                        if (widthProp.TryGetInt32(out var width) && heightProp.TryGetInt32(out var height))
+                        {
+                            viewportWidth = width;
+                            viewportHeight = height;
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.LogWarning(ex, "Failed to get viewport size from page evaluation");
+            }
+            
+            if (viewportWidth == null || viewportHeight == null)
+            {
+                viewportWidth = (int)containerClip.Width;
+                viewportHeight = (int)containerClip.Height;
+            }
+        }
+        
+        if (viewportWidth.HasValue && viewportHeight.HasValue)
+        {
+            if (cropArea.X < 0)
+            {
+                cropArea = new Clip { X = 0, Y = cropArea.Y, Width = cropArea.Width + cropArea.X, Height = cropArea.Height };
+            }
+            if (cropArea.Y < 0)
+            {
+                cropArea = new Clip { X = cropArea.X, Y = 0, Width = cropArea.Width, Height = cropArea.Height + cropArea.Y };
+            }
+            
+            if (cropArea.X + cropArea.Width > viewportWidth.Value)
+            {
+                var newWidth = viewportWidth.Value - cropArea.X;
+                cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = newWidth, Height = cropArea.Height };
+            }
+            if (cropArea.Y + cropArea.Height > viewportHeight.Value)
+            {
+                var newHeight = viewportHeight.Value - cropArea.Y;
+                cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = cropArea.Width, Height = newHeight };
+            }
+        }
+        
+        if (cropArea.Width <= 0 || cropArea.Height <= 0)
+        {
+            Logger.LogError("Invalid crop dimensions after validation: {Width}x{Height}, using full container", cropArea.Width, cropArea.Height);
+            cropArea = containerClip;
+        }
+        
+        if (cropArea.Width <= 0 || cropArea.Height <= 0)
+        {
+            throw new Exception($"Cannot create valid crop area. Container: {containerClip.Width}x{containerClip.Height}, Viewport: {viewportWidth ?? 0}x{viewportHeight ?? 0}");
+        }
+        
+        try
+        {
+            await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 5000 });
+        }
+        catch
+        {
+            // Continue if network idle timeout
+        }
+        
+        await DismissModalOverlaysAsync(page);
+        
+        await page.ScreenshotAsync(new PageScreenshotOptions
+        {
+            Path = outputPath,
+            Clip = cropArea,
+            Type = ScreenshotType.Png,
+            Animations = ScreenshotAnimations.Disabled
+        });
+        
+        Logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})", 
+            outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height);
+    }
+    
+    private Clip CalculateCropArea(Clip containerClip)
+    {
+        var x = containerClip.X + _cropConfig.X;
+        var y = containerClip.Y + _cropConfig.Y;
+        var width = Math.Max(0, containerClip.Width - _cropConfig.X - _cropConfig.RightOffset);
+        var height = _cropConfig.Height ?? Math.Max(0, containerClip.Height - _cropConfig.Y);
+        
+        if (x < containerClip.X || y < containerClip.Y)
+        {
+            x = containerClip.X;
+            y = containerClip.Y;
+        }
+        
+        var maxWidth = containerClip.Width - (x - containerClip.X);
+        var maxHeight = containerClip.Height - (y - containerClip.Y);
+        
+        if (width > maxWidth)
+        {
+            width = maxWidth;
+        }
+        
+        if (height > maxHeight)
+        {
+            height = maxHeight;
+        }
+        
+        if (width <= 0 || height <= 0)
+        {
+            throw new Exception($"Invalid crop dimensions: {width}x{height}");
+        }
+        
+        return new Clip
+        {
+            X = x,
+            Y = y,
+            Width = width,
+            Height = height
+        };
+    }
+}
+

Services/Scraping/Steps/Map/CalculateMapBoundsStep.cs

@@ -0,0 +1,69 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Map;
+
+public class CalculateMapBoundsStep : BaseScrapingStep
+{
+    public override string Name => "CalculateMapBounds";
+    public override string[] Prerequisites => new[] { "ResetToFirstFrame" };
+    
+    public CalculateMapBoundsStep(
+        ILogger<CalculateMapBoundsStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsMapReady;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Preparing map container for screenshot", Name);
+            
+            var mapContainer = SelectorService.GetLocator(context.Page, Selectors.MapContainer);
+            await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
+            
+            await context.Page.WaitForFunctionAsync(
+                JavaScriptTemplates.WaitForMapContainer, 
+                new PageWaitForFunctionOptions { Timeout = 10000 });
+            
+            var boundingBox = await mapContainer.BoundingBoxAsync();
+            if (boundingBox == null || boundingBox.Width <= 0 || boundingBox.Height <= 0)
+            {
+                var errorMsg = $"Map container has invalid bounds: {boundingBox?.Width ?? 0}x{boundingBox?.Height ?? 0}";
+                Logger.LogError("Step {Step}: {Error}", Name, errorMsg);
+                await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
+                return ScrapingStepResult.Failed(errorMsg);
+            }
+            
+            var containerClip = new Clip
+            {
+                X = boundingBox.X,
+                Y = boundingBox.Y,
+                Width = boundingBox.Width,
+                Height = boundingBox.Height
+            };
+            
+            context.MapContainer = mapContainer;
+            context.MapBoundingBox = containerClip;
+            
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to calculate map bounds: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to calculate map bounds: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Map/PauseRadarStep.cs

@@ -0,0 +1,66 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Map;
+
+public class PauseRadarStep : BaseScrapingStep
+{
+    public override string Name => "PauseRadar";
+    public override string[] Prerequisites => new[] { "WaitForMapReady" };
+    
+    public PauseRadarStep(
+        ILogger<PauseRadarStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsMapReady;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Checking if radar loop is paused", Name);
+            
+            var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton);
+            await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
+            
+            var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
+            if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true)
+            {
+                Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name);
+                await playPauseButton.ClickAsync();
+                await context.Page.WaitForTimeoutAsync(300);
+                
+                buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
+                if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true)
+                {
+                    Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name);
+                }
+            }
+            else
+            {
+                Logger.LogInformation("Step {Step}: Radar is already paused", Name);
+            }
+            
+            await SaveDebugAsync(context, 8, "radar_paused", cancellationToken);
+            
+            context.CurrentState = PageState.SlideshowPaused;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to pause radar: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to pause radar: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs

@@ -0,0 +1,103 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Map;
+
+public class ResetToFirstFrameStep : BaseScrapingStep
+{
+    public override string Name => "ResetToFirstFrame";
+    public override string[] Prerequisites => new[] { "PauseRadar" };
+    
+    public ResetToFirstFrameStep(
+        ILogger<ResetToFirstFrameStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.CurrentState >= PageState.SlideshowPaused;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name);
+            
+            try
+            {
+                // Try JavaScript click first to avoid thumb interception
+                var clicked = await context.Page.EvaluateAsync<bool>(@"() => {
+                    const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]');
+                    if (segment) {
+                        segment.click();
+                        return true;
+                    }
+                    return false;
+                }");
+                
+                if (clicked)
+                {
+                    await context.Page.WaitForTimeoutAsync(500);
+                    Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name);
+                }
+                else
+                {
+                    // Fallback to locator click with force
+                    var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
+                    await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
+                    await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true });
+                    await context.Page.WaitForTimeoutAsync(500);
+                    Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name);
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name);
+            }
+            
+            await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken);
+            
+            // Verify scrubber is at position 0
+            Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name);
+            try
+            {
+                var activeSegment = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment);
+                
+                if (activeSegment)
+                {
+                    Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name);
+                }
+                else
+                {
+                    Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name);
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name);
+            }
+            
+            await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken);
+            
+            // Wait for frame 0 tiles to fully render
+            var tileRenderWaitMs = Configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
+            await context.Page.WaitForTimeoutAsync(tileRenderWaitMs);
+            
+            context.CurrentState = PageState.Frame0Selected;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to reset to first frame: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to reset to first frame: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Map/WaitForMapReadyStep.cs

@@ -0,0 +1,73 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Map;
+
+public class WaitForMapReadyStep : BaseScrapingStep
+{
+    private readonly int _tileRenderWaitMs;
+    
+    public override string Name => "WaitForMapReady";
+    public override string[] Prerequisites => new[] { "ClickRadarLink" };
+    
+    public WaitForMapReadyStep(
+        ILogger<WaitForMapReadyStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+        _tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsRadarPageLoaded;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Waiting for weather map page to load", Name);
+            
+            await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
+            
+            Logger.LogInformation("Step {Step}: Waiting for map canvas element to render", Name);
+            var mapCanvas = SelectorService.GetLocator(context.Page, Selectors.MapCanvas);
+            await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 });
+            
+            await context.Page.WaitForFunctionAsync(
+                JavaScriptTemplates.WaitForMapCanvas, 
+                new PageWaitForFunctionOptions { Timeout = 15000 });
+            
+            Logger.LogInformation("Step {Step}: Map canvas is ready - waiting for map to render", Name);
+            
+            try
+            {
+                await context.Page.WaitForFunctionAsync(
+                    JavaScriptTemplates.WaitForEsriView, 
+                    new PageWaitForFunctionOptions { Timeout = 30000 });
+                Logger.LogInformation("Step {Step}: Esri map view is ready", Name);
+            }
+            catch
+            {
+                Logger.LogInformation("Step {Step}: Esri view ready check timed out, continuing with fixed wait", Name);
+            }
+            
+            await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs);
+            await SaveDebugAsync(context, 7, "weather_map_ready", cancellationToken);
+            
+            context.CurrentState = PageState.MapReady;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to wait for map ready: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to wait for map ready: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs

@@ -0,0 +1,68 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+
+namespace BomLocalService.Services.Scraping.Steps.Metadata;
+
+public class ExtractMetadataStep : BaseScrapingStep
+{
+    private readonly ITimeParsingService _timeParsingService;
+    
+    public override string Name => "ExtractMetadata";
+    public override string[] Prerequisites => new[] { "ResetToFirstFrame" };
+    
+    public ExtractMetadataStep(
+        ILogger<ExtractMetadataStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration,
+        ITimeParsingService timeParsingService)
+        : base(logger, selectorService, debugService, configuration)
+    {
+        _timeParsingService = timeParsingService;
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsMapReady;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name);
+            
+            var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page);
+            context.LastUpdatedInfo = lastUpdatedInfo;
+            
+            // Extract frame info
+            try
+            {
+                var frameInfo = await context.Page.EvaluateAsync<object[]>(JavaScriptTemplates.ExtractFrameInfo);
+                var result = new List<(int index, int minutesAgo)>();
+                for (int i = 0; i < 7; i++)
+                {
+                    var minutesAgo = 40 - (i * 5);
+                    result.Add((i, minutesAgo));
+                }
+                context.FrameInfo = result;
+            }
+            catch (Exception ex)
+            {
+                Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name);
+                context.FrameInfo = Enumerable.Range(0, 7)
+                    .Select(i => (i, 40 - (i * 5)))
+                    .ToList();
+            }
+            
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to extract metadata: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to extract metadata: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Navigation/ClickRadarLinkStep.cs

@@ -0,0 +1,56 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Navigation;
+
+public class ClickRadarLinkStep : BaseScrapingStep
+{
+    public override string Name => "ClickRadarLink";
+    public override string[] Prerequisites => new[] { "SelectSearchResult" };
+    
+    public ClickRadarLinkStep(
+        ILogger<ClickRadarLinkStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsForecastPageLoaded;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Looking for 'Rain radar and weather map' link", Name);
+            
+            var radarLink = await SelectorService.FindElementAsync(context.Page, Selectors.RadarLink, cancellationToken);
+            
+            if (radarLink == null)
+            {
+                var errorMsg = Selectors.RadarLink.ErrorMessage ?? $"Could not find 'Rain radar and weather map' link for {context.Suburb}, {context.State}";
+                await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
+                return ScrapingStepResult.Failed(errorMsg);
+            }
+            
+            context.RadarLink = radarLink;
+            await radarLink.ClickAsync();
+            await SaveDebugAsync(context, 6, "radar_link_clicked", cancellationToken);
+            
+            context.CurrentState = PageState.RadarPageLoaded;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to click radar link: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to click radar link: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Navigation/ClickSearchButtonStep.cs

@@ -0,0 +1,65 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Navigation;
+
+public class ClickSearchButtonStep : BaseScrapingStep
+{
+    public override string Name => "ClickSearchButton";
+    public override string[] Prerequisites => new[] { "NavigateHomepage" };
+    
+    public ClickSearchButtonStep(
+        ILogger<ClickSearchButtonStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsHomepageLoaded;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Clicking 'Search for a location' button", Name);
+            
+            var searchButton = await SelectorService.FindElementAsync(context.Page, Selectors.SearchButton, cancellationToken);
+            
+            if (searchButton == null)
+            {
+                var errorMsg = Selectors.SearchButton.ErrorMessage ?? "Could not find 'Search for a location' button on BOM homepage.";
+                await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
+                return ScrapingStepResult.Failed(errorMsg);
+            }
+            
+            context.SearchButton = searchButton;
+            await searchButton.ClickAsync();
+            
+            // Wait for search input to appear
+            var searchInputReady = SelectorService.GetLocator(context.Page, Selectors.SearchInput);
+            await searchInputReady.WaitForAsync(new LocatorWaitForOptions 
+            { 
+                Timeout = Selectors.SearchInput.TimeoutMs, 
+                State = WaitForSelectorState.Visible 
+            });
+            
+            await SaveDebugAsync(context, 2, "search_button_clicked", cancellationToken);
+            
+            context.CurrentState = PageState.SearchModalOpen;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to click search button: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to click search button: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Navigation/NavigateHomepageStep.cs

@@ -0,0 +1,62 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Navigation;
+
+public class NavigateHomepageStep : BaseScrapingStep
+{
+    private readonly string _baseUrl;
+    
+    public override string Name => "NavigateHomepage";
+    public override string[] Prerequisites => Array.Empty<string>();
+    
+    public NavigateHomepageStep(
+        ILogger<NavigateHomepageStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+        _baseUrl = configuration.GetValue<string>("Scraping:BaseUrl") ?? "https://www.bom.gov.au/";
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.CurrentState == PageState.Initial;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Navigating to BOM homepage", Name);
+            
+            await context.Page.GotoAsync(_baseUrl, new PageGotoOptions
+            {
+                WaitUntil = WaitUntilState.DOMContentLoaded,
+                Timeout = 30000
+            });
+            
+            // Wait for search button to be ready
+            var searchButtonReady = SelectorService.GetLocator(context.Page, Selectors.SearchButton);
+            await searchButtonReady.WaitForAsync(new LocatorWaitForOptions 
+            { 
+                Timeout = Selectors.SearchButton.TimeoutMs, 
+                State = WaitForSelectorState.Visible 
+            });
+            
+            await SaveDebugAsync(context, 1, "homepage_loaded", cancellationToken);
+            
+            context.CurrentState = PageState.HomepageLoaded;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to navigate to homepage: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to navigate to homepage: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Search/FillSearchInputStep.cs

@@ -0,0 +1,48 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Search;
+
+public class FillSearchInputStep : BaseScrapingStep
+{
+    public override string Name => "FillSearchInput";
+    public override string[] Prerequisites => new[] { "ClickSearchButton" };
+    
+    public FillSearchInputStep(
+        ILogger<FillSearchInputStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsSearchModalOpen;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Searching for suburb: {Suburb}", Name, context.Suburb);
+            
+            var searchInput = SelectorService.GetLocator(context.Page, Selectors.SearchInput);
+            await searchInput.FillAsync(context.Suburb);
+            
+            context.SearchInput = searchInput;
+            await SaveDebugAsync(context, 3, "search_input_filled", cancellationToken);
+            
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to fill search input: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to fill search input: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Search/SelectSearchResultStep.cs

@@ -0,0 +1,192 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using BomLocalService.Utilities;
+using Microsoft.Playwright;
+using System.Text.RegularExpressions;
+
+namespace BomLocalService.Services.Scraping.Steps.Search;
+
+public class SelectSearchResultStep : BaseScrapingStep
+{
+    public override string Name => "SelectSearchResult";
+    public override string[] Prerequisites => new[] { "WaitForSearchResults" };
+    
+    public SelectSearchResultStep(
+        ILogger<SelectSearchResultStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.CurrentState >= PageState.SearchResultsVisible;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Looking for matching search result for {Suburb}, {State}", Name, context.Suburb, context.State);
+            
+            var suburbLower = context.Suburb.ToLower().Trim();
+            var stateLower = context.State.ToLower().Trim();
+            
+            // Get the actual count from the summary element
+            var resultsTitle = SelectorService.GetLocator(context.Page, Selectors.ResultsTitle);
+            var summaryText = await resultsTitle.TextContentAsync();
+            int? actualCount = null;
+            
+            if (!string.IsNullOrEmpty(summaryText))
+            {
+                var countMatch = Regex.Match(summaryText, TextPatterns.ResultsCountPattern, RegexOptions.IgnoreCase);
+                if (countMatch.Success && countMatch.Groups.Count >= 3 && countMatch.Groups[2].Success)
+                {
+                    if (int.TryParse(countMatch.Groups[2].Value, out var total))
+                    {
+                        actualCount = total;
+                    }
+                }
+            }
+            
+            // Fetch all result data
+            List<(string name, string desc, string fullText)> results = new();
+            try
+            {
+                var resultData = await context.Page.EvaluateAsync<string[][]>(JavaScriptTemplates.ExtractSearchResults);
+                
+                results = resultData.Select(arr => (
+                    name: arr.Length > 0 ? arr[0] : "",
+                    desc: arr.Length > 1 ? arr[1] : "",
+                    fullText: arr.Length > 2 ? arr[2] : ""
+                )).ToList();
+            }
+            catch (Exception ex)
+            {
+                Logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content");
+                var resultTexts = await context.Page.EvaluateAsync<string[]>(JavaScriptTemplates.ExtractSearchResultsFallback);
+                results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList();
+            }
+            
+            if (actualCount.HasValue)
+            {
+                Logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim());
+                if (results.Count > actualCount.Value)
+                {
+                    results = results.Take(actualCount.Value).ToList();
+                }
+            }
+            else
+            {
+                Logger.LogInformation("Found {Count} search results", results.Count);
+            }
+            
+            int? matchingIndex = null;
+            int bestMatchScore = -1;
+            
+            for (int i = 0; i < results.Count; i++)
+            {
+                var (name, desc, fullText) = results[i];
+                var nameLower = name.ToLower().Trim();
+                var descLower = desc.ToLower().Trim();
+                var fullTextLower = fullText.ToLower();
+                
+                Logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'", 
+                    i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText);
+                
+                var matchesSuburb = false;
+                var matchScore = 0;
+                
+                if (!string.IsNullOrEmpty(name))
+                {
+                    if (nameLower == suburbLower)
+                    {
+                        matchesSuburb = true;
+                        matchScore = 100;
+                    }
+                    else if (nameLower.StartsWith(suburbLower + " ") || nameLower.StartsWith(suburbLower + "("))
+                    {
+                        matchesSuburb = true;
+                        matchScore = 80;
+                    }
+                    else if (nameLower.Contains("(" + suburbLower + ")") || nameLower.Contains("(" + suburbLower + " "))
+                    {
+                        matchesSuburb = true;
+                        matchScore = 60;
+                    }
+                    else if (nameLower.Contains(suburbLower))
+                    {
+                        matchesSuburb = true;
+                        matchScore = 40;
+                    }
+                }
+                else
+                {
+                    if (fullTextLower.Contains(suburbLower))
+                    {
+                        matchesSuburb = true;
+                        matchScore = 20;
+                    }
+                }
+                
+                var matchesState = false;
+                if (!string.IsNullOrEmpty(desc))
+                {
+                    matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower);
+                }
+                if (!matchesState)
+                {
+                    matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower);
+                }
+                
+                Logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (score={Score}), matchesState={MatchesState}", 
+                    i, matchesSuburb, matchScore, matchesState);
+                
+                if (matchesSuburb && matchesState && matchScore > bestMatchScore)
+                {
+                    matchingIndex = i;
+                    bestMatchScore = matchScore;
+                    Logger.LogInformation("New best match found: {Name} - {Desc} (score: {Score})", name, desc, matchScore);
+                }
+            }
+            
+            if (matchingIndex.HasValue)
+            {
+                Logger.LogInformation("Using best matching result at index {Index} with score {Score}", matchingIndex.Value, bestMatchScore);
+            }
+            else
+            {
+                Logger.LogInformation("No exact match found, using first result");
+            }
+            
+            context.SearchResults = results;
+            context.SelectedResultIndex = matchingIndex;
+            
+            // Click the matching result
+            var resultsList = context.Page.Locator("ul[aria-labelledby='location-results-title']");
+            var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']");
+            var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First;
+            
+            await resultToClick.ClickAsync();
+            await SaveDebugAsync(context, 4, "search_result_selected", cancellationToken);
+            
+            // Wait for forecast page to load
+            await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
+            var dynamicContentWaitMs = Configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
+            await context.Page.WaitForTimeoutAsync(dynamicContentWaitMs);
+            await SaveDebugAsync(context, 5, "forecast_page_loaded", cancellationToken);
+            
+            context.CurrentState = PageState.ForecastPageLoaded;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to select search result: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to select search result: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/Steps/Search/WaitForSearchResultsStep.cs

@@ -0,0 +1,47 @@
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services.Scraping.Steps.Search;
+
+public class WaitForSearchResultsStep : BaseScrapingStep
+{
+    public override string Name => "WaitForSearchResults";
+    public override string[] Prerequisites => new[] { "FillSearchInput" };
+    
+    public WaitForSearchResultsStep(
+        ILogger<WaitForSearchResultsStep> logger,
+        ISelectorService selectorService,
+        IDebugService debugService,
+        IConfiguration configuration)
+        : base(logger, selectorService, debugService, configuration)
+    {
+    }
+    
+    public override bool CanExecute(ScrapingContext context)
+    {
+        return context.IsSearchModalOpen;
+    }
+    
+    public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        try
+        {
+            Logger.LogInformation("Step {Step}: Waiting for autocomplete suggestions", Name);
+            
+            await context.Page.WaitForFunctionAsync(
+                JavaScriptTemplates.WaitForSearchResults, 
+                new PageWaitForFunctionOptions { Timeout = 10000 });
+            
+            context.CurrentState = PageState.SearchResultsVisible;
+            return ScrapingStepResult.Successful();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Step {Step} failed", Name);
+            await SaveErrorDebugAsync(context, $"Failed to wait for search results: {ex.Message}", cancellationToken);
+            return ScrapingStepResult.Failed($"Failed to wait for search results: {ex.Message}");
+        }
+    }
+}
+

Services/Scraping/WorkflowFactory.cs

@@ -0,0 +1,22 @@
+namespace BomLocalService.Services.Scraping;
+
+public class WorkflowFactory : IWorkflowFactory
+{
+    private readonly IServiceProvider _serviceProvider;
+    
+    public WorkflowFactory(IServiceProvider serviceProvider)
+    {
+        _serviceProvider = serviceProvider;
+    }
+    
+    public IWorkflow<TResponse> GetWorkflow<TResponse>(string name)
+    {
+        return name switch
+        {
+            "RadarScraping" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.RadarScrapingWorkflow>(),
+            "TemperatureMap" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.TemperatureMapWorkflow>(),
+            _ => throw new ArgumentException($"Unknown workflow: {name}")
+        };
+    }
+}
+

Services/Scraping/Workflows/RadarScrapingWorkflow.cs

@@ -0,0 +1,210 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+using BomLocalService.Utilities;
+
+namespace BomLocalService.Services.Scraping.Workflows;
+
+public class RadarScrapingWorkflow : IWorkflow<RadarResponse>
+{
+    private readonly ILogger<RadarScrapingWorkflow> _logger;
+    private readonly IScrapingStepRegistry _stepRegistry;
+    private readonly IConfiguration _configuration;
+    private readonly ICacheService _cacheService;
+    private readonly double _cacheExpirationMinutes;
+    private readonly int _cacheManagementCheckIntervalMinutes;
+    
+    public string Name => "RadarScraping";
+    public string Description => "Scrapes radar images for a location";
+    
+    // Fixed step sequence - order matters due to dependencies
+    public string[] StepNames => new[]
+    {
+        "NavigateHomepage",
+        "ClickSearchButton",
+        "FillSearchInput",
+        "WaitForSearchResults",
+        "SelectSearchResult",
+        "ClickRadarLink",
+        "WaitForMapReady",
+        "PauseRadar",
+        "ResetToFirstFrame",
+        "ExtractMetadata",
+        "CalculateMapBounds",
+        "CaptureFrames"
+    };
+    
+    public RadarScrapingWorkflow(
+        ILogger<RadarScrapingWorkflow> logger,
+        IScrapingStepRegistry stepRegistry,
+        IConfiguration configuration,
+        ICacheService cacheService)
+    {
+        _logger = logger;
+        _stepRegistry = stepRegistry;
+        _configuration = configuration;
+        _cacheService = cacheService;
+        _cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 12.5);
+        _cacheManagementCheckIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
+    }
+    
+    public async Task<RadarResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        var workflowConfig = _configuration.GetSection($"Scraping:Workflows:{Name}").Get<ScrapingWorkflowConfig>();
+        var workflowStartTime = DateTime.UtcNow;
+        var stepTimings = new List<(string stepName, double durationSeconds)>();
+        
+        // Record Initializing phase start for metrics tracking
+        if (!string.IsNullOrEmpty(context.CacheFolderPath))
+        {
+            _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Initializing);
+        }
+        
+        foreach (var stepName in StepNames)
+        {
+            var stepConfig = workflowConfig?.Steps?.GetValueOrDefault(stepName) ?? new ScrapingStepConfig { Enabled = true };
+            
+            if (!stepConfig.Enabled)
+            {
+                _logger.LogInformation("Step {Step} is disabled, skipping", stepName);
+                continue;
+            }
+            
+            var step = _stepRegistry.GetStep(stepName);
+            if (step == null)
+            {
+                throw new InvalidOperationException($"Step {stepName} not found in registry");
+            }
+            
+            if (!ValidatePrerequisites(step, context))
+            {
+                throw new InvalidOperationException(
+                    $"Step {stepName} prerequisites not met. Required: {string.Join(", ", step.Prerequisites)}");
+            }
+            
+            if (!step.CanExecute(context))
+            {
+                throw new InvalidOperationException(
+                    $"Step {stepName} cannot execute in current page state: {context.CurrentState}");
+            }
+            
+            var stepStartTime = DateTime.UtcNow;
+            _logger.LogInformation("Executing step {Step}", stepName);
+            
+            var result = await step.ExecuteAsync(context, cancellationToken);
+            
+            var stepDuration = (DateTime.UtcNow - stepStartTime).TotalSeconds;
+            stepTimings.Add((stepName, stepDuration));
+            
+            // Record step timing in metrics
+            _cacheService.RecordStepCompletion(stepName, stepDuration);
+            
+            // Get historical average for comparison
+            var avgDuration = _cacheService.GetAverageStepDuration(stepName);
+            if (avgDuration > 0)
+            {
+                var diff = stepDuration - avgDuration;
+                var diffPercent = (diff / avgDuration) * 100;
+                var isSlow = stepDuration > avgDuration * 1.5; // 50% slower than average
+                
+                if (isSlow)
+                {
+                    _logger.LogWarning("Step {Step} took significantly longer than average: {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower)",
+                        stepName, stepDuration, avgDuration, diff, diffPercent);
+                }
+                else if (Math.Abs(diffPercent) > 20) // More than 20% difference (faster or slower)
+                {
+                    _logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##;-#.##}s, {DiffPercent:+#0.#;-#0.#}%)",
+                        stepName, stepDuration, avgDuration, diff, diffPercent);
+                }
+                else
+                {
+                    _logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s)", stepName, stepDuration, avgDuration);
+                }
+            }
+            else
+            {
+                _logger.LogInformation("Step {Step} completed in {Duration:F2}s", stepName, stepDuration);
+            }
+            
+            if (!result.Success)
+            {
+                throw new Exception($"Step {stepName} failed: {result.ErrorMessage}");
+            }
+            
+            context.CompletedSteps.Add(stepName);
+        }
+        
+        var totalDuration = (DateTime.UtcNow - workflowStartTime).TotalSeconds;
+        
+        // Check if workflow duration is significantly longer than average
+        var avgTotalDuration = _cacheService.GetAverageTotalDuration();
+        var isWorkflowSlow = avgTotalDuration > 0 && totalDuration > avgTotalDuration * 1.3; // 30% slower than average
+        
+        // Log step breakdown
+        var stepBreakdown = string.Join(", ", stepTimings.Select(t => $"{t.stepName}={t.durationSeconds:F2}s"));
+        
+        if (isWorkflowSlow)
+        {
+            var workflowDiff = totalDuration - avgTotalDuration;
+            var workflowDiffPercent = (workflowDiff / avgTotalDuration) * 100;
+            _logger.LogWarning("Workflow {Workflow} took significantly longer than average: {TotalDuration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower) for {Suburb}, {State}",
+                Name, totalDuration, avgTotalDuration, workflowDiff, workflowDiffPercent, context.Suburb, context.State);
+        }
+        else
+        {
+            _logger.LogInformation("Workflow {Workflow} completed in {TotalDuration:F2}s for {Suburb}, {State}. Step breakdown: {StepBreakdown}",
+                Name, totalDuration, context.Suburb, context.State, stepBreakdown);
+        }
+        
+        // Log step performance summary if we have historical data
+        var stepMetrics = _cacheService.GetStepMetrics();
+        if (stepMetrics.Count > 0)
+        {
+            var slowSteps = stepTimings
+                .Where(t => stepMetrics.ContainsKey(t.stepName) && t.durationSeconds > stepMetrics[t.stepName] * 1.5)
+                .Select(t => $"{t.stepName} ({t.durationSeconds:F2}s vs avg {stepMetrics[t.stepName]:F2}s)")
+                .ToList();
+            
+            if (slowSteps.Any())
+            {
+                _logger.LogWarning("Workflow {Workflow} had slower-than-average steps: {SlowSteps}",
+                    Name, string.Join(", ", slowSteps));
+            }
+        }
+        
+        return BuildResponse(context);
+    }
+    
+    private bool ValidatePrerequisites(IScrapingStep step, ScrapingContext context)
+    {
+        return step.Prerequisites.All(prereq => context.CompletedSteps.Contains(prereq));
+    }
+    
+    private RadarResponse BuildResponse(ScrapingContext context)
+    {
+        if (context.LastUpdatedInfo == null)
+        {
+            throw new InvalidOperationException("LastUpdatedInfo is required to build response");
+        }
+        
+        if (context.Frames == null || context.Frames.Count == 0)
+        {
+            throw new InvalidOperationException("Frames are required to build response");
+        }
+        
+        var cacheExpiresAt = context.LastUpdatedInfo.ObservationTime.AddMinutes(_cacheExpirationMinutes);
+        
+        return ResponseBuilder.CreateRadarResponse(
+            context.CacheFolderPath,
+            context.Frames,
+            _cacheManagementCheckIntervalMinutes,
+            context.LastUpdatedInfo,
+            context.Suburb,
+            context.State,
+            cacheIsValid: true,
+            cacheExpiresAt: cacheExpiresAt,
+            isUpdating: false);
+    }
+}
+

Services/Scraping/Workflows/TemperatureMapWorkflow.cs

@@ -0,0 +1,41 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Services.Scraping;
+
+namespace BomLocalService.Services.Scraping.Workflows;
+
+/// <summary>
+/// Future workflow for temperature map scraping (not yet implemented)
+/// </summary>
+public class TemperatureMapWorkflow : IWorkflow<RadarResponse>
+{
+    private readonly ILogger<TemperatureMapWorkflow> _logger;
+    
+    public string Name => "TemperatureMap";
+    public string Description => "Scrapes temperature forecast map (future feature)";
+    
+    public string[] StepNames => new[]
+    {
+        "NavigateHomepage",
+        "ClickSearchButton",
+        "FillSearchInput",
+        "WaitForSearchResults",
+        "SelectSearchResult",
+        // Future: "ClickTemperatureMapLink",
+        // Future: "WaitForMapReady",
+        // Future: "ExtractMetadata",
+        // Future: "CalculateMapBounds",
+        // Future: "CaptureTemperatureFrames"
+    };
+    
+    public TemperatureMapWorkflow(ILogger<TemperatureMapWorkflow> logger)
+    {
+        _logger = logger;
+    }
+    
+    public Task<RadarResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException("TemperatureMap workflow is not yet implemented");
+    }
+}
+

Services/ScrapingService.cs

@@ -1,6 +1,6 @@
 using BomLocalService.Models;
 using BomLocalService.Services.Interfaces;
-using BomLocalService.Utilities;
+using BomLocalService.Services.Scraping;
 using Microsoft.Playwright;
 
 namespace BomLocalService.Services;
@@ -8,67 +8,16 @@ namespace BomLocalService.Services;
 public class ScrapingService : IScrapingService
 {
     private readonly ILogger<ScrapingService> _logger;
-    private readonly IBrowserService _browserService;
-    private readonly ITimeParsingService _timeParsingService;
-    private readonly ICacheService _cacheService;
-    private readonly IDebugService _debugService;
-    private readonly int _dynamicContentWaitMs;
-    private readonly int _tileRenderWaitMs;
-    private readonly ScreenshotCropConfig _cropConfig;
-
-    // Selector constants
-    private static readonly string[] SearchButtonSelectors = new[]
-    {
-        "button[data-testid='searchLabel']",
-        "button[aria-label='Search for a location']",
-        "button.search-location__trigger-button",
-        "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
-        "button:has-text('Search for a location')",
-        "[data-testid='showSearchModal'] button"
-    };
-
-    private static readonly string[] RadarLinkSelectors = new[]
-    {
-        "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
-        ".cta.button a.jump-link:has-text('Rain radar and weather map')",
-        "a.jump-link:has-text('Rain radar and weather map')",
-        ".cta.button a:has-text('Rain radar and weather map')",
-        "a.bom-button--secondary:has-text('Rain radar and weather map')",
-        "a:has-text('Rain radar and weather map')",
-        "a:has-text('rain radar')",
-        "a[href*='radar']"
-    };
-
+    private readonly IWorkflowFactory _workflowFactory;
+    
     public ScrapingService(
         ILogger<ScrapingService> logger,
-        IBrowserService browserService,
-        ITimeParsingService timeParsingService,
-        ICacheService cacheService,
-        IDebugService debugService,
-        IConfiguration configuration)
+        IWorkflowFactory workflowFactory)
     {
         _logger = logger;
-        _browserService = browserService;
-        _timeParsingService = timeParsingService;
-        _cacheService = cacheService;
-        _debugService = debugService;
-        _dynamicContentWaitMs = configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
-        _tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
-        
-        // Load crop configuration
-        var cropSection = configuration.GetSection("Screenshot:Crop");
-        _cropConfig = new ScreenshotCropConfig
-        {
-            X = cropSection.GetValue<int>("X", 0),
-            Y = cropSection.GetValue<int>("Y", 0),
-            RightOffset = cropSection.GetValue<int>("RightOffset", 0),
-            Height = cropSection.GetValue<int?>("Height")
-        };
-        
-        _logger.LogInformation("Screenshot crop config: X={X}, Y={Y}, RightOffset={RightOffset}, Height={Height}",
-            _cropConfig.X, _cropConfig.Y, _cropConfig.RightOffset, _cropConfig.Height);
+        _workflowFactory = workflowFactory;
     }
-
+    
     /// <summary>
     /// Scrapes the BOM website to get a radar screenshot for a location
     /// </summary>
@@ -82,812 +31,22 @@ public class ScrapingService : IScrapingService
         List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
         CancellationToken cancellationToken = default)
     {
-        try
+        var context = new ScrapingContext
         {
-            // Step 1: Navigate to BOM homepage and wait for search button
-            _logger.LogInformation("Navigating to BOM homepage");
-            await page.GotoAsync("https://www.bom.gov.au/", new PageGotoOptions
-            {
-                WaitUntil = WaitUntilState.DOMContentLoaded,
-                Timeout = 30000
-            });
-            
-            // Wait for search button to be ready instead of NetworkIdle (faster)
-            var searchButtonReady = page.Locator("button[data-testid='searchLabel'], button[aria-label='Search for a location'], button.search-location__trigger-button").First;
-            await searchButtonReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000, State = WaitForSelectorState.Visible });
-            await _debugService.SaveStepDebugAsync(debugFolder, 1, "homepage_loaded", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 2: Click "Search for a location" button to open search UI
-            _logger.LogInformation("Clicking 'Search for a location' button");
-            var searchButton = await SelectorHelper.FindLocatorBySelectorsAsync(page, SearchButtonSelectors, _logger);
-            
-            if (searchButton == null)
-            {
-                var errorMsg = "Could not find 'Search for a location' button on BOM homepage.";
-                await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
-                throw new Exception(errorMsg);
-            }
-
-            await searchButton.ClickAsync();
-            
-            // Wait for search input to appear
-            var searchInputReady = page.Locator("#search-enter-keyword").First;
-            await searchInputReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000, State = WaitForSelectorState.Visible });
-            await _debugService.SaveStepDebugAsync(debugFolder, 2, "search_button_clicked", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 3: Find and fill search box with suburb name
-            _logger.LogInformation("Searching for suburb: {Suburb}", suburb);
-            var searchInput = page.Locator("#search-enter-keyword").First;
-            await searchInput.FillAsync(suburb);
-            
-            // Step 4: Wait for autocomplete suggestions to appear
-            _logger.LogInformation("Waiting for autocomplete suggestions");
-            await page.WaitForFunctionAsync(@"() => {
-                const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
-                return results.length > 0 && results.some(r => r.offsetParent !== null);
-            }", new PageWaitForFunctionOptions { Timeout = 10000 });
-            await _debugService.SaveStepDebugAsync(debugFolder, 3, "search_input_filled", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 5: Find the matching search result based on suburb and state
-            _logger.LogInformation("Looking for matching search result for {Suburb}, {State}", suburb, state);
-            var suburbLower = suburb.ToLower().Trim();
-            var stateLower = state.ToLower().Trim();
-            
-            // Get the actual count from the summary element
-            var summaryText = await page.Locator("#location-results-title, [data-testid='location-results-title']").First.TextContentAsync();
-            int? actualCount = null;
-            if (!string.IsNullOrEmpty(summaryText))
-            {
-                // Parse "3 of 3 location results" or similar
-                // Pattern has 2 capture groups: (\d+) of (\d+)
-                // Groups[0] = full match, Groups[1] = first number, Groups[2] = second number (total)
-                var countMatch = System.Text.RegularExpressions.Regex.Match(summaryText, @"(\d+)\s+of\s+(\d+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
-                if (countMatch.Success && countMatch.Groups.Count >= 3 && countMatch.Groups[2].Success)
-                {
-                    if (int.TryParse(countMatch.Groups[2].Value, out var total))
-                    {
-                        actualCount = total;
-                    }
-                }
-            }
-            
-            // Fetch all result data in one JavaScript evaluation - extract location name and state separately
-            List<(string name, string desc, string fullText)> results = new();
-            try
-            {
-                var resultData = await page.EvaluateAsync<string[][]>(@"() => {
-                    // Scope to the location results list - find the ul with aria-labelledby pointing to location-results-title
-                    const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
-                    if (!resultsList) {
-                        console.log('Location results list not found');
-                        return [];
-                    }
-                    const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
-                    console.log('Found', results.length, 'location results');
-                    return results.map((r, index) => {
-                        // Query from the li element - elements are nested inside <a> tag
-                        const nameEl = r.querySelector('[data-testid=""location-name""]');
-                        const descEl = r.querySelector('.bom-linklist-item__desc');
-                        const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
-                        const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
-                        const fullText = (r.textContent || r.innerText || '').trim();
-                        console.log('Result', index, ':', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
-                        return [name, desc, fullText];
-                    });
-                }");
-                
-                // Convert to structured data
-                results = resultData.Select(arr => (
-                    name: arr.Length > 0 ? arr[0] : "",
-                    desc: arr.Length > 1 ? arr[1] : "",
-                    fullText: arr.Length > 2 ? arr[2] : ""
-                )).ToList();
-            }
-            catch (Exception ex)
-            {
-                _logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content");
-                // Fallback to simple text extraction
-                var resultTexts = await page.EvaluateAsync<string[]>(@"() => { 
-                    const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); 
-                    return results.map(r => r.textContent || ''); 
-                }");
-                results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList();
-            }
-            
-            if (actualCount.HasValue)
-            {
-                _logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim());
-                // Only use the first N results that match the actual count
-                if (results.Count > actualCount.Value)
-                {
-                    results = results.Take(actualCount.Value).ToList();
-                }
-            }
-            else
-            {
-                _logger.LogInformation("Found {Count} search results", results.Count);
-            }
-            
-            int? matchingIndex = null;
-            for (int i = 0; i < results.Count; i++)
-            {
-                var (name, desc, fullText) = results[i];
-                var nameLower = name.ToLower().Trim();
-                var descLower = desc.ToLower().Trim();
-                var fullTextLower = fullText.ToLower();
-                
-                _logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'", 
-                    i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText);
-                
-                // Check if suburb name matches (from location-name element or fallback to fullText)
-                var matchesSuburb = false;
-                if (!string.IsNullOrEmpty(name))
-                {
-                    matchesSuburb = nameLower == suburbLower || nameLower.Contains(suburbLower);
-                }
-                else
-                {
-                    // Fallback to fullText if name extraction failed
-                    matchesSuburb = fullTextLower.Contains(suburbLower);
-                }
-                
-                // Check if state matches (from description which contains "Queensland 4300" or "New South Wales 2469")
-                var matchesState = false;
-                if (!string.IsNullOrEmpty(desc))
-                {
-                    matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower);
-                }
-                // Always check fullText for state as fallback
-                if (!matchesState)
-                {
-                    matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower);
-                }
-                
-                _logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (suburb='{Suburb}' vs name='{Name}'), matchesState={MatchesState} (state='{State}' vs desc='{Desc}')", 
-                    i, matchesSuburb, suburbLower, nameLower, matchesState, stateLower, descLower);
-                
-                if (matchesSuburb && matchesState)
-                {
-                    matchingIndex = i;
-                    _logger.LogInformation("Found matching result: {Name} - {Desc}", name, desc);
-                    break;
-                }
-            }
-            
-            // Click the matching result (or first if no match)
-            // Scope to the location results list
-            var resultsList = page.Locator("ul[aria-labelledby='location-results-title']");
-            var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']");
-            var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First;
-            
-            if (!matchingIndex.HasValue)
-            {
-                _logger.LogInformation("No exact match found, using first result");
-            }
-            
-            await resultToClick.ClickAsync();
-            await _debugService.SaveStepDebugAsync(debugFolder, 4, "search_result_selected", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Wait for forecast page to load
-            await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
-            await page.WaitForTimeoutAsync(_dynamicContentWaitMs); // Brief wait for dynamic content
-            await _debugService.SaveStepDebugAsync(debugFolder, 5, "forecast_page_loaded", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 6: Find and click "Rain radar and weather map" link
-            _logger.LogInformation("Looking for 'Rain radar and weather map' link");
-            var radarLink = await SelectorHelper.FindLocatorBySelectorsAsync(page, RadarLinkSelectors, _logger);
-            
-            if (radarLink == null)
-            {
-                var errorMsg = $"Could not find 'Rain radar and weather map' link for {suburb}, {state}";
-                await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
-                throw new Exception(errorMsg);
-            }
-
-            await radarLink.ClickAsync();
-            await _debugService.SaveStepDebugAsync(debugFolder, 6, "radar_link_clicked", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 7: Wait for weather map page to load and map to fully render
-            _logger.LogInformation("Waiting for weather map page to load");
-            await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
-            
-            // Wait for the map canvas element to appear with proper dimensions
-            _logger.LogInformation("Waiting for map canvas element to render");
-            var mapCanvas = page.Locator(".esri-view-surface canvas").First;
-            await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 });
-            
-            // Wait for canvas to have valid dimensions
-            await page.WaitForFunctionAsync(@"() => {
-                const canvas = document.querySelector('.esri-view-surface canvas');
-                return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
-            }", new PageWaitForFunctionOptions { Timeout = 15000 });
-            
-            _logger.LogInformation("Map canvas is ready - waiting for map to render");
-            
-            // Wait for Esri map view to be ready
-            try
-            {
-                await page.WaitForFunctionAsync(@"() => {
-                    try {
-                        const elements = document.querySelectorAll('.esri-view');
-                        for (let el of elements) {
-                            if (el.__view && el.__view.ready) {
-                                return true;
-                            }
-                        }
-                    } catch(e) {}
-                    return false;
-                }", new PageWaitForFunctionOptions { Timeout = 30000 });
-                _logger.LogInformation("Esri map view is ready");
-            }
-            catch
-            {
-                _logger.LogInformation("Esri view ready check timed out, continuing with fixed wait");
-            }
-            
-            // Additional wait for tiles to render
-            await page.WaitForTimeoutAsync(_tileRenderWaitMs);
-            
-            await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 8: Ensure radar is paused before capturing frames
-            _logger.LogInformation("Checking if radar loop is paused");
-            var playPauseButton = page.Locator("button[data-testid='bom-time-scrub-play-pause']").First;
-            await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
-
-            // Check if button shows "Play" (paused) or "Pause" (playing)
-            var buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync();
-            if (buttonLabel?.Trim().Equals("Pause", StringComparison.OrdinalIgnoreCase) == true)
-            {
-                _logger.LogInformation("Radar is playing, pausing it");
-                await playPauseButton.ClickAsync();
-                // Wait for pause to take effect
-                await page.WaitForTimeoutAsync(500);
-                
-                // Verify it's now paused
-                buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync();
-                if (buttonLabel?.Trim().Equals("Play", StringComparison.OrdinalIgnoreCase) != true)
-                {
-                    _logger.LogWarning("Radar may not be paused after click, continuing anyway");
-                }
-            }
-            else
-            {
-                _logger.LogInformation("Radar is already paused");
-            }
-
-            await _debugService.SaveStepDebugAsync(debugFolder, 8, "radar_paused", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 9: Click on first frame segment to ensure we start at frame 0
-            _logger.LogInformation("Resetting to first frame (frame 0)");
-            try
-            {
-                var firstFrameSegment = page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
-                await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
-                await firstFrameSegment.ClickAsync();
-                // Wait for frame to update
-                await page.WaitForTimeoutAsync(1000);
-                _logger.LogInformation("Successfully clicked frame 0 segment");
-            }
-            catch (Exception ex)
-            {
-                _logger.LogWarning(ex, "Failed to click first frame segment, continuing anyway");
-            }
-
-            await _debugService.SaveStepDebugAsync(debugFolder, 9, "frame_0_selected", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 10: Verify scrubber is at position 0 before capturing first frame
-            _logger.LogInformation("Verifying scrubber is at position 0");
-            try
-            {
-                // Check if the active segment has data-id="0" (simpler and more reliable than checking thumb position)
-                var activeSegment = await page.EvaluateAsync<bool>(@"() => {
-                    const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
-                    const activeSegment = segments.find(s => {
-                        const style = window.getComputedStyle(s);
-                        return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
-                    });
-                    return activeSegment && activeSegment.getAttribute('data-id') === '0';
-                }");
-                
-                if (activeSegment)
-                {
-                    _logger.LogInformation("Scrubber confirmed at position 0 (active segment is frame 0)");
-                }
-                else
-                {
-                    _logger.LogDebug("Could not confirm scrubber position via active segment, but continuing (frame 0 was clicked)");
-                }
-            }
-            catch (Exception ex)
-            {
-                _logger.LogDebug(ex, "Scrubber position verification failed, continuing anyway (non-critical check)");
-            }
-
-            await _debugService.SaveStepDebugAsync(debugFolder, 10, "scrubber_at_position_0", page, consoleMessages, networkRequests, cancellationToken);
-
-            // Step 11: Wait for frame 0 tiles to fully load before calculating bounding box
-            // This ensures the map viewport is stable and prevents jiggle between frames
-            _logger.LogInformation("Waiting for frame 0 tiles to fully render");
-            await page.WaitForTimeoutAsync(_tileRenderWaitMs);
-
-            // Step 12: Extract metadata and frame information
-            _logger.LogInformation("Extracting metadata and frame information");
-            var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page);
-            var frameInfo = await ExtractFrameInfoAsync(page);
-
-            // Step 13: Get map container and calculate bounding box once
-            // Calculate after frame 0 tiles are loaded to ensure consistent viewport
-            _logger.LogInformation("Preparing map container for screenshot");
-            var mapContainer = page.Locator(".esri-view-surface").First;
-            await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
-
-            // Ensure the map container is visible and has dimensions
-            await page.WaitForFunctionAsync(@"() => {
-                const container = document.querySelector('.esri-view-surface');
-                return container && container.offsetWidth > 0 && container.offsetHeight > 0;
-            }", new PageWaitForFunctionOptions { Timeout = 10000 });
-
-            var boundingBox = await mapContainer.BoundingBoxAsync();
-            if (boundingBox == null)
-            {
-                throw new Exception("Could not determine map container bounds");
-            }
-            
-            // Convert BoundingBox to Clip for crop calculation
-            var containerClip = new Clip
-            {
-                X = boundingBox.X,
-                Y = boundingBox.Y,
-                Width = boundingBox.Width,
-                Height = boundingBox.Height
-            };
-
-            // Step 14: Use provided cache folder (already created by BomRadarService)
-            Directory.CreateDirectory(cacheFolderPath);
-            _logger.LogInformation("Using cache folder: {Path}", cacheFolderPath);
-
-            // Step 15-21: Capture all 7 frames
-            var frames = new List<RadarFrame>();
-            var stepForwardButton = page.Locator("button[data-testid='bom-scrub-utils__right__step-forward']").First;
-            int? previousMinutesAgo = null;
-
-            for (int frameIndex = 0; frameIndex < 7; frameIndex++)
-            {
-                _logger.LogInformation("Capturing frame {FrameIndex} of 7", frameIndex);
-                
-                // Wait for map to stabilize (tiles to load for current frame)
-                await page.WaitForTimeoutAsync(_tileRenderWaitMs);
-                
-                // Extract actual minutes ago from the display label (e.g., "17 minutes ago" -> 17)
-                var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
-                if (minutesAgo == null)
-                {
-                    // Fallback to calculated value if extraction fails
-                    var (_, defaultMinutesAgo) = frameInfo[frameIndex];
-                    minutesAgo = defaultMinutesAgo;
-                    _logger.LogWarning("Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", frameIndex, minutesAgo);
-                }
-                
-                // Validate that minutesAgo is different from previous frame (unless it's frame 0)
-                if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value)
-                {
-                    _logger.LogWarning("Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", frameIndex, minutesAgo);
-                    // Wait for display label to change
-                    await WaitForDisplayLabelToChangeAsync(page, previousMinutesAgo.Value);
-                    // Re-extract after waiting
-                    minutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
-                    if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value)
-                    {
-                        // Still same or failed - use calculated fallback
-                        var (_, defaultMinutesAgo) = frameInfo[frameIndex];
-                        minutesAgo = defaultMinutesAgo;
-                        _logger.LogWarning("Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", frameIndex, minutesAgo);
-                    }
-                }
-                
-                // Take screenshot with crop configuration
-                // Ensure radar subfolder exists
-                var radarFolder = FilePathHelper.GetDataTypeFolderPath(cacheFolderPath, CachedDataType.Radar);
-                if (!Directory.Exists(radarFolder))
-                {
-                    Directory.CreateDirectory(radarFolder);
-                }
-                
-                var framePath = FilePathHelper.GetFrameFilePath(cacheFolderPath, CachedDataType.Radar, frameIndex);
-                await CaptureMapScreenshotAsync(page, mapContainer, framePath, containerClip);
-                
-                frames.Add(new RadarFrame
-                {
-                    FrameIndex = frameIndex,
-                    ImagePath = framePath,
-                    MinutesAgo = minutesAgo.Value
-                });
-                
-                previousMinutesAgo = minutesAgo;
-                
-                _logger.LogInformation("Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)", 
-                    frameIndex, framePath, minutesAgo.Value);
-                
-                // Save debug screenshot BEFORE clicking step forward
-                await _debugService.SaveStepDebugAsync(debugFolder, 15 + frameIndex, $"frame_{frameIndex}_captured", page, consoleMessages, networkRequests, cancellationToken);
-                
-                // If not the last frame, click step forward to prepare for next frame
-                if (frameIndex < 6)
-                {
-                    // Dismiss any modal overlays (BOM, reCAPTCHA, feedback forms) before clicking
-                    await DismissModalOverlaysAsync(page);
-                    
-                    // Get current minutesAgo before clicking (to detect change)
-                    var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
-                    
-                    // Use force click to bypass any remaining overlays
-                    await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
-                    
-                    // Wait for display label to actually change (not just a fixed timeout)
-                    if (currentMinutesAgo.HasValue)
-                    {
-                        await WaitForDisplayLabelToChangeAsync(page, currentMinutesAgo.Value);
-                    }
-                    else
-                    {
-                        // Fallback to fixed wait if we can't detect change
-                        await page.WaitForTimeoutAsync(1000);
-                    }
-                }
-            }
-
-            _logger.LogInformation("All 7 frames captured successfully");
-
-            // Step 22: Save metadata and frame information
-            await _cacheService.SaveMetadataAsync(cacheFolderPath, lastUpdatedInfo, cancellationToken);
-            await _cacheService.SaveFramesMetadataAsync(cacheFolderPath, CachedDataType.Radar, frames, cancellationToken);
-
-            // Step 23: Return response with all frames
-            // Note: ScrapingService doesn't have access to cache management check interval
-            // Default to 5 minutes (standard check interval)
-            return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, lastUpdatedInfo, suburb, state, cacheIsValid: true, cacheExpiresAt: null, isUpdating: false, cacheManagementCheckIntervalMinutes: 5);
-        }
-        catch (Exception ex)
-        {
-            // Save error debug info if debug is enabled
-            await _debugService.SaveErrorDebugAsync(debugFolder, $"Exception: {ex.Message}\n\nStackTrace:\n{ex.StackTrace}", page, consoleMessages, networkRequests, cancellationToken);
-            _logger.LogError(ex, "Error during radar screenshot capture");
-            throw;
-        }
-    }
-
-    /// <summary>
-    /// Calculates the crop area for screenshot based on configuration
-    /// </summary>
-    private Clip CalculateCropArea(Clip containerClip)
-    {
-        // Start with container's position plus offset
-        var x = containerClip.X + _cropConfig.X;
-        var y = containerClip.Y + _cropConfig.Y;
-        
-        // Calculate width: container width minus left offset (X) minus right offset
-        var width = Math.Max(0, containerClip.Width - _cropConfig.X - _cropConfig.RightOffset);
-        
-        // Calculate height (use configured or remaining height)
-        var height = _cropConfig.Height ?? Math.Max(0, containerClip.Height - _cropConfig.Y);
-        
-        // Validate bounds
-        if (x < containerClip.X || y < containerClip.Y)
-        {
-            _logger.LogWarning("Crop offset is outside container bounds, using container bounds");
-            x = containerClip.X;
-            y = containerClip.Y;
-        }
-        
-        var maxWidth = containerClip.Width - (x - containerClip.X);
-        var maxHeight = containerClip.Height - (y - containerClip.Y);
-        
-        if (width > maxWidth)
-        {
-            _logger.LogWarning("Crop width exceeds container bounds, adjusting from {Requested} to {Max}", width, maxWidth);
-            width = maxWidth;
-        }
-        
-        if (height > maxHeight)
-        {
-            _logger.LogWarning("Crop height exceeds container bounds, adjusting from {Requested} to {Max}", height, maxHeight);
-            height = maxHeight;
-        }
-        
-        if (width <= 0 || height <= 0)
-        {
-            throw new Exception($"Invalid crop dimensions: {width}x{height}");
-        }
-        
-        _logger.LogDebug("Crop area calculated: X={X}, Y={Y}, Width={Width}, Height={Height} (container: {ContainerX}, {ContainerY}, {ContainerWidth}x{ContainerHeight})",
-            x, y, width, height, containerClip.X, containerClip.Y, containerClip.Width, containerClip.Height);
-        
-        return new Clip
-        {
-            X = x,
-            Y = y,
-            Width = width,
-            Height = height
+            Page = page,
+            Suburb = suburb,
+            State = state,
+            CacheFolderPath = cacheFolderPath,
+            DebugFolder = debugFolder,
+            ConsoleMessages = consoleMessages,
+            NetworkRequests = networkRequests
         };
-    }
-
-    /// <summary>
-    /// Extracts frame information from timeline segments
-    /// </summary>
-    private async Task<List<(int index, int minutesAgo)>> ExtractFrameInfoAsync(IPage page)
-    {
-        try
-        {
-            var frameInfo = await page.EvaluateAsync<object[]>(@"() => {
-                const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
-                return segments.map((seg, index) => {
-                    const ariaLabel = seg.getAttribute('aria-label') || '';
-                    // Extract minutes from '40 minutes ago', '35 minutes ago', etc.
-                    const minutesMatch = ariaLabel.match(/(\d+)\s+minutes?\s+ago/);
-                    const minutes = minutesMatch ? parseInt(minutesMatch[1]) : null;
-                    return { index: index, minutesAgo: minutes };
-                });
-            }");
-            
-            var result = new List<(int index, int minutesAgo)>();
-            for (int i = 0; i < 7; i++)
-            {
-                // Default values if extraction fails
-                var minutesAgo = 40 - (i * 5);
-                
-                // Try to use extracted values if available
-                if (frameInfo != null && i < frameInfo.Length)
-                {
-                    // The EvaluateAsync returns object[], we'd need to deserialize properly
-                    // For now, use defaults
-                }
-                
-                result.Add((i, minutesAgo));
-            }
-            
-            return result;
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Failed to extract frame info, using defaults");
-            // Return default frame info
-            return Enumerable.Range(0, 7)
-                .Select(i => (i, 40 - (i * 5)))
-                .ToList();
-        }
-    }
-
-    /// <summary>
-    /// Extracts the actual minutes ago value from the display label (e.g., "17 minutes ago" -> 17)
-    /// </summary>
-    private async Task<int?> ExtractMinutesAgoFromDisplayAsync(IPage page)
-    {
-        try
-        {
-            var timeLabel = await page.Locator(".bom-scrub-display-label").First.TextContentAsync();
-            if (string.IsNullOrEmpty(timeLabel))
-            {
-                return null;
-            }
-            
-            // Parse "17 minutes ago" or "41 minutes ago" etc.
-            var match = System.Text.RegularExpressions.Regex.Match(
-                timeLabel.Trim(), 
-                @"(\d+)\s+minutes?\s+ago", 
-                System.Text.RegularExpressions.RegexOptions.IgnoreCase
-            );
-            
-            if (match.Success && match.Groups.Count >= 2)
-            {
-                if (int.TryParse(match.Groups[1].Value, out var minutes))
-                {
-                    return minutes;
-                }
-            }
-            
-            return null;
-        }
-        catch (Exception ex)
-        {
-            _logger.LogDebug(ex, "Failed to extract minutes from display label");
-            return null;
-        }
-    }
-
-    /// <summary>
-    /// Waits for the display label to change from the current minutesAgo value
-    /// </summary>
-    private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000)
-    {
-        try
-        {
-            var startTime = DateTime.UtcNow;
-            while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
-            {
-                var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
-                if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo)
-                {
-                    // Display has changed
-                    return;
-                }
-                await page.WaitForTimeoutAsync(200); // Check every 200ms
-            }
-            _logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogDebug(ex, "Error waiting for display label to change");
-        }
-    }
-
-    /// <summary>
-    /// Dismisses any modal overlays (BOM modals, reCAPTCHA, feedback forms) that might block the radar
-    /// Uses a single efficient check and minimal delays
-    /// </summary>
-    private async Task DismissModalOverlaysAsync(IPage page)
-    {
-        try
-        {
-            // Single check for all modal types (BOM, reCAPTCHA, feedback forms)
-            var hasModal = await page.EvaluateAsync<bool>(@"() => {
-                // Check BOM modal overlay
-                const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
-                if (bomOverlay && bomOverlay.style.display !== 'none') {
-                    return true;
-                }
-                
-                // Check reCAPTCHA modals (large visible elements)
-                const recaptchaSelectors = [
-                    '.g-recaptcha',
-                    '#g-recaptcha',
-                    '.rc-anchor-container'
-                ];
-                for (const selector of recaptchaSelectors) {
-                    const el = document.querySelector(selector);
-                    if (el) {
-                        const style = window.getComputedStyle(el);
-                        const rect = el.getBoundingClientRect();
-                        if (style.display !== 'none' && style.visibility !== 'hidden' && 
-                            rect.width > 200 && rect.height > 200) {
-                            return true;
-                        }
-                    }
-                }
-                
-                // Check feedback forms with reCAPTCHA
-                const allForms = document.querySelectorAll('form');
-                for (const form of allForms) {
-                    const action = form.getAttribute('action') || '';
-                    const id = form.getAttribute('id') || '';
-                    if (action.includes('feedback') || id.includes('feedback')) {
-                        const style = window.getComputedStyle(form);
-                        const rect = form.getBoundingClientRect();
-                        if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
-                            const text = form.textContent || '';
-                            if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) {
-                                return true;
-                            }
-                        }
-                    }
-                }
-                
-                return false;
-            }");
-            
-            if (!hasModal)
-            {
-                return; // No modal, exit quickly
-            }
-            
-            _logger.LogDebug("Modal overlay detected, dismissing");
-            
-            // Single Escape key press (works for most modals)
-            await page.Keyboard.PressAsync("Escape");
-            await page.WaitForTimeoutAsync(300); // Minimal delay
-            
-            // If still visible, try clicking outside (on map)
-            var stillVisible = await page.EvaluateAsync<bool>(@"() => {
-                const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
-                if (bomOverlay && bomOverlay.style.display !== 'none') return true;
-                
-                // Check feedback forms
-                const allForms = document.querySelectorAll('form');
-                for (const form of allForms) {
-                    const action = form.getAttribute('action') || '';
-                    const id = form.getAttribute('id') || '';
-                    if (action.includes('feedback') || id.includes('feedback')) {
-                        const style = window.getComputedStyle(form);
-                        const rect = form.getBoundingClientRect();
-                        if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
-                            return true;
-                        }
-                    }
-                }
-                return false;
-            }");
-            
-            if (stillVisible)
-            {
-                // Click on map to dismiss (only if Escape didn't work)
-                try
-                {
-                    var mapContainer = page.Locator(".esri-view-surface").First;
-                    await mapContainer.ClickAsync(new LocatorClickOptions { Force = true });
-                    await page.WaitForTimeoutAsync(200); // Minimal delay
-                }
-                catch
-                {
-                    // Ignore if click fails
-                }
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogDebug(ex, "Error dismissing modal overlay, continuing");
-        }
-    }
-
-    /// <summary>
-    /// Captures map screenshot with crop configuration
-    /// </summary>
-    private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip)
-    {
-        Clip cropArea;
-        try
-        {
-            cropArea = CalculateCropArea(containerClip);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Failed to calculate crop area, using full container bounds. Container: X={X}, Y={Y}, Width={Width}, Height={Height}", 
-                containerClip.X, containerClip.Y, containerClip.Width, containerClip.Height);
-            // Fallback to full container if crop calculation fails
-            cropArea = containerClip;
-        }
         
-        // Validate crop area is within page bounds before attempting screenshot
-        var viewportSize = page.ViewportSize;
-        if (viewportSize == null || cropArea.X < 0 || cropArea.Y < 0 || 
-            cropArea.X + cropArea.Width > viewportSize.Width || 
-            cropArea.Y + cropArea.Height > viewportSize.Height)
-        {
-            _logger.LogWarning("Crop area is outside viewport bounds, using full container. Crop: X={X}, Y={Y}, Width={Width}, Height={Height}, Viewport: {ViewportWidth}x{ViewportHeight}", 
-                cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height, viewportSize?.Width ?? 0, viewportSize?.Height ?? 0);
-            cropArea = containerClip;
-        }
+        // Get the appropriate workflow (currently only RadarScraping is implemented)
+        var workflow = _workflowFactory.GetWorkflow<RadarResponse>("RadarScraping");
         
-        // Final validation - ensure dimensions are positive
-        if (cropArea.Width <= 0 || cropArea.Height <= 0)
-        {
-            _logger.LogError("Invalid crop dimensions: {Width}x{Height}, using full container", cropArea.Width, cropArea.Height);
-            cropArea = containerClip;
-        }
+        _logger.LogInformation("Executing {Workflow} workflow for {Suburb}, {State}", workflow.Name, suburb, state);
         
-        // Wait for fonts to be loaded to prevent text rendering artifacts
-        try
-        {
-            await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 5000 });
-        }
-        catch
-        {
-            // Continue if network idle timeout - fonts may already be loaded
-        }
-        
-        // Dismiss any modal overlays (BOM, reCAPTCHA, feedback forms) before taking screenshot
-        await DismissModalOverlaysAsync(page);
-        
-        // Take high-quality screenshot with explicit PNG format and disabled animations
-        await page.ScreenshotAsync(new PageScreenshotOptions
-        {
-            Path = outputPath,
-            Clip = cropArea,
-            Type = ScreenshotType.Png, // Explicit PNG for lossless quality
-            Animations = ScreenshotAnimations.Disabled // Disable animations to prevent artifacts
-        });
-        
-        _logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})", 
-            outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height);
+        return await workflow.ExecuteAsync(context, cancellationToken);
     }
 }
-

Services/SelectorService.cs

@@ -0,0 +1,60 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using Microsoft.Playwright;
+
+namespace BomLocalService.Services;
+
+public class SelectorService : ISelectorService
+{
+    private readonly ILogger<SelectorService> _logger;
+    
+    public SelectorService(ILogger<SelectorService> logger)
+    {
+        _logger = logger;
+    }
+    
+    public async Task<ILocator?> FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default)
+    {
+        foreach (var selector in config.Selectors)
+        {
+            try
+            {
+                var locator = page.Locator(selector).First;
+                if (await locator.IsVisibleAsync())
+                {
+                    _logger.LogInformation("Found {Name} with selector: {Selector}", config.Name, selector);
+                    return locator;
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Selector {Selector} not found or not visible for {Name}", selector, config.Name);
+                continue;
+            }
+        }
+        
+        if (config.Required)
+        {
+            var errorMsg = config.ErrorMessage ?? $"Required element {config.Name} not found with any selector";
+            _logger.LogWarning("{Message}", errorMsg);
+        }
+        else
+        {
+            _logger.LogDebug("Optional element {Name} not found with any selector", config.Name);
+        }
+        
+        return null;
+    }
+    
+    public ILocator GetLocator(IPage page, SelectorConfig config)
+    {
+        if (config.Selectors.Length == 0)
+        {
+            throw new ArgumentException($"No selectors configured for {config.Name}");
+        }
+        
+        // Returns first selector as locator (for cases where we know it exists)
+        return page.Locator(config.Selectors[0]).First;
+    }
+}
+

Services/TimeParsingService.cs

@@ -78,68 +78,94 @@ public class TimeParsingService : ITimeParsingService
         var info = new LastUpdatedInfo();
         _logger.LogInformation("Parsing last updated text: {Text}", text);
 
-        // Try to parse "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD"
-        // And "Forecast: 41 minutes ago, 7:50 pm AEST"
-        // Prefer parsing the actual time string over "minutes ago" for accuracy
-
-        // Parse observation time - improved regex to better capture time and timezone
-        // Stop timezone capture before "at" (e.g., "AEST at Gympie" or "AESTat Gympie" should capture just "AEST")
-        // Pattern: "Observations: X minutes ago, 9:40 pm AESTat Gympie..." (note: sometimes no space between AEST and "at")
-        // Capture timezone as 3-4 uppercase letters (AEST, AEDT, etc.) optionally followed by "at"
-        var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
-        _logger.LogDebug("Observation regex match result: Success={Success}, Groups.Count={Count}", observationMatch.Success, observationMatch.Groups.Count);
-        if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
+        // Parse observation time - use "minutes ago" as primary source for accuracy
+        // "Minutes ago" is reliable and doesn't require date guessing
+        // Time string is used only as fallback if "minutes ago" is unavailable
+        var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
+        if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
         {
-            var timeStr = observationMatch.Groups[1].Value.Trim();
-            var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null;
-            
-            // Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST")
-            if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2)
+            // Use "minutes ago" as primary source - it's accurate and doesn't require date guessing
+            info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
+            _logger.LogInformation("Calculated observation time from 'minutes ago': {MinutesAgo} minutes ago = {Time} UTC", 
+                minutesAgo, info.ObservationTime);
+        }
+        else
+        {
+            // Fallback to time string parsing only if "minutes ago" is not available
+            var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
+            if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
             {
-                timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2);
-            }
-            
-            _logger.LogDebug("Extracted observation time string: '{TimeStr}', timezone: '{TzStr}'", timeStr, timezoneStr ?? "null");
-            
-            if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
-            {
-                info.ObservationTime = observationTime;
-                _logger.LogInformation("Successfully parsed observation time: {Time} UTC (from '{TimeStr}' {TzStr})", 
-                    observationTime, timeStr, timezoneStr ?? "default timezone");
+                var timeStr = observationMatch.Groups[1].Value.Trim();
+                var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null;
+                
+                // Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST")
+                if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2)
+                {
+                    timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2);
+                }
+                
+                if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
+                {
+                    info.ObservationTime = observationTime;
+                    _logger.LogInformation("Used time string fallback for observation time: {Time} UTC (from '{TimeStr}' {TzStr})", 
+                        observationTime, timeStr, timezoneStr ?? "default timezone");
+                }
+                else
+                {
+                    _logger.LogWarning("Failed to parse observation time from both 'minutes ago' and time string, using current time");
+                    info.ObservationTime = DateTime.UtcNow;
+                }
             }
             else
             {
-                // Fallback to "minutes ago" if time parsing fails
-                _logger.LogWarning("Failed to parse observation time from '{TimeStr}', falling back to 'minutes ago'", timeStr);
-                var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
-                if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
-                {
-                    info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
-                    _logger.LogInformation("Used 'minutes ago' fallback: {MinutesAgo} minutes ago = {Time} UTC", 
-                        minutesAgo, info.ObservationTime);
-                }
+                _logger.LogWarning("No observation time found in text, using current time");
+                info.ObservationTime = DateTime.UtcNow;
             }
         }
 
-        // Parse forecast time
-        var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase);
-        if (forecastMatch.Success && forecastMatch.Groups[1].Success)
+        // Parse forecast time - use "minutes ago" or "an hour ago" as primary source
+        // Handle both "X minutes ago" and "an hour ago" formats
+        var forecastMinutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
+        var forecastHourAgoMatch = Regex.Match(text, @"Forecast:\s*an\s+hour\s+ago", RegexOptions.IgnoreCase);
+
+        if (forecastMinutesAgoMatch.Success && int.TryParse(forecastMinutesAgoMatch.Groups[1].Value, out var forecastMinutesAgo))
         {
-            var timeStr = forecastMatch.Groups[1].Value.Trim();
-            var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null;
-            
-            if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
+            // Use "minutes ago" as primary source
+            info.ForecastTime = DateTime.UtcNow.AddMinutes(-forecastMinutesAgo);
+            _logger.LogInformation("Calculated forecast time from 'minutes ago': {MinutesAgo} minutes ago = {Time} UTC", 
+                forecastMinutesAgo, info.ForecastTime);
+        }
+        else if (forecastHourAgoMatch.Success)
+        {
+            // Handle "an hour ago" format
+            info.ForecastTime = DateTime.UtcNow.AddHours(-1);
+            _logger.LogInformation("Calculated forecast time from 'an hour ago': {Time} UTC", info.ForecastTime);
+        }
+        else
+        {
+            // Fallback to time string parsing only if "minutes ago" or "an hour ago" is not available
+            var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:an\s+hour\s+ago|\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase);
+            if (forecastMatch.Success && forecastMatch.Groups[1].Success)
             {
-                info.ForecastTime = forecastTime;
+                var timeStr = forecastMatch.Groups[1].Value.Trim();
+                var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null;
+                
+                if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
+                {
+                    info.ForecastTime = forecastTime;
+                    _logger.LogInformation("Used time string fallback for forecast time: {Time} UTC (from '{TimeStr}' {TzStr})", 
+                        forecastTime, timeStr, timezoneStr ?? "default timezone");
+                }
+                else
+                {
+                    _logger.LogWarning("Failed to parse forecast time from both 'minutes ago' and time string, using current time");
+                    info.ForecastTime = DateTime.UtcNow;
+                }
             }
             else
             {
-                // Fallback to "minutes ago" if time parsing fails
-                var minutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
-                if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
-                {
-                    info.ForecastTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
-                }
+                _logger.LogWarning("No forecast time found in text, using current time");
+                info.ForecastTime = DateTime.UtcNow;
             }
         }
 
@@ -223,18 +249,18 @@ public class TimeParsingService : ITimeParsingService
             var nowInTz = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz);
             var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay);
             
-            // If the parsed time is in the future (e.g., it's 9pm but we parsed 8pm and it's actually yesterday),
-            // or if it's more than 12 hours in the past, assume it's from yesterday
+            // Only adjust to yesterday if the time is clearly in the future
+            // Don't use the 12-hour threshold - it's too aggressive for recent observations
+            // This method is now only used as fallback when "minutes ago" is unavailable
             if (dateTimeInTz > nowInTz)
             {
                 // Time is in the future, so it must be from yesterday
                 dateTimeInTz = dateTimeInTz.AddDays(-1);
+                _logger.LogDebug("Adjusted date to yesterday because parsed time {ParsedTime} is in the future (current: {CurrentTime})", 
+                    dateTimeInTz, nowInTz);
             }
-            else if ((nowInTz - dateTimeInTz).TotalHours > 12)
-            {
-                // More than 12 hours ago, likely from yesterday
-                dateTimeInTz = dateTimeInTz.AddDays(-1);
-            }
+            // Removed the 12-hour threshold check - it was causing incorrect date assignments
+            // If we're using this fallback, we accept the date as-is (today) unless it's clearly in the future
             
             // Convert to UTC
             // Note: Brisbane (Australia/Brisbane) is always UTC+10 (AEST), no daylight saving

Utilities/CacheHelper.cs

@@ -61,5 +61,28 @@ public static class CacheHelper
     {
         return IsCacheFolderCompleteForDataType(cacheFolderPath, CachedDataType.Radar, configuration);
     }
+
+    /// <summary>
+    /// Calculates the estimated cache update duration in seconds.
+    /// This is used as a fallback when metrics-based estimation is not available yet.
+    /// Calculates based on frame count and configured wait times.
+    /// </summary>
+    public static int GetEstimatedUpdateDurationSeconds(IConfiguration configuration, CachedDataType dataType = CachedDataType.Radar)
+    {
+        // Calculate based on actual wait times and frame count
+        var frameCount = GetFrameCountForDataType(configuration, dataType);
+        var tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
+        var dynamicContentWaitMs = configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
+        
+        // Rough calculation:
+        // - Initial page load and navigation: ~10-15 seconds
+        // - Per frame: tileRenderWaitMs (default 5s) + overhead (~1-2s for clicking, waiting, etc.)
+        // - Final processing and metadata saving: ~5 seconds
+        var perFrameSeconds = (tileRenderWaitMs + 1500) / 1000.0; // Add 1.5s overhead per frame
+        var baseOverheadSeconds = 15; // Initial load + final processing
+        var estimatedSeconds = (int)Math.Ceiling(baseOverheadSeconds + (frameCount * perFrameSeconds));
+        
+        return estimatedSeconds;
+    }
 }
 

Utilities/LocationHelper.cs

@@ -22,7 +22,8 @@ public static class LocationHelper
     }
 
     /// <summary>
-    /// Parses suburb and state from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png)
+    /// Parses suburb and state from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS)
+    /// Handles multi-word suburbs like "Gold Coast" which becomes "Gold_Coast_QLD_20251216_130831"
     /// Returns null if parsing fails
     /// </summary>
     public static (string suburb, string state)? ParseLocationFromFilename(string fileName)
@@ -30,12 +31,21 @@ public static class LocationHelper
         var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
         var parts = fileNameWithoutExtension.Split('_');
         
-        if (parts.Length >= 2)
+        // Need at least: suburb, state, date, time (4 parts minimum)
+        // Format: [Suburb_Parts...]_State_YYYYMMDD_HHMMSS
+        if (parts.Length < 4)
         {
-            return (parts[0], parts[1]);
+            return null;
         }
         
-        return null;
+        // Last two parts are always timestamp (YYYYMMDD, HHMMSS)
+        // Second-to-last part is the state
+        // Everything before that is the suburb (may contain underscores from multi-word suburbs)
+        var state = parts[^3]; // Third from end
+        var suburbParts = parts.Take(parts.Length - 3).ToArray();
+        var suburb = string.Join(" ", suburbParts); // Join with spaces (original format had spaces converted to underscores)
+        
+        return (suburb, state);
     }
 
     /// <summary>

Utilities/ResponseBuilder.cs

@@ -4,20 +4,48 @@ namespace BomLocalService.Utilities;
 
 public static class ResponseBuilder
 {
+    /// <summary>
+    /// Calculates the next time the background cache management service will check for updates.
+    /// Rounds up to the next interval boundary within the current hour, or wraps to next hour if needed.
+    /// </summary>
+    public static DateTime CalculateNextServiceCheck(int checkIntervalMinutes)
+    {
+        var now = DateTime.UtcNow;
+        var currentMinute = now.Minute;
+        
+        // Calculate how many complete intervals have elapsed this hour
+        var intervalsElapsed = currentMinute / checkIntervalMinutes;
+        
+        // Next interval starts at (intervalsElapsed + 1) * checkIntervalMinutes
+        var nextIntervalStartMinute = (intervalsElapsed + 1) * checkIntervalMinutes;
+        
+        // If next interval is beyond 60 minutes, wrap to next hour
+        if (nextIntervalStartMinute >= 60)
+        {
+            // Next check is at the start of next hour (00 minutes)
+            return now.Date.AddHours(now.Hour + 1).AddMinutes(0);
+        }
+        
+        // Next check is within current hour
+        return now.Date.AddHours(now.Hour).AddMinutes(nextIntervalStartMinute);
+    }
+
     /// <summary>
     /// Creates a RadarResponse from a cache folder path, frames, and metadata
     /// </summary>
     /// <param name="cacheManagementCheckIntervalMinutes">The interval in minutes that the background cache management service checks for updates. Used to calculate NextUpdateTime when cache is invalid.</param>
+    /// <param name="estimatedUpdateDurationSeconds">Estimated duration in seconds for a cache update to complete. Used to calculate NextUpdateTime when update is in progress.</param>
     public static RadarResponse CreateRadarResponse(
         string cacheFolderPath, 
         List<RadarFrame> frames,
+        int cacheManagementCheckIntervalMinutes,
         LastUpdatedInfo? metadata = null,
         string? suburb = null,
         string? state = null,
         bool? cacheIsValid = null,
         DateTime? cacheExpiresAt = null,
         bool isUpdating = false,
-        int cacheManagementCheckIntervalMinutes = 5)
+        int? estimatedUpdateDurationSeconds = null)
     {
         var folderInfo = new DirectoryInfo(cacheFolderPath);
         var lastWriteTime = folderInfo.Exists 
@@ -35,6 +63,20 @@ public static class ResponseBuilder
             }
         }
 
+        // Calculate AbsoluteObservationTime for all frames if metadata is available
+        // This ensures consistency with the time series endpoint
+        if (metadata != null && metadata.ObservationTime > DateTime.MinValue.AddYears(1))
+        {
+            foreach (var frame in frames)
+            {
+                // Only calculate if not already set and MinutesAgo is valid
+                if (!frame.AbsoluteObservationTime.HasValue && frame.MinutesAgo >= 0)
+                {
+                    frame.AbsoluteObservationTime = metadata.ObservationTime.AddMinutes(-frame.MinutesAgo);
+                }
+            }
+        }
+
         // Calculate NextUpdateTime based on cache status:
         // - If cache is valid: NextUpdateTime = max(CacheExpiresAt, next background service check)
         //   (Background service checks every N minutes, so update might happen at next check after expiry)
@@ -44,13 +86,13 @@ public static class ResponseBuilder
         
         // Calculate next background service check time (rounds up to next check interval)
         var now = DateTime.UtcNow;
-        var minutesUntilNextCheck = cacheManagementCheckIntervalMinutes - (now.Minute % cacheManagementCheckIntervalMinutes);
-        var nextServiceCheck = now.AddMinutes(minutesUntilNextCheck);
+        var nextServiceCheck = CalculateNextServiceCheck(cacheManagementCheckIntervalMinutes);
         
         if (isUpdating)
         {
-            // Update in progress - estimate completion in ~2 minutes
-            nextUpdateTime = now.AddMinutes(2);
+            // Update in progress - estimate completion based on configured/calculated duration
+            var durationSeconds = estimatedUpdateDurationSeconds ?? 120; // Default to 2 minutes if not provided
+            nextUpdateTime = now.AddSeconds(durationSeconds);
         }
         else if (cacheIsValid == true && cacheExpiresAt.HasValue)
         {

Views/RadarTest/Index.cshtml

@@ -736,7 +736,10 @@
                 <div class="info-card">
                     <h3>Cache & Update Status</h3>
                     <div class="value" id="cache-status">Checking...</div>
-                    <div class="timestamp" id="update-status-detail"></div>
+                    <div class="timestamp" id="update-status-detail" style="cursor: help;"></div>
+                    <div style="font-size: 0.85em; color: #666; margin-top: 5px; font-style: italic;">
+                        <span id="estimation-note" style="display: none;">Estimates improve as metrics are collected from completed updates</span>
+                    </div>
                 </div>
                 
                 <div class="info-card">
@@ -837,8 +840,8 @@
                 
                 <div style="margin-bottom: 20px;">
                     <label style="display: block; margin-bottom: 8px; font-weight: 600; color: #333;">Frame Interval (seconds)</label>
-                    <input type="number" id="frame-interval-input" min="0.5" max="10" step="0.5" style="width: 100%; padding: 12px; border: 2px solid #667eea; border-radius: 8px; font-size: 1em;">
-                    <div class="timestamp" style="margin-top: 6px;">Time between frames in slideshow</div>
+                    <input type="number" id="frame-interval-input" min="0.1" step="0.1" style="width: 100%; padding: 12px; border: 2px solid #667eea; border-radius: 8px; font-size: 1em;">
+                    <div class="timestamp" style="margin-top: 6px;">Time between frames in slideshow (minimum 0.1 seconds)</div>
                 </div>
             </div>
             
@@ -847,8 +850,8 @@
                 
                 <div style="margin-bottom: 20px;">
                     <label style="display: block; margin-bottom: 8px; font-weight: 600; color: #333;">Auto Refresh Interval (seconds)</label>
-                    <input type="number" id="refresh-interval-input" min="10" max="300" step="10" style="width: 100%; padding: 12px; border: 2px solid #667eea; border-radius: 8px; font-size: 1em;">
-                    <div class="timestamp" style="margin-top: 6px;">How often to check for new radar data</div>
+                    <input type="number" id="refresh-interval-input" min="5" step="5" style="width: 100%; padding: 12px; border: 2px solid #667eea; border-radius: 8px; font-size: 1em;">
+                    <div class="timestamp" style="margin-top: 6px;">How often to check for new radar data (minimum 5 seconds)</div>
                 </div>
             </div>
             
@@ -860,7 +863,7 @@
                 <label style="display: block; margin-bottom: 8px; font-weight: 600; color: #333;">End Time</label>
                 <input type="datetime-local" id="end-time-input" style="width: 100%; padding: 12px; border: 2px solid #667eea; border-radius: 8px; font-size: 1em;">
                 
-                <div class="timestamp" style="margin-top: 6px;">Select custom time range (times are in local timezone)</div>
+                <div class="timestamp" style="margin-top: 6px;">Select custom time range (times are in local timezone). Maximum range is based on your cache retention settings.</div>
             </div>
             
             <div style="margin-bottom: 25px; padding-top: 20px; border-top: 2px solid #f0f0f0;">
@@ -1034,10 +1037,23 @@
                 }
                 
                 // Update update status detail with relative time
+                const estimationNoteEl = document.getElementById('estimation-note');
                 if (radarData.isUpdating && radarData.nextUpdateTime && updateStatusDetailEl) {
-                    updateStatusDetailEl.textContent = 'Estimated completion: ' + formatDate(radarData.nextUpdateTime) + ' (' + getRelativeTime(radarData.nextUpdateTime) + ')';
+                    const relativeTime = getRelativeTime(radarData.nextUpdateTime);
+                    updateStatusDetailEl.textContent = 'Update in progress • Estimated completion: ' + formatDate(radarData.nextUpdateTime) + ' (' + relativeTime + ')';
+                    updateStatusDetailEl.title = 'Estimate is based on historical metrics and current progress. Accuracy improves as more updates complete.';
+                    if (estimationNoteEl) estimationNoteEl.style.display = 'inline';
                 } else if (!radarData.cacheIsValid && !radarData.isUpdating && radarData.nextUpdateTime && updateStatusDetailEl) {
-                    updateStatusDetailEl.textContent = 'Update scheduled: ' + formatDate(radarData.nextUpdateTime) + ' (' + getRelativeTime(radarData.nextUpdateTime) + ')';
+                    const relativeTime = getRelativeTime(radarData.nextUpdateTime);
+                    updateStatusDetailEl.textContent = 'Update scheduled: ' + formatDate(radarData.nextUpdateTime) + ' (' + relativeTime + ')';
+                    updateStatusDetailEl.title = 'Estimated time when cache update will be triggered or completed. Based on metrics from previous updates.';
+                    if (estimationNoteEl) estimationNoteEl.style.display = 'inline';
+                } else if (radarData.cacheIsValid && updateStatusDetailEl) {
+                    updateStatusDetailEl.textContent = 'Cache is valid, no update needed';
+                    updateStatusDetailEl.title = '';
+                    if (estimationNoteEl) estimationNoteEl.style.display = 'none';
+                } else {
+                    if (estimationNoteEl) estimationNoteEl.style.display = 'none';
                 }
                 
                 // Update next client check time
@@ -1121,8 +1137,71 @@
                 });
                 
                 if (!response.ok) {
-                    const error = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-                    throw new Error(error.error || 'Failed to fetch historical radar');
+                    const errorData = await response.json().catch(() => ({ message: `HTTP ${response.status}` }));
+                    
+                    // Handle 400 Bad Request (e.g., time range too large, validation errors)
+                    if (response.status === 400) {
+                        const errorMessage = errorData.message || 'Invalid request';
+                        let details = '';
+                        // Check if it's a time range error with details
+                        if (errorData.errorCode === 'TIME_RANGE_ERROR' && errorData.details?.requestedRange) {
+                            const requested = errorData.details.requestedRange;
+                            if (requested.requestedHours && errorData.details.maxHours) {
+                                details = ` Requested: ${requested.requestedHours.toFixed(1)} hours, Maximum: ${errorData.details.maxHours} hours.`;
+                            }
+                        }
+                        throw new Error(errorMessage + details);
+                    }
+                    
+                    // Handle 404 Not Found
+                    if (response.status === 404) {
+                        // Check error code to determine type
+                        if (errorData.errorCode === 'CACHE_NOT_FOUND') {
+                            // Location doesn't exist - trigger cache update if endpoint provided
+                            if (errorData.suggestions?.refreshEndpoint) {
+                                fetch(errorData.suggestions.refreshEndpoint, {
+                                    method: 'POST',
+                                    signal: AbortSignal.timeout(5000)
+                                }).catch(err => {
+                                    console.debug('Background cache refresh trigger failed (non-critical):', err);
+                                });
+                            }
+                            
+                            // Return special error code to indicate location missing
+                            return { 
+                                frames: null, 
+                                error: 'location_missing',
+                                cacheStatus: errorData.details || errorData
+                            };
+                        } else if (errorData.errorCode === 'TIME_RANGE_ERROR' && errorData.details?.availableRange) {
+                            // Cache exists but no data in requested range
+                            const availableRange = errorData.details.availableRange;
+                            let rangeMessage = '';
+                            
+                            // Use suggested range from error response if available
+                            if (errorData.suggestions?.suggestedRange) {
+                                const suggested = errorData.suggestions.suggestedRange;
+                                rangeMessage = ` Suggested range: ${formatDate(suggested.start)} to ${formatDate(suggested.end)}.`;
+                            } else if (availableRange.oldest && availableRange.newest) {
+                                rangeMessage = ` Available data: ${formatDate(availableRange.oldest)} to ${formatDate(availableRange.newest)}.`;
+                            } else if (availableRange.totalCacheFolders > 0) {
+                                rangeMessage = ` ${availableRange.totalCacheFolders} cache folders available.`;
+                            }
+                            
+                            // Include suggestion text if provided
+                            if (errorData.suggestions?.suggestion) {
+                                rangeMessage += ` ${errorData.suggestions.suggestion}`;
+                            }
+                            
+                            throw new Error(errorData.message + rangeMessage);
+                        } else {
+                            // Generic 404
+                            throw new Error(errorData.message || 'No historical data found');
+                        }
+                    }
+                    
+                    // Other errors
+                    throw new Error(errorData.message || 'Failed to fetch historical radar');
                 }
                 
                 const data = await response.json();
@@ -1282,6 +1361,29 @@
                         throw new Error('Network error - API unavailable');
                     }
                     
+                    // Handle location missing case (cache update triggered)
+                    if (result.error === 'location_missing') {
+                        const cacheStatus = result.cacheStatus || {};
+                        const isUpdating = cacheStatus.updateTriggered === true || 
+                                         (cacheStatus.message && cacheStatus.message.includes('in progress'));
+                        
+                        // Create a data-like object with cache status for updateUI
+                        const cacheStatusData = {
+                            frames: [],
+                            isUpdating: isUpdating,
+                            cacheIsValid: cacheStatus.cacheIsValid || false,
+                            cacheExpiresAt: cacheStatus.cacheExpiresAt || null,
+                            nextUpdateTime: cacheStatus.nextUpdateTime || null
+                        };
+                        updateUI(cacheStatusData);
+                        
+                        const retryMessage = cacheStatus.retryAfter ? 
+                            ` Please retry in ${cacheStatus.retryAfter} seconds.` : 
+                            ' Please retry in a few moments.';
+                        showNoFramesMessage('No cached data found for this location. Cache update has been triggered in background.' + retryMessage);
+                        return null;
+                    }
+                    
                     const allFrames = result.frames;
                     if (!allFrames || allFrames.length === 0) {
                         showNoFramesMessage('No frames found for selected timespan. Try a different range or wait for more cache data.');
@@ -1351,7 +1453,7 @@
                     });
                     
                     if (!response.ok) {
-                        const error = await response.json().catch(() => ({ error: `HTTP ${response.status}: ${response.statusText}` }));
+                        const error = await response.json().catch(() => ({ message: `HTTP ${response.status}: ${response.statusText}` }));
                         
                         // Handle 404 more gracefully - cache is being generated
                         if (response.status === 404) {
@@ -1370,10 +1472,12 @@
                                 updateApiStatus('connected');
                             }
                             
+                            // Use standardized error response format
                             // Explicitly trigger cache refresh if refreshEndpoint is provided
-                            if (errorData.refreshEndpoint) {
+                            const refreshEndpoint = errorData.suggestions?.refreshEndpoint || errorData.refreshEndpoint;
+                            if (refreshEndpoint) {
                                 // Trigger cache update in background (fire and forget)
-                                fetch(errorData.refreshEndpoint, {
+                                fetch(refreshEndpoint, {
                                     method: 'POST',
                                     signal: AbortSignal.timeout(5000)
                                 }).catch(err => {
@@ -1382,26 +1486,24 @@
                             }
                             
                             // Use actual cache status from API if available
-                            // Determine if update is in progress: UpdateTriggered=true OR Message indicates update in progress
-                            if (errorData.updateTriggered !== undefined || errorData.cacheIsValid !== undefined || errorData.cacheExists !== undefined) {
+                            // Check both new format (details) and old format for backward compatibility
+                            const details = errorData.details || errorData;
+                            if (details.updateTriggered !== undefined || details.cacheIsValid !== undefined || details.cacheExists !== undefined) {
                                 // Determine if update is in progress
-                                const isUpdating = errorData.updateTriggered === true || 
-                                                 (errorData.message && errorData.message.includes('in progress'));
+                                const isUpdating = details.updateTriggered === true || 
+                                                 (errorData.message && errorData.message.includes('in progress')) ||
+                                                 (details.statusMessage && details.statusMessage.includes('in progress'));
                                 
                                 // Create a data-like object with cache status for updateUI
                                 const cacheStatusData = {
                                     frames: [],
                                     isUpdating: isUpdating,
-                                    cacheIsValid: errorData.cacheIsValid || false,
-                                    cacheExpiresAt: errorData.cacheExpiresAt || null,
-                                    nextUpdateTime: errorData.nextUpdateTime || null
+                                    cacheIsValid: details.cacheIsValid || false,
+                                    cacheExpiresAt: details.cacheExpiresAt || null,
+                                    nextUpdateTime: details.nextUpdateTime || null
                                 };
                                 console.debug('Updating UI with cache status from 404:', cacheStatusData);
                                 updateUI(cacheStatusData);
-                            } else if (errorData.retryAfter) {
-                                // Fallback to old method if cache status not available
-                                console.debug('Cache status not in 404 response, using fallback method');
-                                updateStatusForCacheUpdate(errorData.retryAfter, errorData.refreshEndpoint);
                             } else {
                                 // No cache status info at all - set default updating status
                                 console.debug('No cache status info available, setting default updating status');
@@ -1413,11 +1515,13 @@
                                 updateUI(defaultStatusData);
                             }
                             
-                            showNoFramesMessage(`Cache is being generated. Please wait ${errorData.retryAfter || 30} seconds and refresh.`, errorData.refreshEndpoint);
+                            const retryAfter = errorData.suggestions?.retryAfter || errorData.retryAfter || 30;
+                            const message = errorData.message || `Cache is being generated. Please wait ${retryAfter} seconds and refresh.`;
+                            showNoFramesMessage(message, refreshEndpoint);
                             return null;
                         }
                         
-                        throw new Error(error.error || 'Failed to fetch radar data');
+                        throw new Error(error.message || error.error || 'Failed to fetch radar data');
                     }
                     
                     const data = await response.json();
@@ -1628,20 +1732,33 @@
                 
                 // Update update status detail (now shown below cache status in combined card)
                 const updateStatusDetailEl = document.getElementById('update-status-detail');
+                const estimationNoteEl = document.getElementById('estimation-note');
                 if (data.isUpdating !== undefined || data.cacheIsValid !== undefined) {
                     if (data.isUpdating) {
                         if (data.nextUpdateTime) {
-                            updateStatusDetailEl.textContent = 'Estimated completion: ' + formatDate(data.nextUpdateTime) + ' (' + getRelativeTime(data.nextUpdateTime) + ')';
+                            const relativeTime = getRelativeTime(data.nextUpdateTime);
+                            updateStatusDetailEl.textContent = 'Update in progress • Estimated completion: ' + formatDate(data.nextUpdateTime) + ' (' + relativeTime + ')';
+                            updateStatusDetailEl.title = 'Estimate is based on historical metrics and current progress. Accuracy improves as more updates complete.';
+                            if (estimationNoteEl) estimationNoteEl.style.display = 'inline';
                         } else {
                             updateStatusDetailEl.textContent = 'Update in progress, please wait...';
+                            updateStatusDetailEl.title = 'Cache update is in progress. Completion time will be estimated once progress tracking begins.';
+                            if (estimationNoteEl) estimationNoteEl.style.display = 'none';
                         }
                     } else if (data.cacheIsValid) {
-                        updateStatusDetailEl.textContent = 'No update needed';
+                        updateStatusDetailEl.textContent = 'Cache is valid, no update needed';
+                        updateStatusDetailEl.title = '';
+                        if (estimationNoteEl) estimationNoteEl.style.display = 'none';
                     } else {
                         if (data.nextUpdateTime) {
-                            updateStatusDetailEl.textContent = 'Update scheduled: ' + formatDate(data.nextUpdateTime) + ' (' + getRelativeTime(data.nextUpdateTime) + ')';
+                            const relativeTime = getRelativeTime(data.nextUpdateTime);
+                            updateStatusDetailEl.textContent = 'Update scheduled: ' + formatDate(data.nextUpdateTime) + ' (' + relativeTime + ')';
+                            updateStatusDetailEl.title = 'Estimated time when cache update will be triggered or completed. Based on metrics from previous updates.';
+                            if (estimationNoteEl) estimationNoteEl.style.display = 'inline';
                         } else {
                             updateStatusDetailEl.textContent = 'Update will be triggered by background service';
+                            updateStatusDetailEl.title = 'The background cache management service will check and update the cache periodically.';
+                            if (estimationNoteEl) estimationNoteEl.style.display = 'none';
                         }
                     }
                 }
@@ -2126,8 +2243,11 @@
                 settingsBtn.addEventListener('click', showSettings);
             }
             document.getElementById('save-settings-btn').addEventListener('click', () => {
-                settings.frameInterval = parseFloat(document.getElementById('frame-interval-input').value) || 2.0;
-                settings.refreshInterval = parseInt(document.getElementById('refresh-interval-input').value) || 30;
+                const frameIntervalValue = parseFloat(document.getElementById('frame-interval-input').value);
+                settings.frameInterval = (frameIntervalValue > 0) ? frameIntervalValue : 2.0;
+                
+                const refreshIntervalValue = parseInt(document.getElementById('refresh-interval-input').value);
+                settings.refreshInterval = (refreshIntervalValue >= 5) ? refreshIntervalValue : 30;
                 settings.autoPlay = document.getElementById('auto-play-input').checked;
                 settings.timespan = document.getElementById('timespan-select').value;
                 

appsettings.json

@@ -6,6 +6,12 @@
     }
   },
   "AllowedHosts": "*",
+  "Cors": {
+    "AllowedOrigins": "*",
+    "AllowedMethods": "GET,POST,OPTIONS",
+    "AllowedHeaders": "*",
+    "AllowCredentials": false
+  },
   "CacheDirectory": "./cache",
   "CacheRetentionHours": 24,
   "CacheExpirationMinutes": 12.5,
@@ -19,8 +25,8 @@
     "IntervalHours": 1
   },
   "Screenshot": {
-    "DynamicContentWaitMs": 2000,
-    "TileRenderWaitMs": 5000,
+    "DynamicContentWaitMs": 1500,
+    "TileRenderWaitMs": 3000,
     "Crop": {
       "X": 250,
       "Y": 0,
@@ -37,5 +43,204 @@
     "Radar": {
       "FrameCount": 7
     }
+  },
+  "TimeSeries": {
+    "WarningFolderCount": 200,
+    "MaxTimeRangeHours": null
+  },
+  "Scraping": {
+    "BaseUrl": "https://www.bom.gov.au/",
+    "Timeouts": {
+      "PageNavigation": 30000,
+      "ElementWait": 10000,
+      "SearchResults": 10000,
+      "MapRender": 15000,
+      "TileRender": 5000
+    },
+    "Selectors": {
+      "SearchButton": {
+        "Name": "Search Button",
+        "Selectors": [
+          "button[data-testid='searchLabel']",
+          "button[aria-label='Search for a location']",
+          "button.search-location__trigger-button",
+          "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
+          "button:has-text('Search for a location')",
+          "[data-testid='showSearchModal'] button"
+        ],
+        "TimeoutMs": 10000,
+        "Required": true,
+        "ErrorMessage": "Could not find 'Search for a location' button on BOM homepage."
+      },
+      "SearchInput": {
+        "Name": "Search Input",
+        "Selectors": ["#search-enter-keyword"],
+        "TimeoutMs": 5000,
+        "Required": true
+      },
+      "SearchResultsList": {
+        "Name": "Search Results List",
+        "Selectors": ["ul[aria-labelledby='location-results-title']"],
+        "TimeoutMs": 10000,
+        "Required": true
+      },
+      "SearchResultItem": {
+        "Name": "Search Result Item",
+        "Selectors": ["li.bom-linklist__item[role='listitem']"],
+        "TimeoutMs": 5000,
+        "Required": true
+      },
+      "LocationName": {
+        "Name": "Location Name",
+        "Selectors": ["[data-testid='location-name']"],
+        "TimeoutMs": 5000,
+        "Required": false
+      },
+      "LocationDescription": {
+        "Name": "Location Description",
+        "Selectors": [".bom-linklist-item__desc"],
+        "TimeoutMs": 5000,
+        "Required": false
+      },
+      "ResultsTitle": {
+        "Name": "Results Title",
+        "Selectors": ["#location-results-title", "[data-testid='location-results-title']"],
+        "TimeoutMs": 5000,
+        "Required": false
+      },
+      "RadarLink": {
+        "Name": "Radar Link",
+        "Selectors": [
+          "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
+          ".cta.button a.jump-link:has-text('Rain radar and weather map')",
+          "a.jump-link:has-text('Rain radar and weather map')",
+          ".cta.button a:has-text('Rain radar and weather map')",
+          "a.bom-button--secondary:has-text('Rain radar and weather map')",
+          "a:has-text('Rain radar and weather map')",
+          "a:has-text('rain radar')",
+          "a[href*='radar']"
+        ],
+        "TimeoutMs": 10000,
+        "Required": true,
+        "ErrorMessage": "Could not find 'Rain radar and weather map' link"
+      },
+      "MapCanvas": {
+        "Name": "Map Canvas",
+        "Selectors": [".esri-view-surface canvas"],
+        "TimeoutMs": 15000,
+        "Required": true
+      },
+      "MapContainer": {
+        "Name": "Map Container",
+        "Selectors": [".esri-view-surface"],
+        "TimeoutMs": 10000,
+        "Required": true
+      },
+      "PlayPauseButton": {
+        "Name": "Play Pause Button",
+        "Selectors": ["button[data-testid='bom-time-scrub-play-pause']"],
+        "TimeoutMs": 5000,
+        "Required": true
+      },
+      "PlayPauseLabel": {
+        "Name": "Play Pause Label",
+        "Selectors": [".bom-scrub-action__label"],
+        "TimeoutMs": 2000,
+        "Required": false
+      },
+      "FrameSegment": {
+        "Name": "Frame Segment",
+        "Selectors": ["[data-testid='bom-scrub-segment']"],
+        "TimeoutMs": 5000,
+        "Required": false
+      },
+      "StepForwardButton": {
+        "Name": "Step Forward Button",
+        "Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"],
+        "TimeoutMs": 5000,
+        "Required": true
+      },
+      "TimeDisplayLabel": {
+        "Name": "Time Display Label",
+        "Selectors": [".bom-scrub-display-label"],
+        "TimeoutMs": 2000,
+        "Required": false
+      },
+      "ModalOverlay": {
+        "Name": "Modal Overlay",
+        "Selectors": [".bom-modal-overlay--after-open"],
+        "TimeoutMs": 1000,
+        "Required": false
+      },
+      "WeatherMetadata": {
+        "Name": "Weather Metadata",
+        "Selectors": ["section[data-testid='weatherMetadata']", "section[aria-label='Last updated']"],
+        "TimeoutMs": 5000,
+        "Required": false
+      }
+    },
+    "JavaScriptTemplates": {
+      "WaitForSearchResults": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.length > 0 && results.some(r => r.offsetParent !== null); }",
+      "ExtractSearchResults": "() => { const resultsList = document.querySelector('ul[aria-labelledby=\"location-results-title\"]'); if (!resultsList) { console.log('Location results list not found'); return []; } const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); console.log('Found', results.length, 'location results'); return results.map((r) => { const nameEl = r.querySelector('[data-testid=\"location-name\"]'); const descEl = r.querySelector('.bom-linklist-item__desc'); const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; const fullText = (r.textContent || r.innerText || '').trim(); console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); return [name, desc, fullText]; }); }",
+      "ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }",
+      "WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }",
+      "WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }",
+      "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
+      "WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }",
+      "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
+      "CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }",
+      "CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }",
+      "GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })",
+      "ExtractWeatherMetadata": "() => { const section = document.querySelector('section[data-testid=\"weatherMetadata\"]') || document.querySelector('section[aria-label=\"Last updated\"]'); if (!section) return null; const divs = section.querySelectorAll('div'); return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); }"
+    },
+    "TextPatterns": {
+      "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
+      "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
+      "ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago",
+      "ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago",
+      "ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago",
+      "WeatherStationPattern": "at\\s+([^,]+)\\s+weather\\s+station",
+      "DistancePattern": "(\\d+)\\s*km\\s+from",
+      "ExpectedTexts": {
+        "PlayButtonLabel": "Play",
+        "PauseButtonLabel": "Pause",
+        "RadarLinkText": "Rain radar and weather map"
+      }
+    },
+    "Workflows": {
+      "RadarScraping": {
+        "Description": "Scrapes radar images for a location",
+        "Steps": {
+          "NavigateHomepage": { "Enabled": true },
+          "ClickSearchButton": { "Enabled": true },
+          "FillSearchInput": { "Enabled": true },
+          "WaitForSearchResults": { "Enabled": true },
+          "SelectSearchResult": { "Enabled": true },
+          "ClickRadarLink": { "Enabled": true },
+          "WaitForMapReady": { "Enabled": true },
+          "PauseRadar": { "Enabled": true },
+          "ResetToFirstFrame": { "Enabled": true },
+          "ExtractMetadata": { "Enabled": true },
+          "CalculateMapBounds": { "Enabled": true },
+          "CaptureFrames": {
+            "Enabled": true,
+            "Parameters": {
+              "FrameCount": 7,
+              "WaitBetweenFrames": 5000
+            }
+          }
+        }
+      },
+      "TemperatureMap": {
+        "Description": "Scrapes temperature forecast map (future feature)",
+        "Steps": {
+          "NavigateHomepage": { "Enabled": true },
+          "ClickSearchButton": { "Enabled": true },
+          "FillSearchInput": { "Enabled": true },
+          "WaitForSearchResults": { "Enabled": true },
+          "SelectSearchResult": { "Enabled": true }
+        }
+      }
+    }
   }
 }

docker-compose.yml

@@ -17,6 +17,14 @@ services:
       # Virtual display for non-headless browser
       - DISPLAY=:99
       
+      # CORS Configuration (for Home Assistant integration)
+      # Multiple origins can be comma-separated: "http://homeassistant.local:8123,http://192.168.1.100:8123"
+      # Use "*" to allow all origins (less secure, but convenient for development)
+      - CORS__ALLOWEDORIGINS=${CORS__ALLOWEDORIGINS:-*}
+      - CORS__ALLOWEDMETHODS=${CORS__ALLOWEDMETHODS:-GET,POST,OPTIONS}
+      - CORS__ALLOWEDHEADERS=${CORS__ALLOWEDHEADERS:-*}
+      - CORS__ALLOWCREDENTIALS=${CORS__ALLOWCREDENTIALS:-false}
+      
       # Application configuration (override appsettings.json)
       # Use double underscore (__) for nested config keys
       - CACHEDIRECTORY=${CACHEDIRECTORY:-/app/cache}

scripts/docker-start.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+# Function to start Xvfb
+start_xvfb() {
+    if ! pgrep -f "Xvfb :99" > /dev/null; then
+        echo "Starting Xvfb on display :99..."
+        Xvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset > /dev/null 2>&1 &
+        sleep 1
+        if ! pgrep -f "Xvfb :99" > /dev/null; then
+            echo "ERROR: Xvfb failed to start" >&2
+            return 1
+        fi
+        echo "Xvfb started successfully"
+    fi
+    return 0
+}
+
+# Function to monitor and restart Xvfb if it dies (runs as background process)
+monitor_xvfb() {
+    # Monitor while PID 1 (dotnet after exec) is still running
+    while kill -0 1 2>/dev/null; do
+        sleep 5
+        if ! pgrep -f "Xvfb :99" > /dev/null; then
+            echo "WARNING: Xvfb process not found, restarting..." >&2
+            start_xvfb || true
+        fi
+    done
+}
+
+# Cleanup function
+cleanup() {
+    echo "Shutting down Xvfb..."
+    pkill -f "Xvfb :99" || true
+}
+
+# Set up signal handlers for cleanup
+trap cleanup SIGTERM SIGINT EXIT
+
+# Start Xvfb
+start_xvfb || exit 1
+
+# Export DISPLAY
+export DISPLAY=:99
+
+# Start monitoring in background (will check if PID 1 is still running)
+monitor_xvfb &
+
+# Start the .NET application (replaces shell as PID 1)
+exec dotnet BomLocalService.dll