Fix frame deduplication and chronological ordering in time series endpoint

- Fix scraping to wait for display label to change after step forward, preventing duplicate minutesAgo values within same folder
- Add hashset-based deduplication by AbsoluteObservationTime across cache folders
- Filter frames that are too close together (< 4 minutes apart)
- Preserve original folder structure while pruning filtered frames
- Sort folders by earliest frame time to maintain chronological order
- Add validation to detect folder overlaps that could cause out-of-order frames

Controllers/CacheController.cs

@@ -0,0 +1,112 @@
+using BomLocalService.Models;
+using BomLocalService.Services.Interfaces;
+using BomLocalService.Utilities;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BomLocalService.Controllers;
+
+[ApiController]
+[Route("api/cache")]
+public class CacheController : ControllerBase
+{
+    private readonly ICacheService _cacheService;
+    private readonly IBomRadarService _bomRadarService;
+    private readonly ILogger<CacheController> _logger;
+
+    public CacheController(
+        ICacheService cacheService, 
+        IBomRadarService bomRadarService,
+        ILogger<CacheController> logger)
+    {
+        _cacheService = cacheService;
+        _bomRadarService = bomRadarService;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Get information about the available cache range for a location (oldest and newest cache folders).
+    /// Helps clients understand what historical data is available before requesting extended timespans.
+    /// </summary>
+    [HttpGet("{suburb}/{state}/range")]
+    public async Task<ActionResult<CacheRange>> GetCacheRange(string suburb, string state, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var validationError = ValidationHelper.ValidateLocation(suburb, state);
+            if (validationError != null)
+            {
+                return BadRequest(new { error = validationError });
+            }
+
+            var result = await _bomRadarService.GetCacheRangeAsync(suburb, state, cancellationToken);
+            
+            if (result.TotalCacheFolders == 0)
+            {
+                return NotFound(new { error = "No cached data found for this location." });
+            }
+            
+            return Ok(result);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error getting cache range for suburb: {Suburb}, state: {State}", suburb, state);
+            return StatusCode(500, new { error = "An error occurred while getting cache range", message = ex.Message });
+        }
+    }
+
+    /// <summary>
+    /// Manually trigger a cache refresh for a location. Returns status of the update operation.
+    /// </summary>
+    [HttpPost("{suburb}/{state}/refresh")]
+    public async Task<ActionResult<CacheUpdateStatus>> RefreshCache(string suburb, string state, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var validationError = ValidationHelper.ValidateLocation(suburb, state);
+            if (validationError != null)
+            {
+                return BadRequest(new { error = validationError });
+            }
+
+            var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
+            return Ok(status);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state);
+            return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message });
+        }
+    }
+
+    /// <summary>
+    /// Delete cached data for a location.
+    /// </summary>
+    [HttpDelete("{suburb}/{state}")]
+    public async Task<ActionResult> DeleteCache(string suburb, string state, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var validationError = ValidationHelper.ValidateLocation(suburb, state);
+            if (validationError != null)
+            {
+                return BadRequest(new { error = validationError });
+            }
+
+            var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
+            
+            if (deleted)
+            {
+                return Ok(new { message = $"Cache deleted for {suburb}, {state}" });
+            }
+            else
+            {
+                return NotFound(new { error = $"No cached data found for {suburb}, {state}" });
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state);
+            return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message });
+        }
+    }
+}

Controllers/RadarController.cs

@@ -10,11 +10,19 @@ namespace BomLocalService.Controllers;
 public class RadarController : ControllerBase
 {
     private readonly IBomRadarService _bomRadarService;
+    private readonly ICacheService _cacheService;
+    private readonly IConfiguration _configuration;
     private readonly ILogger<RadarController> _logger;
 
-    public RadarController(IBomRadarService bomRadarService, ILogger<RadarController> logger)
+    public RadarController(
+        IBomRadarService bomRadarService, 
+        ICacheService cacheService,
+        IConfiguration configuration,
+        ILogger<RadarController> logger)
     {
         _bomRadarService = bomRadarService;
+        _cacheService = cacheService;
+        _configuration = configuration;
         _logger = logger;
     }
 
@@ -50,10 +58,28 @@ public class RadarController : ControllerBase
             
             if (result == null || result.Frames.Count == 0)
             {
+                // Get cache status to include in 404 response
+                // Use cache service directly since this is cache management, not radar-specific
+                var cacheExpirationMinutes = (int)_configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
+                var cacheManagementCheckIntervalMinutes = _configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
+                var cacheStatus = await _cacheService.GetCacheStatusAsync(
+                    suburb, 
+                    state, 
+                    CachedDataType.Radar,
+                    cacheExpirationMinutes,
+                    cacheManagementCheckIntervalMinutes,
+                    cancellationToken);
+                
                 return NotFound(new { 
                     error = "Screenshots not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
                     retryAfter = 30, // seconds
-                    refreshEndpoint = $"/api/radar/{suburb}/{state}/refresh"
+                    refreshEndpoint = $"/api/cache/{suburb}/{state}/refresh",
+                    updateTriggered = cacheStatus.UpdateTriggered,
+                    cacheExists = cacheStatus.CacheExists,
+                    cacheIsValid = cacheStatus.CacheIsValid,
+                    cacheExpiresAt = cacheStatus.CacheExpiresAt,
+                    nextUpdateTime = cacheStatus.NextUpdateTime,
+                    message = cacheStatus.Message
                 });
             }
             
@@ -89,12 +115,18 @@ public class RadarController : ControllerBase
                 return BadRequest(new { error = validationError });
             }
             
-            if (frameIndex < 0 || frameIndex > 6)
-            {
-                return BadRequest(new { error = "Frame index must be between 0 and 6" });
-            }
+            RadarFrame? frame;
             
-            var frame = await _bomRadarService.GetCachedFrameAsync(suburb, state, frameIndex, cancellationToken);
+            // Support cacheFolder query parameter for historical data
+            var cacheFolder = Request.Query["cacheFolder"].FirstOrDefault();
+            if (!string.IsNullOrEmpty(cacheFolder))
+            {
+                frame = await _bomRadarService.GetFrameFromCacheFolderAsync(suburb, state, cacheFolder, frameIndex, cancellationToken);
+            }
+            else
+            {
+                frame = await _bomRadarService.GetCachedFrameAsync(suburb, state, frameIndex, cancellationToken);
+            }
             
             if (frame == null || !System.IO.File.Exists(frame.ImagePath))
             {
@@ -129,7 +161,7 @@ public class RadarController : ControllerBase
             
             if (result == null)
             {
-                return NotFound(new { error = "No cached data found. Use POST /api/radar/{suburb}/{state}/refresh to trigger cache update." });
+                return NotFound(new { error = "No cached data found. Use POST /api/cache/{suburb}/{state}/refresh to trigger cache update." });
             }
             
             return Ok(result);
@@ -142,10 +174,16 @@ public class RadarController : ControllerBase
     }
 
     /// <summary>
-    /// Manually trigger a cache refresh for a location. Returns status of the update operation.
+    /// Get historical radar data (frames from multiple cache folders) between specified timestamps.
+    /// Radar frames can be joined across cache folders because they represent a historical time series.
     /// </summary>
-    [HttpPost("{suburb}/{state}/refresh")]
-    public async Task<ActionResult<CacheUpdateStatus>> RefreshCache(string suburb, string state, CancellationToken cancellationToken = default)
+    [HttpGet("{suburb}/{state}/timeseries")]
+    public async Task<ActionResult<RadarTimeSeriesResponse>> GetRadarTimeSeries(
+        string suburb, 
+        string state, 
+        [FromQuery] string? startTime = null, 
+        [FromQuery] string? endTime = null,
+        CancellationToken cancellationToken = default)
     {
         try
         {
@@ -155,45 +193,45 @@ public class RadarController : ControllerBase
                 return BadRequest(new { error = validationError });
             }
 
-            var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
-            return Ok(status);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message });
-        }
-    }
-
-    /// <summary>
-    /// Delete cached data for a location.
-    /// </summary>
-    [HttpDelete("{suburb}/{state}")]
-    public async Task<ActionResult> DeleteCache(string suburb, string state, CancellationToken cancellationToken = default)
-    {
-        try
-        {
-            var validationError = ValidationHelper.ValidateLocation(suburb, state);
-            if (validationError != null)
-            {
-                return BadRequest(new { error = validationError });
-            }
-
-            var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
+            DateTime? start = null;
+            DateTime? end = null;
             
-            if (deleted)
+            if (!string.IsNullOrEmpty(startTime))
             {
-                return Ok(new { message = $"Cache deleted for {suburb}, {state}" });
+                if (!DateTime.TryParse(startTime, null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsedStart))
+                {
+                    return BadRequest(new { error = "Invalid startTime format. Use ISO 8601 format (e.g., 2025-12-07T00:00:00Z)" });
+                }
+                start = parsedStart.ToUniversalTime();
             }
-            else
+            
+            if (!string.IsNullOrEmpty(endTime))
             {
-                return NotFound(new { error = $"No cached data found for {suburb}, {state}" });
+                if (!DateTime.TryParse(endTime, null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsedEnd))
+                {
+                    return BadRequest(new { error = "Invalid endTime format. Use ISO 8601 format (e.g., 2025-12-07T12:00:00Z)" });
+                }
+                end = parsedEnd.ToUniversalTime();
             }
+            
+            if (start.HasValue && end.HasValue && start.Value > end.Value)
+            {
+                return BadRequest(new { error = "startTime must be before or equal to endTime" });
+            }
+
+            var result = await _bomRadarService.GetRadarTimeSeriesAsync(suburb, state, start, end, cancellationToken);
+            
+            if (result.CacheFolders.Count == 0)
+            {
+                return NotFound(new { error = "No historical data found for the specified time range." });
+            }
+            
+            return Ok(result);
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state);
-            return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message });
+            _logger.LogError(ex, "Error getting radar time series for suburb: {Suburb}, state: {State}", suburb, state);
+            return StatusCode(500, new { error = "An error occurred while getting radar time series", message = ex.Message });
         }
     }
 }