diff --git a/Models/RadarResponse.cs b/Models/RadarResponse.cs index b4e05f2..00e530a 100644 --- a/Models/RadarResponse.cs +++ b/Models/RadarResponse.cs @@ -49,5 +49,33 @@ public class RadarResponse /// May be null if the distance cannot be parsed or if metadata is not available. /// public string? Distance { get; set; } + + /// + /// Indicates whether the cached data is still considered valid based on its observation time + /// and the configured cache expiration period. + /// + public bool CacheIsValid { get; set; } + + /// + /// The UTC date and time when the current cached data is expected to expire. + /// This is calculated based on the observation time and a configured buffer (e.g., 15.5 minutes). + /// Null if cache is not valid or metadata is not available. + /// + public DateTime? CacheExpiresAt { get; set; } + + /// + /// Indicates whether a cache update is currently in progress for this location. + /// When true, clients should wait before requesting a refresh, as a new cache is being generated. + /// + public bool IsUpdating { get; set; } + + /// + /// The UTC date and time when the next cache update is expected or recommended. + /// - If cache is valid: equals (check again when cache expires). + /// - If cache is invalid and an update is in progress: estimated completion time (approximately 2 minutes from now). + /// - If cache is invalid and no update is in progress: null (client should trigger an update). + /// This may differ from when an update is actively in progress. + /// + public DateTime? NextUpdateTime { get; set; } } diff --git a/Models/ScreenshotCropConfig.cs b/Models/ScreenshotCropConfig.cs index 8882bfb..20a6f02 100644 --- a/Models/ScreenshotCropConfig.cs +++ b/Models/ScreenshotCropConfig.cs @@ -20,7 +20,7 @@ public class ScreenshotCropConfig /// /// Width of the crop area in pixels. - /// If null, uses the full container width minus X offset. + /// If null, uses the remaining container width from X offset to the right edge. /// public int? Width { get; set; } diff --git a/Services/BomRadarService.cs b/Services/BomRadarService.cs index d2b51cf..72cce41 100644 --- a/Services/BomRadarService.cs +++ b/Services/BomRadarService.cs @@ -2,6 +2,7 @@ using BomLocalService.Models; using BomLocalService.Services.Interfaces; using BomLocalService.Utilities; using Microsoft.Playwright; +using System.Collections.Concurrent; namespace BomLocalService.Services; @@ -13,6 +14,7 @@ public class BomRadarService : IBomRadarService, IDisposable private readonly IScrapingService _scrapingService; private readonly IDebugService _debugService; private readonly double _cacheExpirationMinutes; + private readonly ConcurrentDictionary _activeCacheFolders = new(); // locationKey -> cacheFolderPath public BomRadarService( ILogger logger, @@ -32,7 +34,9 @@ public class BomRadarService : IBomRadarService, IDisposable public async Task GetCachedRadarAsync(string suburb, string state, CancellationToken cancellationToken = default) { - var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var locationKey = LocationHelper.GetLocationKey(suburb, state); + var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null; + var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken); if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath)) { @@ -45,7 +49,12 @@ public class BomRadarService : IBomRadarService, IDisposable return null; } - return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state); + // Determine cache state + var isValid = cachedMetadata != null && _cacheService.IsCacheValid(cachedMetadata); + var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null; + var isUpdating = _activeCacheFolders.ContainsKey(locationKey); + + return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating); } public async Task?> GetCachedFramesAsync(string suburb, string state, CancellationToken cancellationToken = default) @@ -69,7 +78,29 @@ public class BomRadarService : IBomRadarService, IDisposable public async Task TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default) { var status = new CacheUpdateStatus(); - var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var locationKey = LocationHelper.GetLocationKey(suburb, state); + + // Check if an update is already in progress + var isAlreadyUpdating = _activeCacheFolders.ContainsKey(locationKey); + var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null; + var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken); + + if (isAlreadyUpdating) + { + _logger.LogDebug("Cache update already in progress for {Suburb}, {State}, skipping trigger", suburb, state); + + status.CacheExists = !string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath); + if (cachedMetadata != null) + { + status.CacheIsValid = _cacheService.IsCacheValid(cachedMetadata); + status.CacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + } + + status.UpdateTriggered = false; + status.Message = "Cache update already in progress"; + status.NextUpdateTime = status.CacheExpiresAt ?? DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes); + return status; + } status.CacheExists = !string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath); @@ -127,7 +158,28 @@ public class BomRadarService : IBomRadarService, IDisposable _logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state); // Check cache FIRST, before acquiring semaphore (cached requests shouldn't block) - var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var locationKey = LocationHelper.GetLocationKey(suburb, state); + var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null; + var isUpdating = !string.IsNullOrEmpty(activeFolder); + var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken); + + // If there's an active update in progress, return existing cache (if available) with IsUpdating=true + if (isUpdating) + { + if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath)) + { + _logger.LogInformation("Cache update in progress for {Suburb}, {State}, returning existing cached data", suburb, state); + var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken); + var isValid = cachedMetadata != null && _cacheService.IsCacheValid(cachedMetadata); + var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null; + return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: true); + } + else + { + _logger.LogInformation("Cache update in progress for {Suburb}, {State}, but no existing cache found, waiting for update to complete", suburb, state); + // Still proceed to acquire semaphore - the update might complete while we wait + } + } if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath) && cachedMetadata != null) { @@ -136,7 +188,8 @@ public class BomRadarService : IBomRadarService, IDisposable { _logger.LogInformation("Returning valid cached screenshots for {Suburb}, {State} (no semaphore needed)", suburb, state); var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken); - return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state); + var cacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: false); } else { @@ -162,18 +215,52 @@ public class BomRadarService : IBomRadarService, IDisposable IBrowserContext? context = null; string? debugFolder = null; string? requestId = null; + string? newCacheFolderPath = null; try { + // Create cache folder and track it before double-checking + var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss"); + newCacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheService.GetCacheDirectory(), suburb, state, timestamp); + Directory.CreateDirectory(newCacheFolderPath); + + // Track this folder as being written to + _activeCacheFolders[locationKey] = newCacheFolderPath; + _logger.LogDebug("Tracking cache folder being written to: {Folder} for {Location}", newCacheFolderPath, locationKey); + // Double-check cache after acquiring semaphore (another request might have just created it) - var (recheckCacheFolderPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + // Exclude the folder we're about to write to + var (recheckCacheFolderPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, newCacheFolderPath, cancellationToken); if (!string.IsNullOrEmpty(recheckCacheFolderPath) && Directory.Exists(recheckCacheFolderPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata)) { + // Remove from tracking since we're not using this folder + _activeCacheFolders.TryRemove(locationKey, out _); + + // Clean up the empty folder we created since we're not using it + if (!string.IsNullOrEmpty(newCacheFolderPath) && Directory.Exists(newCacheFolderPath)) + { + try + { + // Check if folder is empty (only . and ..) + var files = Directory.GetFiles(newCacheFolderPath); + if (files.Length == 0) + { + Directory.Delete(newCacheFolderPath, recursive: true); + _logger.LogDebug("Cleaned up empty cache folder that was created but not used: {Folder}", newCacheFolderPath); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to clean up empty cache folder: {Folder}", newCacheFolderPath); + } + } + _logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshots"); var recheckFrames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken); - return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state); + var recheckCacheExpiresAt = recheckCachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state, cacheIsValid: true, recheckCacheExpiresAt, isUpdating: false); } - - // Need to capture new screenshot - create debug folder only now + + // Create debug folder only now requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}"; debugFolder = _debugService.CreateRequestFolder(requestId); @@ -182,14 +269,27 @@ public class BomRadarService : IBomRadarService, IDisposable try { - return await _scrapingService.ScrapeRadarScreenshotAsync( + var result = await _scrapingService.ScrapeRadarScreenshotAsync( suburb, state, + newCacheFolderPath, debugFolder, page, consoleMessages, networkRequests, cancellationToken); + + // Remove from active tracking once complete + _activeCacheFolders.TryRemove(locationKey, out _); + _logger.LogDebug("Cache folder complete, removed from active tracking: {Folder}", newCacheFolderPath); + + // Update result with cache state + result.IsUpdating = false; + result.CacheIsValid = true; + result.CacheExpiresAt = result.ObservationTime.AddMinutes(_cacheExpirationMinutes); + result.NextUpdateTime = result.CacheExpiresAt; + + return result; } catch (Exception ex) { @@ -233,7 +333,9 @@ public class BomRadarService : IBomRadarService, IDisposable public async Task GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default) { // Return cached metadata if available, otherwise return null - var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var locationKey = LocationHelper.GetLocationKey(suburb, state); + var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null; + var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken); if (metadata != null) { diff --git a/Services/CacheService.cs b/Services/CacheService.cs index b9db08d..eb8f2a2 100644 --- a/Services/CacheService.cs +++ b/Services/CacheService.cs @@ -27,6 +27,7 @@ public class CacheService : ICacheService public async Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync( string suburb, string state, + string? excludeFolder = null, CancellationToken cancellationToken = default) { var pattern = FilePathHelper.GetCacheFolderPattern(suburb, state); @@ -47,14 +48,52 @@ public class CacheService : ICacheService }) .ToList(); - var cacheFolderPath = folders.FirstOrDefault(); - if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath)) + // Find the first complete cache folder (has all 7 frames + metadata) + foreach (var folder in folders) { - return (null, null); + if (!Directory.Exists(folder)) + continue; + + // Skip the folder if it's being excluded (currently being written to) + if (!string.IsNullOrEmpty(excludeFolder) && Path.GetFullPath(folder).Equals(Path.GetFullPath(excludeFolder), StringComparison.OrdinalIgnoreCase)) + { + _logger.LogDebug("Skipping excluded cache folder (being written to): {Folder}", folder); + continue; + } + + // Check if folder is complete: must have all 7 frames and metadata.json + var hasAllFrames = true; + for (int i = 0; i < 7; i++) + { + var framePath = FilePathHelper.GetFrameFilePath(folder, i); + if (!File.Exists(framePath)) + { + hasAllFrames = false; + break; + } + } + + if (!hasAllFrames) + { + _logger.LogDebug("Skipping incomplete cache folder: {Folder}", folder); + continue; // Skip incomplete folders (currently being written to) + } + + // Check if metadata exists (indicates folder is complete) + var metadataPath = FilePathHelper.GetMetadataFilePath(folder); + if (!File.Exists(metadataPath)) + { + _logger.LogDebug("Skipping cache folder without metadata: {Folder}", folder); + continue; // Skip folders without metadata + } + + // This folder is complete + var metadata = await LoadMetadataAsync(folder, cancellationToken); + return (folder, metadata); } - var metadata = await LoadMetadataAsync(cacheFolderPath, cancellationToken); - return (cacheFolderPath, metadata); + // No complete cache folder found + return (null, null); } /// @@ -65,7 +104,7 @@ public class CacheService : ICacheService string state, CancellationToken cancellationToken = default) { - var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, null, cancellationToken); if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath)) { @@ -115,7 +154,7 @@ public class CacheService : ICacheService return null; } - var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, null, cancellationToken); if (string.IsNullOrEmpty(cacheFolderPath)) { diff --git a/Services/Interfaces/ICacheService.cs b/Services/Interfaces/ICacheService.cs index 6a119bb..9737d67 100644 --- a/Services/Interfaces/ICacheService.cs +++ b/Services/Interfaces/ICacheService.cs @@ -14,11 +14,13 @@ public interface ICacheService /// /// The suburb name (e.g., "Pomona", "Brisbane") /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Optional folder path to exclude from search (e.g., folder currently being written to) /// Cancellation token to cancel the operation /// Tuple containing the cache folder path and metadata, or (null, null) if not found Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync( string suburb, string state, + string? excludeFolder = null, CancellationToken cancellationToken = default); /// diff --git a/Services/Interfaces/IScrapingService.cs b/Services/Interfaces/IScrapingService.cs index a027b08..c9ac07f 100644 --- a/Services/Interfaces/IScrapingService.cs +++ b/Services/Interfaces/IScrapingService.cs @@ -16,6 +16,7 @@ public interface IScrapingService /// /// The suburb name (e.g., "Pomona", "Brisbane") /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// The cache folder path where frames and metadata will be saved /// Folder path for saving debug files (screenshots, HTML, logs) if debug mode is enabled /// The Playwright page instance to use for scraping /// List to capture console messages from the browser (for debugging) @@ -25,6 +26,7 @@ public interface IScrapingService Task ScrapeRadarScreenshotAsync( string suburb, string state, + string cacheFolderPath, string debugFolder, IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, diff --git a/Services/ScrapingService.cs b/Services/ScrapingService.cs index c3e582f..934d15c 100644 --- a/Services/ScrapingService.cs +++ b/Services/ScrapingService.cs @@ -75,6 +75,7 @@ public class ScrapingService : IScrapingService public async Task ScrapeRadarScreenshotAsync( string suburb, string state, + string cacheFolderPath, string debugFolder, IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, @@ -444,14 +445,9 @@ public class ScrapingService : IScrapingService Height = boundingBox.Height }; - // Step 13: Create cache folder - var locationKey = LocationHelper.GetLocationKey(suburb, state); - var safeLocationKey = LocationHelper.SanitizeFileName(locationKey); - var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss"); - var cacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheService.GetCacheDirectory(), suburb, state, timestamp); + // Step 13: Use provided cache folder (already created by BomRadarService) Directory.CreateDirectory(cacheFolderPath); - - _logger.LogInformation("Created cache folder: {Path}", cacheFolderPath); + _logger.LogInformation("Using cache folder: {Path}", cacheFolderPath); // Step 14-20: Capture all 7 frames var frames = new List(); @@ -528,6 +524,21 @@ public class ScrapingService : IScrapingService // Step 21: Save metadata and frame information await _cacheService.SaveMetadataAsync(cacheFolderPath, lastUpdatedInfo, cancellationToken); await _cacheService.SaveFramesMetadataAsync(cacheFolderPath, frames, cancellationToken); + + // Remove lock file to indicate cache folder is complete + var lockFilePath = FilePathHelper.GetCacheLockFilePath(cacheFolderPath); + try + { + if (File.Exists(lockFilePath)) + { + File.Delete(lockFilePath); + _logger.LogDebug("Removed cache lock file: {Path}", lockFilePath); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to remove cache lock file: {Path}", lockFilePath); + } // Step 22: Return response with all frames return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, lastUpdatedInfo, suburb, state); @@ -550,9 +561,8 @@ public class ScrapingService : IScrapingService var x = containerClip.X + _cropConfig.X; var y = containerClip.Y + _cropConfig.Y; - // Calculate width (use configured or remaining width minus right offset) - var rightOffset = 200; - var width = _cropConfig.Width ?? (containerClip.Width - _cropConfig.X - rightOffset); + // Calculate width (use configured or remaining width from X to right edge) + var width = _cropConfig.Width ?? (containerClip.Width - _cropConfig.X); // Calculate height (use configured or remaining height) var height = _cropConfig.Height ?? (containerClip.Height - _cropConfig.Y); diff --git a/Utilities/FilePathHelper.cs b/Utilities/FilePathHelper.cs index 11a6991..e271416 100644 --- a/Utilities/FilePathHelper.cs +++ b/Utilities/FilePathHelper.cs @@ -78,5 +78,15 @@ public static class FilePathHelper var safeLocationKey = LocationHelper.SanitizeFileName(locationKey); return $"{safeLocationKey}_*"; } + + /// + /// Gets the lock file path for a cache folder. + /// This file indicates the folder is currently being written to. + /// Format: "{CacheFolderPath}/.writing" + /// + public static string GetCacheLockFilePath(string cacheFolderPath) + { + return Path.Combine(cacheFolderPath, ".writing"); + } } diff --git a/Utilities/ResponseBuilder.cs b/Utilities/ResponseBuilder.cs index 704b69a..7a78bac 100644 --- a/Utilities/ResponseBuilder.cs +++ b/Utilities/ResponseBuilder.cs @@ -12,7 +12,10 @@ public static class ResponseBuilder List frames, LastUpdatedInfo? metadata = null, string? suburb = null, - string? state = null) + string? state = null, + bool? cacheIsValid = null, + DateTime? cacheExpiresAt = null, + bool isUpdating = false) { var folderInfo = new DirectoryInfo(cacheFolderPath); var lastWriteTime = folderInfo.Exists @@ -30,26 +33,21 @@ public static class ResponseBuilder } } - if (metadata == null) - { - return new RadarResponse - { - Frames = frames, - LastUpdated = lastWriteTime, - ObservationTime = DateTime.UtcNow, - ForecastTime = DateTime.UtcNow - }; - } - - return new RadarResponse + var response = new RadarResponse { Frames = frames, LastUpdated = lastWriteTime, - ObservationTime = metadata.ObservationTime, - ForecastTime = metadata.ForecastTime, - WeatherStation = metadata.WeatherStation, - Distance = metadata.Distance + ObservationTime = metadata?.ObservationTime ?? DateTime.UtcNow, + ForecastTime = metadata?.ForecastTime ?? DateTime.UtcNow, + WeatherStation = metadata?.WeatherStation, + Distance = metadata?.Distance, + CacheIsValid = cacheIsValid ?? false, + CacheExpiresAt = cacheExpiresAt, + IsUpdating = isUpdating, + NextUpdateTime = cacheExpiresAt ?? (isUpdating ? DateTime.UtcNow.AddMinutes(2) : null) // Estimate 2 min for update if in progress }; + + return response; } /// diff --git a/appsettings.json b/appsettings.json index 2721cf1..82708fe 100644 --- a/appsettings.json +++ b/appsettings.json @@ -22,7 +22,7 @@ "DynamicContentWaitMs": 2000, "TileRenderWaitMs": 5000, "Crop": { - "X": 200, + "X": 250, "Y": 0, "Width": null, "Height": null