ElixirForge/bom-local-service
1
0
Fork 0
mirror of https://github.com/alexhopeoconnor/bom-local-service.git synced 2026-10-04 03:28:11 +10:00
Code Releases 7 Activity

Add active cache folder tracking and cache state to responses

Browse Source 
- Track active cache folders being written to prevent returning incomplete data
- Add cache state fields to RadarResponse (CacheIsValid, CacheExpiresAt, IsUpdating, NextUpdateTime)
- Detect active updates before acquiring semaphore for better performance
- Prevent duplicate cache updates when one is already in progress
- Clean up empty cache folders created but not used
This commit is contained in:
alex
2025-12-07 13:49:21 +10:00
parent 41e1f6d685
commit 18e0a7a01f
10 changed files with 238 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Models/RadarResponse.cs
+28
View File
@@ -49,5 +49,33 @@ public class RadarResponse
/// May be null if the distance cannot be parsed or if metadata is not available.
/// </summary>
public string? Distance { get; set; }
/// <summary>
/// Indicates whether the cached data is still considered valid based on its observation time
/// and the configured cache expiration period.
/// </summary>
public bool CacheIsValid { get; set; }
/// <summary>
/// The UTC date and time when the current cached data is expected to expire.
/// This is calculated based on the observation time and a configured buffer (e.g., 15.5 minutes).
/// Null if cache is not valid or metadata is not available.
/// </summary>
public DateTime? CacheExpiresAt { get; set; }
/// <summary>
/// Indicates whether a cache update is currently in progress for this location.
/// When true, clients should wait before requesting a refresh, as a new cache is being generated.
/// </summary>
public bool IsUpdating { get; set; }
/// <summary>
/// The UTC date and time when the next cache update is expected or recommended.
/// - If cache is valid: equals <see cref="CacheExpiresAt"/> (check again when cache expires).
/// - If cache is invalid and an update is in progress: estimated completion time (approximately 2 minutes from now).
/// - If cache is invalid and no update is in progress: null (client should trigger an update).
/// This may differ from <see cref="CacheExpiresAt"/> when an update is actively in progress.
/// </summary>
public DateTime? NextUpdateTime { get; set; }
}
Models/ScreenshotCropConfig.cs
+1 -1
View File
@@ -20,7 +20,7 @@ public class ScreenshotCropConfig
/// <summary>
/// Width of the crop area in pixels.
/// If null, uses the full container width minus X offset.
/// If null, uses the remaining container width from X offset to the right edge.
/// </summary>
public int? Width { get; set; }
Services/BomRadarService.cs
+112 -10
View File
@@ -2,6 +2,7 @@ using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Playwright;
using System.Collections.Concurrent;
namespace BomLocalService.Services;
@@ -13,6 +14,7 @@ public class BomRadarService : IBomRadarService, IDisposable
private readonly IScrapingService _scrapingService;
private readonly IDebugService _debugService;
private readonly double _cacheExpirationMinutes;
private readonly ConcurrentDictionary<string, string> _activeCacheFolders = new(); // locationKey -> cacheFolderPath
public BomRadarService(
ILogger<BomRadarService> logger,
@@ -32,7 +34,9 @@ public class BomRadarService : IBomRadarService, IDisposable
public async Task<RadarResponse?> GetCachedRadarAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null;
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken);
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
{
@@ -45,7 +49,12 @@ public class BomRadarService : IBomRadarService, IDisposable
return null;
}
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state);
// Determine cache state
var isValid = cachedMetadata != null && _cacheService.IsCacheValid(cachedMetadata);
var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null;
var isUpdating = _activeCacheFolders.ContainsKey(locationKey);
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating);
}
public async Task<List<RadarFrame>?> GetCachedFramesAsync(string suburb, string state, CancellationToken cancellationToken = default)
@@ -69,7 +78,29 @@ public class BomRadarService : IBomRadarService, IDisposable
public async Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var status = new CacheUpdateStatus();
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var locationKey = LocationHelper.GetLocationKey(suburb, state);
// Check if an update is already in progress
var isAlreadyUpdating = _activeCacheFolders.ContainsKey(locationKey);
var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null;
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken);
if (isAlreadyUpdating)
{
_logger.LogDebug("Cache update already in progress for {Suburb}, {State}, skipping trigger", suburb, state);
status.CacheExists = !string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath);
if (cachedMetadata != null)
{
status.CacheIsValid = _cacheService.IsCacheValid(cachedMetadata);
status.CacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
}
status.UpdateTriggered = false;
status.Message = "Cache update already in progress";
status.NextUpdateTime = status.CacheExpiresAt ?? DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes);
return status;
}
status.CacheExists = !string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath);
@@ -127,7 +158,28 @@ public class BomRadarService : IBomRadarService, IDisposable
_logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
// Check cache FIRST, before acquiring semaphore (cached requests shouldn't block)
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null;
var isUpdating = !string.IsNullOrEmpty(activeFolder);
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken);
// If there's an active update in progress, return existing cache (if available) with IsUpdating=true
if (isUpdating)
{
if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath))
{
_logger.LogInformation("Cache update in progress for {Suburb}, {State}, returning existing cached data", suburb, state);
var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
var isValid = cachedMetadata != null && _cacheService.IsCacheValid(cachedMetadata);
var cacheExpiresAt = cachedMetadata != null ? cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes) : (DateTime?)null;
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: true);
}
else
{
_logger.LogInformation("Cache update in progress for {Suburb}, {State}, but no existing cache found, waiting for update to complete", suburb, state);
// Still proceed to acquire semaphore - the update might complete while we wait
}
}
if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath) && cachedMetadata != null)
{
@@ -136,7 +188,8 @@ public class BomRadarService : IBomRadarService, IDisposable
{
_logger.LogInformation("Returning valid cached screenshots for {Suburb}, {State} (no semaphore needed)", suburb, state);
var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state);
var cacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state, isValid, cacheExpiresAt, isUpdating: false);
}
else
{
@@ -162,18 +215,52 @@ public class BomRadarService : IBomRadarService, IDisposable
IBrowserContext? context = null;
string? debugFolder = null;
string? requestId = null;
string? newCacheFolderPath = null;
try
{
// Create cache folder and track it before double-checking
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
newCacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheService.GetCacheDirectory(), suburb, state, timestamp);
Directory.CreateDirectory(newCacheFolderPath);
// Track this folder as being written to
_activeCacheFolders[locationKey] = newCacheFolderPath;
_logger.LogDebug("Tracking cache folder being written to: {Folder} for {Location}", newCacheFolderPath, locationKey);
// Double-check cache after acquiring semaphore (another request might have just created it)
var (recheckCacheFolderPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
// Exclude the folder we're about to write to
var (recheckCacheFolderPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, newCacheFolderPath, cancellationToken);
if (!string.IsNullOrEmpty(recheckCacheFolderPath) && Directory.Exists(recheckCacheFolderPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata))
{
// Remove from tracking since we're not using this folder
_activeCacheFolders.TryRemove(locationKey, out _);
// Clean up the empty folder we created since we're not using it
if (!string.IsNullOrEmpty(newCacheFolderPath) && Directory.Exists(newCacheFolderPath))
{
try
{
// Check if folder is empty (only . and ..)
var files = Directory.GetFiles(newCacheFolderPath);
if (files.Length == 0)
{
Directory.Delete(newCacheFolderPath, recursive: true);
_logger.LogDebug("Cleaned up empty cache folder that was created but not used: {Folder}", newCacheFolderPath);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to clean up empty cache folder: {Folder}", newCacheFolderPath);
}
}
_logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshots");
var recheckFrames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state);
var recheckCacheExpiresAt = recheckCachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state, cacheIsValid: true, recheckCacheExpiresAt, isUpdating: false);
}
// Need to capture new screenshot - create debug folder only now
// Create debug folder only now
requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}";
debugFolder = _debugService.CreateRequestFolder(requestId);
@@ -182,14 +269,27 @@ public class BomRadarService : IBomRadarService, IDisposable
try
{
return await _scrapingService.ScrapeRadarScreenshotAsync(
var result = await _scrapingService.ScrapeRadarScreenshotAsync(
suburb,
state,
newCacheFolderPath,
debugFolder,
page,
consoleMessages,
networkRequests,
cancellationToken);
// Remove from active tracking once complete
_activeCacheFolders.TryRemove(locationKey, out _);
_logger.LogDebug("Cache folder complete, removed from active tracking: {Folder}", newCacheFolderPath);
// Update result with cache state
result.IsUpdating = false;
result.CacheIsValid = true;
result.CacheExpiresAt = result.ObservationTime.AddMinutes(_cacheExpirationMinutes);
result.NextUpdateTime = result.CacheExpiresAt;
return result;
}
catch (Exception ex)
{
@@ -233,7 +333,9 @@ public class BomRadarService : IBomRadarService, IDisposable
public async Task<LastUpdatedInfo?> GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
// Return cached metadata if available, otherwise return null
var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var excludeFolder = _activeCacheFolders.TryGetValue(locationKey, out var activeFolder) ? activeFolder : null;
var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, excludeFolder, cancellationToken);
if (metadata != null)
{
Services/CacheService.cs
+46 -7
View File
@@ -27,6 +27,7 @@ public class CacheService : ICacheService
public async Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
string? excludeFolder = null,
CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFolderPattern(suburb, state);
@@ -47,14 +48,52 @@ public class CacheService : ICacheService
})
.ToList();
var cacheFolderPath = folders.FirstOrDefault();
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
// Find the first complete cache folder (has all 7 frames + metadata)
foreach (var folder in folders)
{
return (null, null);
if (!Directory.Exists(folder))
continue;
// Skip the folder if it's being excluded (currently being written to)
if (!string.IsNullOrEmpty(excludeFolder) && Path.GetFullPath(folder).Equals(Path.GetFullPath(excludeFolder), StringComparison.OrdinalIgnoreCase))
{
_logger.LogDebug("Skipping excluded cache folder (being written to): {Folder}", folder);
continue;
}
// Check if folder is complete: must have all 7 frames and metadata.json
var hasAllFrames = true;
for (int i = 0; i < 7; i++)
{
var framePath = FilePathHelper.GetFrameFilePath(folder, i);
if (!File.Exists(framePath))
{
hasAllFrames = false;
break;
}
}
if (!hasAllFrames)
{
_logger.LogDebug("Skipping incomplete cache folder: {Folder}", folder);
continue; // Skip incomplete folders (currently being written to)
}
// Check if metadata exists (indicates folder is complete)
var metadataPath = FilePathHelper.GetMetadataFilePath(folder);
if (!File.Exists(metadataPath))
{
_logger.LogDebug("Skipping cache folder without metadata: {Folder}", folder);
continue; // Skip folders without metadata
}
// This folder is complete
var metadata = await LoadMetadataAsync(folder, cancellationToken);
return (folder, metadata);
}
var metadata = await LoadMetadataAsync(cacheFolderPath, cancellationToken);
return (cacheFolderPath, metadata);
// No complete cache folder found
return (null, null);
}
/// <summary>
@@ -65,7 +104,7 @@ public class CacheService : ICacheService
string state,
CancellationToken cancellationToken = default)
{
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, null, cancellationToken);
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
{
@@ -115,7 +154,7 @@ public class CacheService : ICacheService
return null;
}
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, null, cancellationToken);
if (string.IsNullOrEmpty(cacheFolderPath))
{
Services/Interfaces/ICacheService.cs
+2
View File
@@ -14,11 +14,13 @@ public interface ICacheService
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="excludeFolder">Optional folder path to exclude from search (e.g., folder currently being written to)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Tuple containing the cache folder path and metadata, or (null, null) if not found</returns>
Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
string? excludeFolder = null,
CancellationToken cancellationToken = default);
/// <summary>
Services/Interfaces/IScrapingService.cs
+2
View File
@@ -16,6 +16,7 @@ public interface IScrapingService
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cacheFolderPath">The cache folder path where frames and metadata will be saved</param>
/// <param name="debugFolder">Folder path for saving debug files (screenshots, HTML, logs) if debug mode is enabled</param>
/// <param name="page">The Playwright page instance to use for scraping</param>
/// <param name="consoleMessages">List to capture console messages from the browser (for debugging)</param>
@@ -25,6 +26,7 @@ public interface IScrapingService
Task<RadarResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string cacheFolderPath,
string debugFolder,
IPage page,
List<(string type, string text, DateTime timestamp)> consoleMessages,
Services/ScrapingService.cs
+20 -10
View File
@@ -75,6 +75,7 @@ public class ScrapingService : IScrapingService
public async Task<RadarResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string cacheFolderPath,
string debugFolder,
IPage page,
List<(string type, string text, DateTime timestamp)> consoleMessages,
@@ -444,14 +445,9 @@ public class ScrapingService : IScrapingService
Height = boundingBox.Height
};
// Step 13: Create cache folder
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
var cacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheService.GetCacheDirectory(), suburb, state, timestamp);
// Step 13: Use provided cache folder (already created by BomRadarService)
Directory.CreateDirectory(cacheFolderPath);
_logger.LogInformation("Created cache folder: {Path}", cacheFolderPath);
_logger.LogInformation("Using cache folder: {Path}", cacheFolderPath);
// Step 14-20: Capture all 7 frames
var frames = new List<RadarFrame>();
@@ -529,6 +525,21 @@ public class ScrapingService : IScrapingService
await _cacheService.SaveMetadataAsync(cacheFolderPath, lastUpdatedInfo, cancellationToken);
await _cacheService.SaveFramesMetadataAsync(cacheFolderPath, frames, cancellationToken);
// Remove lock file to indicate cache folder is complete
var lockFilePath = FilePathHelper.GetCacheLockFilePath(cacheFolderPath);
try
{
if (File.Exists(lockFilePath))
{
File.Delete(lockFilePath);
_logger.LogDebug("Removed cache lock file: {Path}", lockFilePath);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to remove cache lock file: {Path}", lockFilePath);
}
// Step 22: Return response with all frames
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, lastUpdatedInfo, suburb, state);
}
@@ -550,9 +561,8 @@ public class ScrapingService : IScrapingService
var x = containerClip.X + _cropConfig.X;
var y = containerClip.Y + _cropConfig.Y;
// Calculate width (use configured or remaining width minus right offset)
var rightOffset = 200;
var width = _cropConfig.Width ?? (containerClip.Width - _cropConfig.X - rightOffset);
// Calculate width (use configured or remaining width from X to right edge)
var width = _cropConfig.Width ?? (containerClip.Width - _cropConfig.X);
// Calculate height (use configured or remaining height)
var height = _cropConfig.Height ?? (containerClip.Height - _cropConfig.Y);
Utilities/FilePathHelper.cs
+10
View File
@@ -78,5 +78,15 @@ public static class FilePathHelper
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
return $"{safeLocationKey}_*";
}
/// <summary>
/// Gets the lock file path for a cache folder.
/// This file indicates the folder is currently being written to.
/// Format: "{CacheFolderPath}/.writing"
/// </summary>
public static string GetCacheLockFilePath(string cacheFolderPath)
{
return Path.Combine(cacheFolderPath, ".writing");
}
}
Utilities/ResponseBuilder.cs
+15 -17
View File
@@ -12,7 +12,10 @@ public static class ResponseBuilder
List<RadarFrame> frames,
LastUpdatedInfo? metadata = null,
string? suburb = null,
string? state = null)
string? state = null,
bool? cacheIsValid = null,
DateTime? cacheExpiresAt = null,
bool isUpdating = false)
{
var folderInfo = new DirectoryInfo(cacheFolderPath);
var lastWriteTime = folderInfo.Exists
@@ -30,26 +33,21 @@ public static class ResponseBuilder
}
}
if (metadata == null)
{
return new RadarResponse
{
Frames = frames,
LastUpdated = lastWriteTime,
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
}
return new RadarResponse
var response = new RadarResponse
{
Frames = frames,
LastUpdated = lastWriteTime,
ObservationTime = metadata.ObservationTime,
ForecastTime = metadata.ForecastTime,
WeatherStation = metadata.WeatherStation,
Distance = metadata.Distance
ObservationTime = metadata?.ObservationTime ?? DateTime.UtcNow,
ForecastTime = metadata?.ForecastTime ?? DateTime.UtcNow,
WeatherStation = metadata?.WeatherStation,
Distance = metadata?.Distance,
CacheIsValid = cacheIsValid ?? false,
CacheExpiresAt = cacheExpiresAt,
IsUpdating = isUpdating,
NextUpdateTime = cacheExpiresAt ?? (isUpdating ? DateTime.UtcNow.AddMinutes(2) : null) // Estimate 2 min for update if in progress
};
return response;
}
/// <summary>
appsettings.json
+1 -1
View File
@@ -22,7 +22,7 @@
"DynamicContentWaitMs": 2000,
"TileRenderWaitMs": 5000,
"Crop": {
"X": 200,
"X": 250,
"Y": 0,
"Width": null,
"Height": null