ElixirForge/bom-local-service
1
0
Fork 0
mirror of https://github.com/alexhopeoconnor/bom-local-service.git synced 2026-10-03 21:21:59 +10:00
Code Releases 7 Activity

Implement multi-frame radar capture with configurable cropping

Browse Source 
- Capture all 7 radar frames (0-6) instead of single screenshot
- Store frames in folder-based cache structure with metadata
- Extract and store minutesAgo for each frame from page display
- Add configurable screenshot cropping (default: 200px left/right)
- Fix step numbering to use whole numbers (1-22)
- Add scrubber position verification before frame capture
- Handle modal overlay blocking step forward button clicks
- Only create debug folders when scraping is actually needed
- Update API endpoints to return JSON with frame URLs
- Add separate endpoints for individual frame images
This commit is contained in:
alex
2025-12-07 12:51:44 +10:00
parent f6366bf346
commit 41e1f6d685
16 changed files with 950 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Controllers/RadarController.cs
+52 -15
View File
@@ -19,10 +19,10 @@ public class RadarController : ControllerBase
}
/// <summary>
/// Get radar screenshot for a location. Returns cached screenshot if available, or triggers background update.
/// Get radar data for a location. Returns JSON with all frames and their URLs. Triggers background update if cache is stale.
/// </summary>
[HttpGet("{suburb}/{state}")]
public async Task<ActionResult> GetScreenshot(string suburb, string state, CancellationToken cancellationToken = default)
public async Task<ActionResult<RadarResponse>> GetRadar(string suburb, string state, CancellationToken cancellationToken = default)
{
try
{
@@ -45,32 +45,69 @@ public class RadarController : ControllerBase
}
});
// Return cached screenshot if it exists (no waiting)
var result = await _bomRadarService.GetCachedScreenshotAsync(suburb, state, cancellationToken);
// Return cached radar data if available (no waiting)
var result = await _bomRadarService.GetCachedRadarAsync(suburb, state, cancellationToken);
if (result == null)
if (result == null || result.Frames.Count == 0)
{
return NotFound(new {
error = "Screenshot not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
error = "Screenshots not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
retryAfter = 30, // seconds
refreshEndpoint = $"/api/radar/{suburb}/{state}/refresh"
});
}
// Return the image file
if (System.IO.File.Exists(result.ImagePath))
// Generate URLs for frames if not already set
if (result.Frames.Any(f => string.IsNullOrEmpty(f.ImageUrl)))
{
var imageBytes = await System.IO.File.ReadAllBytesAsync(result.ImagePath, cancellationToken);
var contentType = "image/png";
return File(imageBytes, contentType, Path.GetFileName(result.ImagePath));
foreach (var frame in result.Frames)
{
frame.ImageUrl = $"/api/radar/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/frame/{frame.FrameIndex}";
}
}
return NotFound(new { error = "Screenshot file not found" });
return Ok(result);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting cached radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while getting the radar screenshot", message = ex.Message });
_logger.LogError(ex, "Error getting cached radar screenshots for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while getting the radar screenshots", message = ex.Message });
}
}
/// <summary>
/// Get a specific frame image for a location.
/// </summary>
[HttpGet("{suburb}/{state}/frame/{frameIndex}")]
public async Task<ActionResult> GetFrame(string suburb, string state, int frameIndex, CancellationToken cancellationToken = default)
{
try
{
var validationError = ValidationHelper.ValidateLocation(suburb, state);
if (validationError != null)
{
return BadRequest(new { error = validationError });
}
if (frameIndex < 0 || frameIndex > 6)
{
return BadRequest(new { error = "Frame index must be between 0 and 6" });
}
var frame = await _bomRadarService.GetCachedFrameAsync(suburb, state, frameIndex, cancellationToken);
if (frame == null || !System.IO.File.Exists(frame.ImagePath))
{
return NotFound(new { error = $"Frame {frameIndex} not found for {suburb}, {state}" });
}
var imageBytes = await System.IO.File.ReadAllBytesAsync(frame.ImagePath, cancellationToken);
return File(imageBytes, "image/png", $"frame_{frameIndex}.png");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting frame {FrameIndex} for suburb: {Suburb}, state: {State}", frameIndex, suburb, state);
return StatusCode(500, new { error = "An error occurred while getting the frame", message = ex.Message });
}
}
Models/FrameMetadata.cs
+18
View File
@@ -0,0 +1,18 @@
namespace BomLocalService.Models;
/// <summary>
/// Metadata for a single radar frame, stored alongside the frame image.
/// </summary>
public class FrameMetadata
{
/// <summary>
/// Frame index (0-6).
/// </summary>
public int FrameIndex { get; set; }
/// <summary>
/// Number of minutes ago this frame represents (40, 35, 30, 25, 20, 15, 10).
/// </summary>
public int MinutesAgo { get; set; }
}
Models/RadarFrame.cs
+33
View File
@@ -0,0 +1,33 @@
namespace BomLocalService.Models;
/// <summary>
/// Represents a single frame from the radar slideshow (0-6).
/// </summary>
public class RadarFrame
{
/// <summary>
/// Frame index (0-6), where 0 is the oldest frame (40 minutes ago) and 6 is the newest (10 minutes ago).
/// </summary>
public int FrameIndex { get; set; }
/// <summary>
/// Full file system path to the frame image file (server-side only).
/// Format: "{CacheDirectory}/{LocationKey}_{Timestamp}/frame_{FrameIndex}.png"
/// Example: "/app/cache/Pomona_QLD_20251207_000906/frame_0.png"
/// </summary>
public string ImagePath { get; set; } = string.Empty;
/// <summary>
/// URL endpoint to retrieve this frame image.
/// Format: "/api/radar/{Suburb}/{State}/frame/{FrameIndex}"
/// Example: "/api/radar/Pomona/QLD/frame/0"
/// </summary>
public string ImageUrl { get; set; } = string.Empty;
/// <summary>
/// Number of minutes ago this frame represents (40, 35, 30, 25, 20, 15, 10).
/// Frame 0 = 40 minutes ago, Frame 6 = 10 minutes ago.
/// </summary>
public int MinutesAgo { get; set; }
}
Models/RadarScreenshotResponse.cs → Models/RadarResponse.cs
+5 -7
View File
@@ -1,18 +1,16 @@
namespace BomLocalService.Models;
/// <summary>
/// Response model containing the radar screenshot image path and associated metadata.
/// Response model containing all radar screenshot frames and associated metadata.
/// This is the primary response returned when requesting a radar screenshot for a location.
/// </summary>
public class RadarScreenshotResponse
public class RadarResponse
{
/// <summary>
/// The full file system path to the cached PNG screenshot image file.
/// This is the path on the server where the screenshot is stored.
/// Format: "{CacheDirectory}/{Suburb}_{State}_{Timestamp}.png"
/// Example: "/app/cache/Pomona_QLD_20251207_000906.png"
/// List of all captured frames (typically 7 frames: 0-6).
/// Frame 0 is oldest (40 minutes ago), Frame 6 is newest (10 minutes ago).
/// </summary>
public string ImagePath { get; set; } = string.Empty;
public List<RadarFrame> Frames { get; set; } = new();
/// <summary>
/// The UTC timestamp when the screenshot file was last written/modified on disk.
Models/ScreenshotCropConfig.cs
+33
View File
@@ -0,0 +1,33 @@
namespace BomLocalService.Models;
/// <summary>
/// Configuration for cropping screenshots to avoid map controls and overlays.
/// All values are relative to the map container's bounding box.
/// </summary>
public class ScreenshotCropConfig
{
/// <summary>
/// X offset in pixels from the left edge of the map container.
/// Default: 0 (start at container's left edge).
/// </summary>
public int X { get; set; } = 0;
/// <summary>
/// Y offset in pixels from the top edge of the map container.
/// Default: 0 (start at container's top edge).
/// </summary>
public int Y { get; set; } = 0;
/// <summary>
/// Width of the crop area in pixels.
/// If null, uses the full container width minus X offset.
/// </summary>
public int? Width { get; set; }
/// <summary>
/// Height of the crop area in pixels.
/// If null, uses the full container height minus Y offset.
/// </summary>
public int? Height { get; set; }
}
Services/BomRadarService.cs
+70 -25
View File
@@ -30,24 +30,48 @@ public class BomRadarService : IBomRadarService, IDisposable
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
}
public async Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
public async Task<RadarResponse?> GetCachedRadarAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (string.IsNullOrEmpty(cachedPath) || !File.Exists(cachedPath))
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
{
return null;
}
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
if (frames == null || frames.Count == 0)
{
return null;
}
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state);
}
public async Task<List<RadarFrame>?> GetCachedFramesAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
// Generate URLs for frames
foreach (var frame in frames)
{
frame.ImageUrl = $"/api/radar/{Uri.EscapeDataString(suburb)}/{Uri.EscapeDataString(state)}/frame/{frame.FrameIndex}";
}
return frames.Count > 0 ? frames : null;
}
public async Task<RadarFrame?> GetCachedFrameAsync(string suburb, string state, int frameIndex, CancellationToken cancellationToken = default)
{
return await _cacheService.GetCachedFrameAsync(suburb, state, frameIndex, cancellationToken);
}
public async Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var status = new CacheUpdateStatus();
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
status.CacheExists = !string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath);
status.CacheExists = !string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath);
if (cachedMetadata != null)
{
@@ -98,58 +122,61 @@ public class BomRadarService : IBomRadarService, IDisposable
return status;
}
private async Task<RadarScreenshotResponse> FetchAndCacheScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
private async Task<RadarResponse> FetchAndCacheScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
// Check cache FIRST, before acquiring semaphore (cached requests shouldn't block)
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
var (cacheFolderPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath) && cachedMetadata != null)
if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath) && cachedMetadata != null)
{
var isValid = _cacheService.IsCacheValid(cachedMetadata);
if (isValid)
{
_logger.LogInformation("Returning valid cached screenshot for {Suburb}, {State} (no semaphore needed)", suburb, state);
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
_logger.LogInformation("Returning valid cached screenshots for {Suburb}, {State} (no semaphore needed)", suburb, state);
var frames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, cachedMetadata, suburb, state);
}
else
{
var nextUpdate = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
var timeUntilExpiry = nextUpdate - DateTime.UtcNow;
_logger.LogInformation("Cached screenshot exists but is stale (observation time: {ObservationTime}, expired {TimeAgo} ago), fetching new one",
_logger.LogInformation("Cached screenshots exist but are stale (observation time: {ObservationTime}, expired {TimeAgo} ago), fetching new ones",
cachedMetadata.ObservationTime, -timeUntilExpiry);
}
}
else if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath))
else if (!string.IsNullOrEmpty(cacheFolderPath) && Directory.Exists(cacheFolderPath))
{
_logger.LogWarning("Cached screenshot found but no metadata file exists, fetching new one");
_logger.LogWarning("Cached folder found but no metadata file exists, fetching new screenshots");
}
else
{
_logger.LogInformation("No cached screenshot found for {Suburb}, {State}, fetching new one", suburb, state);
_logger.LogInformation("No cached screenshots found for {Suburb}, {State}, fetching new ones", suburb, state);
}
// Only acquire semaphore if we need to fetch new screenshot
var semaphore = _browserService.GetSemaphore();
await semaphore.WaitAsync(cancellationToken);
// Generate request ID for debugging
var requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}";
var debugFolder = _debugService.CreateRequestFolder(requestId);
IBrowserContext? context = null;
string? debugFolder = null;
string? requestId = null;
try
{
// Double-check cache after acquiring semaphore (another request might have just created it)
var (recheckCachedPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (!string.IsNullOrEmpty(recheckCachedPath) && File.Exists(recheckCachedPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata))
var (recheckCacheFolderPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (!string.IsNullOrEmpty(recheckCacheFolderPath) && Directory.Exists(recheckCacheFolderPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata))
{
_logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshot");
return ResponseBuilder.CreateRadarScreenshotResponse(recheckCachedPath, recheckCachedMetadata);
_logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshots");
var recheckFrames = await _cacheService.GetCachedFramesAsync(suburb, state, cancellationToken);
return ResponseBuilder.CreateRadarResponse(recheckCacheFolderPath, recheckFrames, recheckCachedMetadata, suburb, state);
}
// Need to capture new screenshot
// Need to capture new screenshot - create debug folder only now
requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}";
debugFolder = _debugService.CreateRequestFolder(requestId);
context = await _browserService.CreateContextAsync();
var (page, consoleMessages, networkRequests) = await _browserService.CreatePageWithDebugAsync(context, requestId);
@@ -176,11 +203,29 @@ public class BomRadarService : IBomRadarService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting radar screenshot for suburb: {Suburb}, state: {State} (RequestId: {RequestId})", suburb, state, requestId);
_logger.LogError(ex, "Error getting radar screenshot for suburb: {Suburb}, state: {State} (RequestId: {RequestId})", suburb, state, requestId ?? "unknown");
throw;
}
finally
{
// Clean up empty debug folder if we returned early without scraping
if (!string.IsNullOrEmpty(debugFolder) && Directory.Exists(debugFolder))
{
try
{
var files = Directory.GetFiles(debugFolder, "*", SearchOption.AllDirectories);
if (files.Length == 0)
{
Directory.Delete(debugFolder, recursive: true);
_logger.LogDebug("Removed empty debug folder: {DebugFolder}", debugFolder);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to clean up empty debug folder: {DebugFolder}", debugFolder);
}
}
semaphore.Release();
}
}
Services/CacheCleanupService.cs
+37 -22
View File
@@ -52,54 +52,47 @@ public class CacheCleanupService : BackgroundService
try
{
// Get all PNG files in cache directory
var pngFiles = Directory.GetFiles(_cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
// Get all cache folders
var folders = Directory.GetDirectories(_cacheDirectory);
foreach (var file in pngFiles)
foreach (var folder in folders)
{
if (cancellationToken.IsCancellationRequested)
break;
try
{
var fileInfo = new FileInfo(file);
var fileTime = fileInfo.LastWriteTimeUtc;
var folderInfo = new DirectoryInfo(folder);
var folderTime = folderInfo.LastWriteTimeUtc;
// Delete if file is older than retention period
if (fileTime < cutoffTime)
// Delete if folder is older than retention period
if (folderTime < cutoffTime)
{
var fileSize = fileInfo.Length;
File.Delete(file);
var folderSize = GetFolderSize(folder);
Directory.Delete(folder, recursive: true);
deletedCount++;
totalSize += fileSize;
totalSize += folderSize;
// Also delete associated metadata file if it exists
var metadataFile = FilePathHelper.GetMetadataFilePath(file);
if (File.Exists(metadataFile))
{
File.Delete(metadataFile);
}
_logger.LogDebug("Deleted old cache file: {File} (age: {Age})",
Path.GetFileName(file), DateTime.UtcNow - fileTime);
_logger.LogDebug("Deleted old cache folder: {Folder} (age: {Age})",
Path.GetFileName(folder), DateTime.UtcNow - folderTime);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete cache file: {File}", file);
_logger.LogWarning(ex, "Failed to delete cache folder: {Folder}", folder);
}
}
if (deletedCount > 0)
{
var sizeInMB = totalSize / (1024.0 * 1024.0);
_logger.LogInformation("Cache cleanup completed. Deleted {Count} files ({Size:F2} MB)",
_logger.LogInformation("Cache cleanup completed. Deleted {Count} folders ({Size:F2} MB)",
deletedCount, sizeInMB);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during cache file enumeration");
_logger.LogError(ex, "Error during cache folder enumeration");
}
// Also clean up old debug directories
@@ -137,5 +130,27 @@ public class CacheCleanupService : BackgroundService
return Task.CompletedTask;
}
/// <summary>
/// Calculates the total size of a folder and all its contents
/// </summary>
private long GetFolderSize(string folderPath)
{
long size = 0;
try
{
var files = Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories);
foreach (var file in files)
{
var fileInfo = new FileInfo(file);
size += fileInfo.Length;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error calculating folder size for: {Folder}", folderPath);
}
return size;
}
}
Services/CacheManagementService.cs
+6 -4
View File
@@ -174,16 +174,18 @@ public class CacheManagementService : BackgroundService
try
{
// Get locations from existing cache files
// Get locations from existing cache folders
var cacheDirectory = FilePathHelper.GetCacheDirectory(_configuration);
if (Directory.Exists(cacheDirectory))
{
var pngFiles = Directory.GetFiles(cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
// Get all folders (not files)
var folders = Directory.GetDirectories(cacheDirectory);
var seen = new HashSet<string>();
foreach (var file in pngFiles)
foreach (var folder in folders)
{
var location = LocationHelper.ParseLocationFromFilename(file);
var folderName = Path.GetFileName(folder);
var location = LocationHelper.ParseLocationFromFilename(folderName);
if (location.HasValue)
{
var locationKey = LocationHelper.GetLocationKey(location.Value.suburb, location.Value.state);
Services/CacheService.cs
+171 -48
View File
@@ -22,67 +22,202 @@ public class CacheService : ICacheService
}
/// <summary>
/// Gets the cached screenshot path and metadata for a location
/// Gets the cached screenshot folder path and metadata for a location
/// </summary>
public async Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
public async Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
_logger.LogDebug("Looking for cached files matching pattern: {Pattern} in directory: {Directory}", pattern, _cacheDirectory);
var pattern = FilePathHelper.GetCacheFolderPattern(suburb, state);
_logger.LogDebug("Looking for cached folders matching pattern: {Pattern} in directory: {Directory}", pattern, _cacheDirectory);
var files = Directory.GetFiles(_cacheDirectory, pattern)
var folders = Directory.GetDirectories(_cacheDirectory, pattern)
.OrderByDescending(f =>
{
// Try to extract timestamp from filename first
var timestamp = LocationHelper.ParseTimestampFromFilename(f);
// Try to extract timestamp from folder name
var folderName = Path.GetFileName(f);
var timestamp = LocationHelper.ParseTimestampFromFilename(folderName);
if (timestamp.HasValue)
{
return timestamp.Value;
}
// Fallback to file write time
return File.GetLastWriteTime(f);
// Fallback to folder creation time
return Directory.GetCreationTime(f);
})
.ToList();
var screenshotPath = files.FirstOrDefault();
if (string.IsNullOrEmpty(screenshotPath) || !File.Exists(screenshotPath))
var cacheFolderPath = folders.FirstOrDefault();
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
{
return (null, null);
}
var metadata = await LoadMetadataAsync(screenshotPath, cancellationToken);
return (screenshotPath, metadata);
var metadata = await LoadMetadataAsync(cacheFolderPath, cancellationToken);
return (cacheFolderPath, metadata);
}
/// <summary>
/// Gets all cached frames for a location
/// </summary>
public async Task<List<RadarFrame>> GetCachedFramesAsync(
string suburb,
string state,
CancellationToken cancellationToken = default)
{
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (string.IsNullOrEmpty(cacheFolderPath) || !Directory.Exists(cacheFolderPath))
{
return new List<RadarFrame>();
}
var frames = new List<RadarFrame>();
// Load frame metadata from frames.json if available
var framesMetadata = await LoadFramesMetadataAsync(cacheFolderPath, cancellationToken);
var metadataDict = framesMetadata.ToDictionary(f => f.FrameIndex, f => f.MinutesAgo);
// Load frames from folder (frame_0.png through frame_6.png)
for (int i = 0; i < 7; i++)
{
var framePath = FilePathHelper.GetFrameFilePath(cacheFolderPath, i);
if (File.Exists(framePath))
{
// Use stored minutesAgo if available, otherwise default
var minutesAgo = metadataDict.ContainsKey(i)
? metadataDict[i]
: 40 - (i * 5);
frames.Add(new RadarFrame
{
FrameIndex = i,
ImagePath = framePath,
MinutesAgo = minutesAgo
});
}
}
return frames;
}
/// <summary>
/// Gets a specific cached frame for a location
/// </summary>
public async Task<RadarFrame?> GetCachedFrameAsync(
string suburb,
string state,
int frameIndex,
CancellationToken cancellationToken = default)
{
if (frameIndex < 0 || frameIndex > 6)
{
return null;
}
var (cacheFolderPath, _) = await GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (string.IsNullOrEmpty(cacheFolderPath))
{
return null;
}
var framePath = FilePathHelper.GetFrameFilePath(cacheFolderPath, frameIndex);
if (!File.Exists(framePath))
{
return null;
}
// Load frame metadata to get accurate minutesAgo
var framesMetadata = await LoadFramesMetadataAsync(cacheFolderPath, cancellationToken);
var frameMetadata = framesMetadata.FirstOrDefault(f => f.FrameIndex == frameIndex);
var minutesAgo = frameMetadata != null
? frameMetadata.MinutesAgo
: 40 - (frameIndex * 5);
return new RadarFrame
{
FrameIndex = frameIndex,
ImagePath = framePath,
MinutesAgo = minutesAgo
};
}
/// <summary>
/// Saves metadata alongside a screenshot
/// Saves metadata in a cache folder
/// </summary>
public async Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default)
public async Task SaveMetadataAsync(string cacheFolderPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default)
{
try
{
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
var metadataPath = FilePathHelper.GetMetadataFilePath(cacheFolderPath);
var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions { WriteIndented = true });
// Use CancellationToken.None for file writes to avoid cancellation issues
// The screenshot is already saved, metadata save is best-effort
// The frames are already saved, metadata save is best-effort
await File.WriteAllTextAsync(metadataPath, json, CancellationToken.None);
_logger.LogDebug("Saved metadata to: {Path}", metadataPath);
}
catch (Exception ex)
{
// Log but don't fail - screenshot is already saved
_logger.LogWarning(ex, "Failed to save metadata for screenshot: {Path}", screenshotPath);
// Log but don't fail - frames are already saved
_logger.LogWarning(ex, "Failed to save metadata for folder: {Path}", cacheFolderPath);
}
}
/// <summary>
/// Saves frame metadata to frames.json
/// </summary>
public async Task SaveFramesMetadataAsync(string cacheFolderPath, List<RadarFrame> frames, CancellationToken cancellationToken = default)
{
try
{
var framesMetadata = frames.Select(f => new FrameMetadata
{
FrameIndex = f.FrameIndex,
MinutesAgo = f.MinutesAgo
}).ToList();
var framesPath = Path.Combine(cacheFolderPath, "frames.json");
var json = JsonSerializer.Serialize(framesMetadata, new JsonSerializerOptions { WriteIndented = true });
await File.WriteAllTextAsync(framesPath, json, CancellationToken.None);
_logger.LogDebug("Saved frames metadata to: {Path}", framesPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to save frames metadata for folder: {Path}", cacheFolderPath);
}
}
/// <summary>
/// Loads frame metadata from frames.json
/// </summary>
private async Task<List<FrameMetadata>> LoadFramesMetadataAsync(string cacheFolderPath, CancellationToken cancellationToken = default)
{
var framesPath = Path.Combine(cacheFolderPath, "frames.json");
if (!File.Exists(framesPath))
{
return new List<FrameMetadata>();
}
try
{
var json = await File.ReadAllTextAsync(framesPath, cancellationToken);
var framesMetadata = JsonSerializer.Deserialize<List<FrameMetadata>>(json);
return framesMetadata ?? new List<FrameMetadata>();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load frames metadata from: {Path}", framesPath);
return new List<FrameMetadata>();
}
}
/// <summary>
/// Loads metadata for a screenshot
/// Loads metadata from a cache folder
/// </summary>
public async Task<LastUpdatedInfo?> LoadMetadataAsync(string screenshotPath, CancellationToken cancellationToken = default)
private async Task<LastUpdatedInfo?> LoadMetadataAsync(string cacheFolderPath, CancellationToken cancellationToken = default)
{
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
var metadataPath = FilePathHelper.GetMetadataFilePath(cacheFolderPath);
if (!File.Exists(metadataPath))
{
return null;
@@ -117,59 +252,47 @@ public class CacheService : ICacheService
}
/// <summary>
/// Gets the path to the cached screenshot for a location (simple version, ordered by creation time)
/// Gets the path to the cached screenshot for a location (returns first frame path for backward compatibility)
/// </summary>
public Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
public async Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
var files = Directory.GetFiles(_cacheDirectory, pattern)
.OrderByDescending(f => File.GetCreationTime(f))
.ToList();
return Task.FromResult(files.FirstOrDefault() ?? string.Empty);
var frames = await GetCachedFramesAsync(suburb, state, cancellationToken);
return frames.FirstOrDefault()?.ImagePath ?? string.Empty;
}
/// <summary>
/// Deletes all cached files for a location
/// Deletes all cached folders for a location
/// </summary>
public Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
var pattern = FilePathHelper.GetCacheFolderPattern(suburb, state);
var deleted = false;
try
{
// Delete all PNG files for this location
var pngFiles = Directory.GetFiles(_cacheDirectory, pattern);
foreach (var pngFile in pngFiles)
// Delete all folders matching the pattern
var folders = Directory.GetDirectories(_cacheDirectory, pattern);
foreach (var folder in folders)
{
try
{
File.Delete(pngFile);
_logger.LogInformation("Deleted cached screenshot: {File}", pngFile);
Directory.Delete(folder, recursive: true);
_logger.LogInformation("Deleted cached folder: {Folder}", folder);
deleted = true;
// Also delete associated metadata JSON file
var metadataFile = FilePathHelper.GetMetadataFilePath(pngFile);
if (File.Exists(metadataFile))
{
File.Delete(metadataFile);
_logger.LogInformation("Deleted metadata file: {File}", metadataFile);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete file: {File}", pngFile);
_logger.LogWarning(ex, "Failed to delete folder: {Folder}", folder);
}
}
if (deleted)
{
_logger.LogInformation("Deleted all cached files for location: {Suburb}, {State}", suburb, state);
_logger.LogInformation("Deleted all cached folders for location: {Suburb}, {State}", suburb, state);
}
else
{
_logger.LogDebug("No cached files found to delete for location: {Suburb}, {State}", suburb, state);
_logger.LogDebug("No cached folders found to delete for location: {Suburb}, {State}", suburb, state);
}
}
catch (Exception ex)
Services/Interfaces/IBomRadarService.cs
+25 -6
View File
@@ -9,14 +9,14 @@ namespace BomLocalService.Services.Interfaces;
public interface IBomRadarService
{
/// <summary>
/// Gets a cached radar screenshot for a location.
/// Returns the screenshot response if available in cache, otherwise returns null.
/// Gets cached radar data for a location.
/// Returns the radar response with all frames if available in cache, otherwise returns null.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Radar screenshot response with image path and metadata, or null if not cached</returns>
Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <returns>Radar response with frames and metadata, or null if not cached</returns>
Task<RadarResponse?> GetCachedRadarAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Triggers a cache update for a location.
@@ -49,12 +49,31 @@ public interface IBomRadarService
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes all cached files (screenshot and metadata) for a location.
/// Deletes all cached folders (containing frames and metadata) for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>True if files were deleted, false if no cached files existed</returns>
/// <returns>True if folders were deleted, false if no cached folders existed</returns>
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Gets all cached frames for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>List of cached frames with URLs, or null if no frames exist</returns>
Task<List<RadarFrame>?> GetCachedFramesAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Gets a specific cached frame for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="frameIndex">Frame index (0-6)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>The cached frame, or null if not found</returns>
Task<RadarFrame?> GetCachedFrameAsync(string suburb, string state, int frameIndex, CancellationToken cancellationToken = default);
}
Services/Interfaces/ICacheService.cs
+44 -10
View File
@@ -9,26 +9,60 @@ namespace BomLocalService.Services.Interfaces;
public interface ICacheService
{
/// <summary>
/// Gets the cached screenshot file path and associated metadata for a location.
/// Searches for PNG files matching the location pattern and loads the corresponding metadata JSON file.
/// Gets the cached screenshot folder path and associated metadata for a location.
/// Searches for cache folders matching the location pattern and loads the corresponding metadata JSON file.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Tuple containing the screenshot file path and metadata, or (null, null) if not found</returns>
Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
/// <returns>Tuple containing the cache folder path and metadata, or (null, null) if not found</returns>
Task<(string? cacheFolderPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
CancellationToken cancellationToken = default);
/// <summary>
/// Saves metadata JSON file alongside a screenshot file.
/// Creates a JSON file with the same base name as the screenshot but with .json extension.
/// Gets all cached frames for a location.
/// </summary>
/// <param name="screenshotPath">Full path to the screenshot PNG file</param>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>List of cached frames, or empty list if not found</returns>
Task<List<RadarFrame>> GetCachedFramesAsync(
string suburb,
string state,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets a specific cached frame for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="frameIndex">Frame index (0-6)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>The cached frame, or null if not found</returns>
Task<RadarFrame?> GetCachedFrameAsync(
string suburb,
string state,
int frameIndex,
CancellationToken cancellationToken = default);
/// <summary>
/// Saves metadata JSON file in a cache folder.
/// Creates a metadata.json file in the specified cache folder.
/// </summary>
/// <param name="cacheFolderPath">Full path to the cache folder</param>
/// <param name="metadata">Metadata to save (observation time, forecast time, weather station, distance)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default);
Task SaveMetadataAsync(string cacheFolderPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default);
/// <summary>
/// Saves frame metadata (frame index and minutes ago) to a frames.json file in the cache folder.
/// </summary>
/// <param name="cacheFolderPath">Full path to the cache folder</param>
/// <param name="frames">List of frames with their metadata</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task SaveFramesMetadataAsync(string cacheFolderPath, List<RadarFrame> frames, CancellationToken cancellationToken = default);
/// <summary>
/// Checks if cached metadata is still valid (not expired).
@@ -50,12 +84,12 @@ public interface ICacheService
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes all cached files (PNG screenshots and JSON metadata) for a location.
/// Deletes all cached folders (containing frames and metadata) for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>True if any files were deleted, false if no cached files existed</returns>
/// <returns>True if any folders were deleted, false if no cached folders existed</returns>
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
Services/Interfaces/IScrapingService.cs
+1 -1
View File
@@ -22,7 +22,7 @@ public interface IScrapingService
/// <param name="networkRequests">List to capture network request information (for debugging)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Radar screenshot response with image path and metadata</returns>
Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
Task<RadarResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string debugFolder,
Services/ScrapingService.cs
+349 -24
View File
@@ -14,6 +14,7 @@ public class ScrapingService : IScrapingService
private readonly IDebugService _debugService;
private readonly int _dynamicContentWaitMs;
private readonly int _tileRenderWaitMs;
private readonly ScreenshotCropConfig _cropConfig;
// Selector constants
private static readonly string[] SearchButtonSelectors = new[]
@@ -53,12 +54,25 @@ public class ScrapingService : IScrapingService
_debugService = debugService;
_dynamicContentWaitMs = configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
_tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
// Load crop configuration
var cropSection = configuration.GetSection("Screenshot:Crop");
_cropConfig = new ScreenshotCropConfig
{
X = cropSection.GetValue<int>("X", 0),
Y = cropSection.GetValue<int>("Y", 0),
Width = cropSection.GetValue<int?>("Width"),
Height = cropSection.GetValue<int?>("Height")
};
_logger.LogInformation("Screenshot crop config: X={X}, Y={Y}, Width={Width}, Height={Height}",
_cropConfig.X, _cropConfig.Y, _cropConfig.Width, _cropConfig.Height);
}
/// <summary>
/// Scrapes the BOM website to get a radar screenshot for a location
/// </summary>
public async Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
public async Task<RadarResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string debugFolder,
@@ -315,53 +329,208 @@ public class ScrapingService : IScrapingService
await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken);
// Step 8: Extract last updated information
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page);
// Step 8: Ensure radar is paused before capturing frames
_logger.LogInformation("Checking if radar loop is paused");
var playPauseButton = page.Locator("button[data-testid='bom-time-scrub-play-pause']").First;
await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
// Step 9: Find the map container and take screenshot of just the map area
_logger.LogInformation("Taking screenshot of map area");
// Check if button shows "Play" (paused) or "Pause" (playing)
var buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync();
if (buttonLabel?.Trim().Equals("Pause", StringComparison.OrdinalIgnoreCase) == true)
{
_logger.LogInformation("Radar is playing, pausing it");
await playPauseButton.ClickAsync();
// Wait for pause to take effect
await page.WaitForTimeoutAsync(500);
// Verify it's now paused
buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync();
if (buttonLabel?.Trim().Equals("Play", StringComparison.OrdinalIgnoreCase) != true)
{
_logger.LogWarning("Radar may not be paused after click, continuing anyway");
}
}
else
{
_logger.LogInformation("Radar is already paused");
}
await _debugService.SaveStepDebugAsync(debugFolder, 8, "radar_paused", page, consoleMessages, networkRequests, cancellationToken);
// Step 9: Click on first frame segment to ensure we start at frame 0
_logger.LogInformation("Resetting to first frame (frame 0)");
try
{
var firstFrameSegment = page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
await firstFrameSegment.ClickAsync();
// Wait for frame to update
await page.WaitForTimeoutAsync(1000);
_logger.LogInformation("Successfully clicked frame 0 segment");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to click first frame segment, continuing anyway");
}
await _debugService.SaveStepDebugAsync(debugFolder, 9, "frame_0_selected", page, consoleMessages, networkRequests, cancellationToken);
// Step 10: Verify scrubber is at position 0 before capturing first frame
_logger.LogInformation("Verifying scrubber is at position 0");
try
{
await page.WaitForFunctionAsync(@"() => {
const thumb = document.querySelector('[data-testid=""bom-scrub-thumb""]');
if (!thumb) return false;
const left = parseFloat(thumb.style.left) || 0;
// Position 0 should be at 0% or very close to it (within 5%)
return left <= 5;
}", new PageWaitForFunctionOptions { Timeout = 5000 });
// Also verify the active segment has data-id="0"
var activeSegment = await page.EvaluateAsync<bool>(@"() => {
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
const activeSegment = segments.find(s => {
const style = window.getComputedStyle(s);
return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
});
return activeSegment && activeSegment.getAttribute('data-id') === '0';
}");
if (activeSegment)
{
_logger.LogInformation("Scrubber confirmed at position 0");
}
else
{
_logger.LogWarning("Could not confirm scrubber is at position 0, but continuing");
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to verify scrubber position, continuing anyway");
}
await _debugService.SaveStepDebugAsync(debugFolder, 10, "scrubber_at_position_0", page, consoleMessages, networkRequests, cancellationToken);
// Step 11: Extract metadata and frame information
_logger.LogInformation("Extracting metadata and frame information");
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page);
var frameInfo = await ExtractFrameInfoAsync(page);
// Step 12: Get map container and calculate bounding box once
_logger.LogInformation("Preparing map container for screenshot");
var mapContainer = page.Locator(".esri-view-surface").First;
await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
// Ensure the map container is visible and has dimensions before taking screenshot
// Ensure the map container is visible and has dimensions
await page.WaitForFunctionAsync(@"() => {
const container = document.querySelector('.esri-view-surface');
return container && container.offsetWidth > 0 && container.offsetHeight > 0;
}", new PageWaitForFunctionOptions { Timeout = 10000 });
// Get bounding box of map container
var boundingBox = await mapContainer.BoundingBoxAsync();
if (boundingBox == null)
{
throw new Exception("Could not determine map container bounds");
}
// Convert BoundingBox to Clip for crop calculation
var containerClip = new Clip
{
X = boundingBox.X,
Y = boundingBox.Y,
Width = boundingBox.Width,
Height = boundingBox.Height
};
// Generate cache filename based on suburb, state and timestamp
// Step 13: Create cache folder
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
var cacheFileName = $"{safeLocationKey}_{timestamp}.png";
var cacheFilePath = Path.Combine(_cacheService.GetCacheDirectory(), cacheFileName);
var cacheFolderPath = FilePathHelper.GetCacheFolderPath(_cacheService.GetCacheDirectory(), suburb, state, timestamp);
Directory.CreateDirectory(cacheFolderPath);
// Take screenshot of the map area only
await page.ScreenshotAsync(new PageScreenshotOptions
_logger.LogInformation("Created cache folder: {Path}", cacheFolderPath);
// Step 14-20: Capture all 7 frames
var frames = new List<RadarFrame>();
var stepForwardButton = page.Locator("button[data-testid='bom-scrub-utils__right__step-forward']").First;
for (int frameIndex = 0; frameIndex < 7; frameIndex++)
{
Path = cacheFilePath,
Clip = new Clip
_logger.LogInformation("Capturing frame {FrameIndex} of 7", frameIndex);
// Wait for map to stabilize (tiles to load for current frame)
await page.WaitForTimeoutAsync(_tileRenderWaitMs);
// Extract actual minutes ago from the display label (e.g., "17 minutes ago" -> 17)
var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
if (minutesAgo == null)
{
X = boundingBox.X,
Y = boundingBox.Y,
Width = boundingBox.Width,
Height = boundingBox.Height
// Fallback to calculated value if extraction fails
var (_, defaultMinutesAgo) = frameInfo[frameIndex];
minutesAgo = defaultMinutesAgo;
_logger.LogWarning("Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", frameIndex, minutesAgo);
}
});
// Take screenshot with crop configuration
var framePath = FilePathHelper.GetFrameFilePath(cacheFolderPath, frameIndex);
await CaptureMapScreenshotAsync(page, mapContainer, framePath, containerClip);
frames.Add(new RadarFrame
{
FrameIndex = frameIndex,
ImagePath = framePath,
MinutesAgo = minutesAgo.Value
});
_logger.LogInformation("Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)",
frameIndex, framePath, minutesAgo.Value);
// Save debug screenshot BEFORE clicking step forward
await _debugService.SaveStepDebugAsync(debugFolder, 14 + frameIndex, $"frame_{frameIndex}_captured", page, consoleMessages, networkRequests, cancellationToken);
// If not the last frame, click step forward to prepare for next frame
if (frameIndex < 6)
{
// Wait for any modal overlays to disappear before clicking
try
{
await page.WaitForFunctionAsync(@"() => {
const overlay = document.querySelector('.bom-modal-overlay--after-open');
return !overlay || overlay.style.display === 'none';
}", new PageWaitForFunctionOptions { Timeout = 5000 });
}
catch
{
// If overlay doesn't disappear, try to dismiss it by clicking outside or pressing Escape
try
{
await page.Keyboard.PressAsync("Escape");
await page.WaitForTimeoutAsync(500);
}
catch
{
// Ignore if Escape doesn't work
}
}
// Use force click to bypass any remaining overlays
await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
// Wait for map to update to next frame
await page.WaitForTimeoutAsync(1000); // Wait for frame transition
}
}
_logger.LogInformation("Screenshot saved to: {Path}", cacheFilePath);
_logger.LogInformation("All 7 frames captured successfully");
// Save metadata alongside the screenshot
await _cacheService.SaveMetadataAsync(cacheFilePath, lastUpdatedInfo, cancellationToken);
// Step 21: Save metadata and frame information
await _cacheService.SaveMetadataAsync(cacheFolderPath, lastUpdatedInfo, cancellationToken);
await _cacheService.SaveFramesMetadataAsync(cacheFolderPath, frames, cancellationToken);
return ResponseBuilder.CreateRadarScreenshotResponse(cacheFilePath, lastUpdatedInfo);
// Step 22: Return response with all frames
return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, lastUpdatedInfo, suburb, state);
}
catch (Exception ex)
{
@@ -371,5 +540,161 @@ public class ScrapingService : IScrapingService
throw;
}
}
/// <summary>
/// Calculates the crop area for screenshot based on configuration
/// </summary>
private Clip CalculateCropArea(Clip containerClip)
{
// Start with container's position plus offset
var x = containerClip.X + _cropConfig.X;
var y = containerClip.Y + _cropConfig.Y;
// Calculate width (use configured or remaining width minus right offset)
var rightOffset = 200;
var width = _cropConfig.Width ?? (containerClip.Width - _cropConfig.X - rightOffset);
// Calculate height (use configured or remaining height)
var height = _cropConfig.Height ?? (containerClip.Height - _cropConfig.Y);
// Validate bounds
if (x < containerClip.X || y < containerClip.Y)
{
_logger.LogWarning("Crop offset is outside container bounds, using container bounds");
x = containerClip.X;
y = containerClip.Y;
}
var maxWidth = containerClip.Width - (x - containerClip.X);
var maxHeight = containerClip.Height - (y - containerClip.Y);
if (width > maxWidth)
{
_logger.LogWarning("Crop width exceeds container bounds, adjusting from {Requested} to {Max}", width, maxWidth);
width = maxWidth;
}
if (height > maxHeight)
{
_logger.LogWarning("Crop height exceeds container bounds, adjusting from {Requested} to {Max}", height, maxHeight);
height = maxHeight;
}
if (width <= 0 || height <= 0)
{
throw new Exception($"Invalid crop dimensions: {width}x{height}");
}
_logger.LogDebug("Crop area calculated: X={X}, Y={Y}, Width={Width}, Height={Height} (container: {ContainerX}, {ContainerY}, {ContainerWidth}x{ContainerHeight})",
x, y, width, height, containerClip.X, containerClip.Y, containerClip.Width, containerClip.Height);
return new Clip
{
X = x,
Y = y,
Width = width,
Height = height
};
}
/// <summary>
/// Extracts frame information from timeline segments
/// </summary>
private async Task<List<(int index, int minutesAgo)>> ExtractFrameInfoAsync(IPage page)
{
try
{
var frameInfo = await page.EvaluateAsync<object[]>(@"() => {
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
return segments.map((seg, index) => {
const ariaLabel = seg.getAttribute('aria-label') || '';
// Extract minutes from '40 minutes ago', '35 minutes ago', etc.
const minutesMatch = ariaLabel.match(/(\d+)\s+minutes?\s+ago/);
const minutes = minutesMatch ? parseInt(minutesMatch[1]) : null;
return { index: index, minutesAgo: minutes };
});
}");
var result = new List<(int index, int minutesAgo)>();
for (int i = 0; i < 7; i++)
{
// Default values if extraction fails
var minutesAgo = 40 - (i * 5);
// Try to use extracted values if available
if (frameInfo != null && i < frameInfo.Length)
{
// The EvaluateAsync returns object[], we'd need to deserialize properly
// For now, use defaults
}
result.Add((i, minutesAgo));
}
return result;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to extract frame info, using defaults");
// Return default frame info
return Enumerable.Range(0, 7)
.Select(i => (i, 40 - (i * 5)))
.ToList();
}
}
/// <summary>
/// Extracts the actual minutes ago value from the display label (e.g., "17 minutes ago" -> 17)
/// </summary>
private async Task<int?> ExtractMinutesAgoFromDisplayAsync(IPage page)
{
try
{
var timeLabel = await page.Locator(".bom-scrub-display-label").First.TextContentAsync();
if (string.IsNullOrEmpty(timeLabel))
{
return null;
}
// Parse "17 minutes ago" or "41 minutes ago" etc.
var match = System.Text.RegularExpressions.Regex.Match(
timeLabel.Trim(),
@"(\d+)\s+minutes?\s+ago",
System.Text.RegularExpressions.RegexOptions.IgnoreCase
);
if (match.Success && match.Groups.Count >= 2)
{
if (int.TryParse(match.Groups[1].Value, out var minutes))
{
return minutes;
}
}
return null;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to extract minutes from display label");
return null;
}
}
/// <summary>
/// Captures map screenshot with crop configuration
/// </summary>
private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip)
{
var cropArea = CalculateCropArea(containerClip);
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = outputPath,
Clip = cropArea
});
_logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})",
outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height);
}
}
Utilities/FilePathHelper.cs
+44 -2
View File
@@ -12,9 +12,10 @@ public static class FilePathHelper
}
/// <summary>
/// Gets the metadata file path for a given screenshot path
/// Gets the metadata file path for a given screenshot path (legacy method for backward compatibility)
/// </summary>
public static string GetMetadataFilePath(string screenshotPath)
[Obsolete("Use GetMetadataFilePath(string cacheFolderPath) for folder-based storage")]
public static string GetMetadataFilePathFromScreenshot(string screenshotPath)
{
return Path.ChangeExtension(screenshotPath, ".json");
}
@@ -36,5 +37,46 @@ public static class FilePathHelper
var locationKey = LocationHelper.GetLocationKey(suburb, state);
return GetCacheFilePattern(locationKey);
}
/// <summary>
/// Gets the cache folder path for a location and timestamp.
/// Format: "{CacheDirectory}/{LocationKey}_{Timestamp}"
/// Example: "/app/cache/Pomona_QLD_20251207_000906"
/// </summary>
public static string GetCacheFolderPath(string cacheDirectory, string suburb, string state, string timestamp)
{
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
return Path.Combine(cacheDirectory, $"{safeLocationKey}_{timestamp}");
}
/// <summary>
/// Gets the frame file path within a cache folder.
/// Format: "{CacheFolderPath}/frame_{FrameIndex}.png"
/// </summary>
public static string GetFrameFilePath(string cacheFolderPath, int frameIndex)
{
return Path.Combine(cacheFolderPath, $"frame_{frameIndex}.png");
}
/// <summary>
/// Gets the metadata file path within a cache folder.
/// Format: "{CacheFolderPath}/metadata.json"
/// </summary>
public static string GetMetadataFilePath(string cacheFolderPath)
{
return Path.Combine(cacheFolderPath, "metadata.json");
}
/// <summary>
/// Gets the cache folder pattern for finding existing folders.
/// Format: "{LocationKey}_*"
/// </summary>
public static string GetCacheFolderPattern(string suburb, string state)
{
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
return $"{safeLocationKey}_*";
}
}
Utilities/ResponseBuilder.cs
+55 -6
View File
@@ -5,9 +5,58 @@ namespace BomLocalService.Utilities;
public static class ResponseBuilder
{
/// <summary>
/// Creates a RadarScreenshotResponse from a file path and metadata
/// Creates a RadarResponse from a cache folder path, frames, and metadata
/// </summary>
public static RadarScreenshotResponse CreateRadarScreenshotResponse(
public static RadarResponse CreateRadarResponse(
string cacheFolderPath,
List<RadarFrame> frames,
LastUpdatedInfo? metadata = null,
string? suburb = null,
string? state = null)
{
var folderInfo = new DirectoryInfo(cacheFolderPath);
var lastWriteTime = folderInfo.Exists
? folderInfo.LastWriteTime
: DateTime.UtcNow;
// Generate URLs for each frame if suburb and state are provided
if (!string.IsNullOrEmpty(suburb) && !string.IsNullOrEmpty(state))
{
var encodedSuburb = Uri.EscapeDataString(suburb);
var encodedState = Uri.EscapeDataString(state);
foreach (var frame in frames)
{
frame.ImageUrl = $"/api/radar/{encodedSuburb}/{encodedState}/frame/{frame.FrameIndex}";
}
}
if (metadata == null)
{
return new RadarResponse
{
Frames = frames,
LastUpdated = lastWriteTime,
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
}
return new RadarResponse
{
Frames = frames,
LastUpdated = lastWriteTime,
ObservationTime = metadata.ObservationTime,
ForecastTime = metadata.ForecastTime,
WeatherStation = metadata.WeatherStation,
Distance = metadata.Distance
};
}
/// <summary>
/// Legacy method for backward compatibility - creates response with single frame
/// </summary>
[Obsolete("Use CreateRadarResponse with cacheFolderPath and frames")]
public static RadarResponse CreateRadarResponse(
string imagePath,
LastUpdatedInfo? metadata = null)
{
@@ -17,18 +66,18 @@ public static class ResponseBuilder
if (metadata == null)
{
return new RadarScreenshotResponse
return new RadarResponse
{
ImagePath = imagePath,
Frames = new List<RadarFrame>(),
LastUpdated = lastWriteTime,
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
}
return new RadarScreenshotResponse
return new RadarResponse
{
ImagePath = imagePath,
Frames = new List<RadarFrame>(),
LastUpdated = lastWriteTime,
ObservationTime = metadata.ObservationTime,
ForecastTime = metadata.ForecastTime,
appsettings.json
+7 -1
View File
@@ -20,7 +20,13 @@
},
"Screenshot": {
"DynamicContentWaitMs": 2000,
"TileRenderWaitMs": 5000
"TileRenderWaitMs": 5000,
"Crop": {
"X": 200,
"Y": 0,
"Width": null,
"Height": null
}
},
"Debug": {
"Enabled": false,