Initial commit: BOM Local Service

This commit is contained in:
2025-12-07 10:47:20 +10:00
parent 6d95920864
commit 86ff2c6f66
32 changed files with 3110 additions and 0 deletions
+27
View File
@@ -0,0 +1,27 @@
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/.idea
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/azds.yaml
**/bin
**/charts
**/docker-compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md
+78
View File
@@ -0,0 +1,78 @@
## .NET build artifacts
bin/
obj/
*.dll
*.exe
*.pdb
*.cache
*.deps.json
*.runtimeconfig.json
*.staticwebassets.endpoints.json
## Build results
[Dd]ebug/
[Rr]elease/
[Bb]uild/
x64/
x86/
[Aa]rm/
[Aa]rm64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
## Visual Studio / Rider
.vs/
.idea/
*.user
*.suo
*.userosscache
*.sln.docstates
*.userprefs
## VS Code
.vscode/
*.code-workspace
## Cache directory (generated screenshots and metadata)
cache/
## NuGet
*.nupkg
*.snupkg
**/packages/*
!**/packages/build/
project.lock.json
project.fragment.lock.json
artifacts/
## Test results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
*.trx
*.coverage
*.coveragexml
## OS files
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db
Desktop.ini
## Temporary files
*.tmp
*.temp
*.swp
*~
.*.sw?
## Docker
# Keep Dockerfile and docker-compose.yml, ignore temporary Docker files
*.dockerignore.bak
+14
View File
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.11" />
<PackageReference Include="Microsoft.Playwright" Version="1.57.0" />
</ItemGroup>
</Project>
+163
View File
@@ -0,0 +1,163 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.AspNetCore.Mvc;
namespace BomLocalService.Controllers;
[ApiController]
[Route("api/radar")]
public class RadarController : ControllerBase
{
private readonly IBomRadarService _bomRadarService;
private readonly ILogger<RadarController> _logger;
public RadarController(IBomRadarService bomRadarService, ILogger<RadarController> logger)
{
_bomRadarService = bomRadarService;
_logger = logger;
}
/// <summary>
/// Get radar screenshot for a location. Returns cached screenshot if available, or triggers background update.
/// </summary>
[HttpGet("{suburb}/{state}")]
public async Task<ActionResult> GetScreenshot(string suburb, string state, CancellationToken cancellationToken = default)
{
try
{
var validationError = ValidationHelper.ValidateLocation(suburb, state);
if (validationError != null)
{
return BadRequest(new { error = validationError });
}
// Check if cache needs updating and trigger if needed (non-blocking)
_ = Task.Run(async () =>
{
try
{
await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Background cache update failed for {Suburb}, {State}", suburb, state);
}
});
// Return cached screenshot if it exists (no waiting)
var result = await _bomRadarService.GetCachedScreenshotAsync(suburb, state, cancellationToken);
if (result == null)
{
return NotFound(new {
error = "Screenshot not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
retryAfter = 30, // seconds
refreshEndpoint = $"/api/radar/{suburb}/{state}/refresh"
});
}
// Return the image file
if (System.IO.File.Exists(result.ImagePath))
{
var imageBytes = await System.IO.File.ReadAllBytesAsync(result.ImagePath, cancellationToken);
var contentType = "image/png";
return File(imageBytes, contentType, Path.GetFileName(result.ImagePath));
}
return NotFound(new { error = "Screenshot file not found" });
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting cached radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while getting the radar screenshot", message = ex.Message });
}
}
/// <summary>
/// Get metadata about the cached radar data for a location (observation time, forecast time, etc.)
/// </summary>
[HttpGet("{suburb}/{state}/metadata")]
public async Task<ActionResult<LastUpdatedInfo>> GetMetadata(string suburb, string state, CancellationToken cancellationToken = default)
{
try
{
var validationError = ValidationHelper.ValidateLocation(suburb, state);
if (validationError != null)
{
return BadRequest(new { error = validationError });
}
var result = await _bomRadarService.GetLastUpdatedInfoAsync(suburb, state, cancellationToken);
if (result == null)
{
return NotFound(new { error = "No cached data found. Use POST /api/radar/{suburb}/{state}/refresh to trigger cache update." });
}
return Ok(result);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting metadata for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while getting metadata", message = ex.Message });
}
}
/// <summary>
/// Manually trigger a cache refresh for a location. Returns status of the update operation.
/// </summary>
[HttpPost("{suburb}/{state}/refresh")]
public async Task<ActionResult<CacheUpdateStatus>> RefreshCache(string suburb, string state, CancellationToken cancellationToken = default)
{
try
{
var validationError = ValidationHelper.ValidateLocation(suburb, state);
if (validationError != null)
{
return BadRequest(new { error = validationError });
}
var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
return Ok(status);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message });
}
}
/// <summary>
/// Delete cached data for a location.
/// </summary>
[HttpDelete("{suburb}/{state}")]
public async Task<ActionResult> DeleteCache(string suburb, string state, CancellationToken cancellationToken = default)
{
try
{
var validationError = ValidationHelper.ValidateLocation(suburb, state);
if (validationError != null)
{
return BadRequest(new { error = validationError });
}
var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
if (deleted)
{
return Ok(new { message = $"Cache deleted for {suburb}, {state}" });
}
else
{
return NotFound(new { error = $"No cached data found for {suburb}, {state}" });
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state);
return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message });
}
}
}
+46
View File
@@ -0,0 +1,46 @@
# Build stage
FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
WORKDIR /src
# Copy csproj and restore dependencies
COPY ["BomLocalService.csproj", "./"]
RUN dotnet restore "BomLocalService.csproj"
# Copy everything else and build
COPY . .
RUN dotnet build "BomLocalService.csproj" -c Release -o /app/build
# Publish stage
FROM build AS publish
RUN dotnet publish "BomLocalService.csproj" -c Release -o /app/publish /p:UseAppHost=false
# Runtime stage - use official Playwright .NET image that has browsers and dependencies pre-installed
FROM mcr.microsoft.com/playwright/dotnet:v1.57.0-noble AS final
WORKDIR /app
# Install .NET 9 runtime and Xvfb for virtual display (Playwright image has browsers but may need .NET 9)
RUN apt-get update && \
wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh && \
chmod +x dotnet-install.sh && \
./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \
rm dotnet-install.sh && \
apt-get install -y xvfb x11vnc fluxbox && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Copy published app
COPY --from=publish /app/publish .
# Create cache directory for screenshots
RUN mkdir -p /app/cache && chmod 777 /app/cache
# Create startup script to run Xvfb and then the app
RUN echo '#!/bin/bash\n\
Xvfb :99 -screen 0 1920x1080x24 > /dev/null 2>&1 &\n\
export DISPLAY=:99\n\
exec dotnet BomLocalService.dll' > /app/start.sh && \
chmod +x /app/start.sh
EXPOSE 8080
ENTRYPOINT ["/app/start.sh"]
+56
View File
@@ -0,0 +1,56 @@
namespace BomLocalService.Models;
/// <summary>
/// Status information about a cache update operation for a location.
/// Returned when manually triggering a cache refresh via the refresh endpoint.
/// </summary>
public class CacheUpdateStatus
{
/// <summary>
/// Indicates whether a background cache update was triggered.
/// True if the cache was missing or stale and an update was initiated.
/// False if the cache is valid and no update was needed.
/// </summary>
public bool UpdateTriggered { get; set; }
/// <summary>
/// Indicates whether cached screenshot files exist for this location.
/// True if PNG screenshot files are found in the cache directory.
/// False if no cached files exist for this location.
/// </summary>
public bool CacheExists { get; set; }
/// <summary>
/// Indicates whether the existing cache is still valid (not expired).
/// Cache is considered valid if the observation time plus expiration buffer (typically 15.5 minutes)
/// is still in the future. BOM updates observations every 15 minutes, so cache expiration
/// includes a small buffer to account for timing variations.
/// True if cache exists and is still valid, false if cache is missing or expired.
/// </summary>
public bool CacheIsValid { get; set; }
/// <summary>
/// The UTC timestamp when the current cache will expire and should be refreshed.
/// Calculated as: ObservationTime + CacheExpirationMinutes (typically observation time + 15.5 minutes).
/// Null if no cache exists or metadata is not available.
/// </summary>
public DateTime? CacheExpiresAt { get; set; }
/// <summary>
/// The UTC timestamp when the next cache update should occur.
/// If cache is valid: equals CacheExpiresAt (when current cache expires).
/// If cache is invalid/missing: equals current time + CacheExpirationMinutes (when background update will complete).
/// Null if cache is valid and CacheExpiresAt is null.
/// </summary>
public DateTime? NextUpdateTime { get; set; }
/// <summary>
/// Human-readable message describing the cache status and action taken.
/// Possible values:
/// - "Cache is valid, no update needed" - Cache exists and is still fresh
/// - "Cache is stale, update triggered" - Cache exists but expired, update initiated
/// - "No cache exists, update triggered" - No cache found, update initiated
/// </summary>
public string? Message { get; set; }
}
+39
View File
@@ -0,0 +1,39 @@
namespace BomLocalService.Models;
/// <summary>
/// Metadata extracted from the BOM weather map page about observation and forecast times.
/// This information is scraped from the weather metadata section on the BOM website.
/// </summary>
public class LastUpdatedInfo
{
/// <summary>
/// The UTC timestamp when the weather observations were taken.
/// Extracted from the BOM website metadata section (e.g., "Observations: 11 minutes ago, 8:20 pm AEST").
/// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour).
/// </summary>
public DateTime ObservationTime { get; set; }
/// <summary>
/// The UTC timestamp when the weather forecast was generated.
/// Extracted from the BOM website metadata section (e.g., "Forecast: 41 minutes ago, 7:50 pm AEST").
/// Forecasts are typically updated less frequently than observations.
/// </summary>
public DateTime ForecastTime { get; set; }
/// <summary>
/// The name of the weather station providing the observations for this location.
/// Extracted from text like "at Gympie weather station" in the metadata section.
/// May be null if the station name cannot be parsed from the page.
/// Example: "Gympie", "Brisbane", "Sydney"
/// </summary>
public string? WeatherStation { get; set; }
/// <summary>
/// The distance from the requested location to the weather station.
/// Extracted from text like "30 km from Pomona, QLD" in the metadata section.
/// Format: "{number} km" (e.g., "30 km", "15 km").
/// May be null if the distance cannot be parsed from the page.
/// </summary>
public string? Distance { get; set; }
}
+55
View File
@@ -0,0 +1,55 @@
namespace BomLocalService.Models;
/// <summary>
/// Response model containing the radar screenshot image path and associated metadata.
/// This is the primary response returned when requesting a radar screenshot for a location.
/// </summary>
public class RadarScreenshotResponse
{
/// <summary>
/// The full file system path to the cached PNG screenshot image file.
/// This is the path on the server where the screenshot is stored.
/// Format: "{CacheDirectory}/{Suburb}_{State}_{Timestamp}.png"
/// Example: "/app/cache/Pomona_QLD_20251207_000906.png"
/// </summary>
public string ImagePath { get; set; } = string.Empty;
/// <summary>
/// The UTC timestamp when the screenshot file was last written/modified on disk.
/// This is the file system modification time, which typically matches when the screenshot was captured.
/// Used as a fallback if metadata is not available.
/// </summary>
public DateTime LastUpdated { get; set; }
/// <summary>
/// The UTC timestamp when the weather observations were taken (from BOM website).
/// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page.
/// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour).
/// If metadata is not available, defaults to DateTime.UtcNow.
/// </summary>
public DateTime ObservationTime { get; set; }
/// <summary>
/// The UTC timestamp when the weather forecast was generated (from BOM website).
/// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page.
/// Forecasts are typically updated less frequently than observations.
/// If metadata is not available, defaults to DateTime.UtcNow.
/// </summary>
public DateTime ForecastTime { get; set; }
/// <summary>
/// The name of the weather station providing the observations for this location.
/// Extracted from the BOM website metadata (e.g., "Gympie", "Brisbane").
/// May be null if the station name cannot be parsed or if metadata is not available.
/// </summary>
public string? WeatherStation { get; set; }
/// <summary>
/// The distance from the requested location to the weather station.
/// Extracted from the BOM website metadata (e.g., "30 km", "15 km").
/// Format: "{number} km".
/// May be null if the distance cannot be parsed or if metadata is not available.
/// </summary>
public string? Distance { get; set; }
}
+51
View File
@@ -0,0 +1,51 @@
using BomLocalService.Services;
using BomLocalService.Services.Interfaces;
var builder = WebApplication.CreateBuilder(args);
// Add services to the container
builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddOpenApi();
// Register core services via interfaces (order matters - dependencies must be registered first)
builder.Services.AddSingleton<IDebugService, DebugService>();
builder.Services.AddSingleton<ICacheService, CacheService>();
builder.Services.AddSingleton<ITimeParsingService, TimeParsingService>();
builder.Services.AddSingleton<IBrowserService, BrowserService>();
builder.Services.AddSingleton<IScrapingService, ScrapingService>();
// Register BOM Radar Service as singleton (orchestrator, depends on all above services)
builder.Services.AddSingleton<IBomRadarService, BomLocalService.Services.BomRadarService>();
// Register background services (order matters - management service needs radar service)
builder.Services.AddHostedService<CacheCleanupService>();
builder.Services.AddHostedService<CacheManagementService>();
// Add configuration - appsettings.json provides default values
// All values can be overridden via environment variables
builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
var app = builder.Build();
// Configure the HTTP request pipeline
if (app.Environment.IsDevelopment())
{
app.MapOpenApi();
}
app.UseHttpsRedirection();
app.UseAuthorization();
app.MapControllers();
// Cleanup on shutdown
app.Lifetime.ApplicationStopped.Register(() =>
{
var service = app.Services.GetService<IBomRadarService>();
if (service is IDisposable disposable)
{
disposable.Dispose();
}
});
app.Run();
+23
View File
@@ -0,0 +1,23 @@
{
"$schema": "https://json.schemastore.org/launchsettings.json",
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": false,
"applicationUrl": "http://localhost:5249",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": false,
"applicationUrl": "https://localhost:7190;http://localhost:5249",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}
+217
View File
@@ -0,0 +1,217 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Playwright;
namespace BomLocalService.Services;
public class BomRadarService : IBomRadarService, IDisposable
{
private readonly ILogger<BomRadarService> _logger;
private readonly ICacheService _cacheService;
private readonly IBrowserService _browserService;
private readonly IScrapingService _scrapingService;
private readonly IDebugService _debugService;
private readonly double _cacheExpirationMinutes;
public BomRadarService(
ILogger<BomRadarService> logger,
ICacheService cacheService,
IBrowserService browserService,
IScrapingService scrapingService,
IDebugService debugService,
IConfiguration configuration)
{
_logger = logger;
_cacheService = cacheService;
_browserService = browserService;
_scrapingService = scrapingService;
_debugService = debugService;
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
}
public async Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (string.IsNullOrEmpty(cachedPath) || !File.Exists(cachedPath))
{
return null;
}
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
}
public async Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var status = new CacheUpdateStatus();
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
status.CacheExists = !string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath);
if (cachedMetadata != null)
{
status.CacheIsValid = _cacheService.IsCacheValid(cachedMetadata);
status.CacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
}
// Check if we need to update
bool needsUpdate = !status.CacheExists || !status.CacheIsValid;
if (needsUpdate)
{
// Trigger async update (fire and forget)
_ = Task.Run(async () =>
{
try
{
await FetchAndCacheScreenshotAsync(suburb, state, CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state);
}
}, cancellationToken);
status.UpdateTriggered = true;
status.Message = status.CacheExists
? "Cache is stale, update triggered"
: "No cache exists, update triggered";
// Calculate next update time
if (status.CacheExpiresAt.HasValue && status.CacheExpiresAt.Value > DateTime.UtcNow)
{
status.NextUpdateTime = status.CacheExpiresAt.Value;
}
else
{
status.NextUpdateTime = DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes);
}
}
else
{
status.UpdateTriggered = false;
status.Message = "Cache is valid, no update needed";
status.NextUpdateTime = status.CacheExpiresAt;
}
return status;
}
private async Task<RadarScreenshotResponse> FetchAndCacheScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
// Check cache FIRST, before acquiring semaphore (cached requests shouldn't block)
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath) && cachedMetadata != null)
{
var isValid = _cacheService.IsCacheValid(cachedMetadata);
if (isValid)
{
_logger.LogInformation("Returning valid cached screenshot for {Suburb}, {State} (no semaphore needed)", suburb, state);
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
}
else
{
var nextUpdate = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
var timeUntilExpiry = nextUpdate - DateTime.UtcNow;
_logger.LogInformation("Cached screenshot exists but is stale (observation time: {ObservationTime}, expired {TimeAgo} ago), fetching new one",
cachedMetadata.ObservationTime, -timeUntilExpiry);
}
}
else if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath))
{
_logger.LogWarning("Cached screenshot found but no metadata file exists, fetching new one");
}
else
{
_logger.LogInformation("No cached screenshot found for {Suburb}, {State}, fetching new one", suburb, state);
}
// Only acquire semaphore if we need to fetch new screenshot
var semaphore = _browserService.GetSemaphore();
await semaphore.WaitAsync(cancellationToken);
// Generate request ID for debugging
var requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}";
var debugFolder = _debugService.CreateRequestFolder(requestId);
IBrowserContext? context = null;
try
{
// Double-check cache after acquiring semaphore (another request might have just created it)
var (recheckCachedPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (!string.IsNullOrEmpty(recheckCachedPath) && File.Exists(recheckCachedPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata))
{
_logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshot");
return ResponseBuilder.CreateRadarScreenshotResponse(recheckCachedPath, recheckCachedMetadata);
}
// Need to capture new screenshot
context = await _browserService.CreateContextAsync();
var (page, consoleMessages, networkRequests) = await _browserService.CreatePageWithDebugAsync(context, requestId);
try
{
return await _scrapingService.ScrapeRadarScreenshotAsync(
suburb,
state,
debugFolder,
page,
consoleMessages,
networkRequests,
cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during radar screenshot capture (RequestId: {RequestId})", requestId);
throw;
}
finally
{
await context.CloseAsync();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting radar screenshot for suburb: {Suburb}, state: {State} (RequestId: {RequestId})", suburb, state, requestId);
throw;
}
finally
{
semaphore.Release();
}
}
public async Task<LastUpdatedInfo?> GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
// Return cached metadata if available, otherwise return null
var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
if (metadata != null)
{
_logger.LogDebug("Returning cached last updated info for {Suburb}, {State}", suburb, state);
return metadata;
}
_logger.LogDebug("No cached metadata found for {Suburb}, {State}", suburb, state);
return null;
}
public Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
return _cacheService.GetCachedScreenshotPathAsync(suburb, state, cancellationToken);
}
public Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
return _cacheService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
}
public void Dispose()
{
_browserService?.Dispose();
}
}
+368
View File
@@ -0,0 +1,368 @@
using BomLocalService.Services.Interfaces;
using Microsoft.Playwright;
namespace BomLocalService.Services;
public class BrowserService : IBrowserService
{
private readonly ILogger<BrowserService> _logger;
private readonly string _timezone;
private readonly IDebugService _debugService;
private IPlaywright? _playwright;
private IBrowser? _browser;
private readonly SemaphoreSlim _semaphore = new(1, 1);
public BrowserService(ILogger<BrowserService> logger, IConfiguration configuration, IDebugService debugService)
{
_logger = logger;
_timezone = configuration.GetValue<string>("Timezone") ?? "Australia/Brisbane";
_debugService = debugService;
}
/// <summary>
/// Gets or creates the Playwright instance
/// </summary>
private async Task<IPlaywright> GetPlaywrightAsync()
{
if (_playwright == null)
{
_playwright = await Microsoft.Playwright.Playwright.CreateAsync();
}
return _playwright;
}
/// <summary>
/// Gets or creates the browser instance
/// </summary>
public async Task<IBrowser> GetBrowserAsync()
{
if (_browser == null)
{
var playwright = await GetPlaywrightAsync();
_browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false, // Run in visible mode with virtual display (Xvfb)
Args = new[]
{
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-blink-features=AutomationControlled",
"--disable-dev-shm-usage",
"--disable-infobars",
"--disable-extensions",
"--disable-background-networking",
"--disable-background-timer-throttling",
"--disable-backgrounding-occluded-windows",
"--disable-breakpad",
"--disable-client-side-phishing-detection",
"--disable-component-extensions-with-background-pages",
"--disable-component-update",
"--disable-default-apps",
"--disable-domain-reliability",
"--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
"--disable-hang-monitor",
"--disable-ipc-flooding-protection",
"--disable-notifications",
"--disable-offer-store-unmasked-wallet-cards",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
"--disable-renderer-backgrounding",
"--disable-sync",
"--force-color-profile=srgb",
"--metrics-recording-only",
"--no-first-run",
"--no-default-browser-check",
"--no-pings",
"--password-store=basic",
"--use-mock-keychain",
"--enable-automation=false",
"--exclude-switches=enable-automation",
"--disable-features=UserAgentClientHint",
"--enable-unsafe-swiftshader", // Required for software WebGL (Chrome 137+ deprecated automatic fallback)
"--use-gl=swiftshader" // Use SwiftShader for WebGL software rendering
}
});
}
return _browser;
}
/// <summary>
/// Creates a new browser context with proper configuration
/// </summary>
public async Task<IBrowserContext> CreateContextAsync()
{
var browser = await GetBrowserAsync();
return await browser.NewContextAsync(new BrowserNewContextOptions
{
ViewportSize = new ViewportSize { Width = 1920, Height = 1080 },
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Locale = "en-AU",
TimezoneId = _timezone,
Permissions = new[] { "geolocation" },
ExtraHTTPHeaders = new Dictionary<string, string>
{
{ "Accept-Language", "en-AU,en;q=0.9" },
{ "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" },
{ "Accept-Encoding", "gzip, deflate, br" },
{ "Connection", "keep-alive" },
{ "Upgrade-Insecure-Requests", "1" },
{ "Sec-Fetch-Dest", "document" },
{ "Sec-Fetch-Mode", "navigate" },
{ "Sec-Fetch-Site", "none" },
{ "Cache-Control", "max-age=0" }
}
});
}
/// <summary>
/// Creates a new page with anti-detection scripts and debug event handlers
/// </summary>
public async Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync(
IBrowserContext context,
string requestId)
{
var page = await context.NewPageAsync();
// Set up console message and network request capture for debugging
var consoleMessages = new List<(string type, string text, DateTime timestamp)>();
var networkRequests = new List<(string method, string url, int? status, string resourceType, DateTime timestamp)>();
if (_debugService.IsEnabled)
{
// Capture page console messages (console.log, console.error, etc.)
page.Console += (sender, e) =>
{
consoleMessages.Add((e.Type.ToString(), e.Text, DateTime.UtcNow));
};
// Capture page errors (unhandled JavaScript errors)
page.PageError += (sender, e) =>
{
consoleMessages.Add(("error", $"PageError: {e}", DateTime.UtcNow));
};
// Capture context console messages (broader scope, may include more warnings)
context.Console += (sender, e) =>
{
consoleMessages.Add((e.Type.ToString(), $"[Context] {e.Text}", DateTime.UtcNow));
};
// Capture network requests
page.Request += (sender, e) =>
{
networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow));
};
// Capture failed requests (these often show up as console errors in browser)
page.RequestFailed += (sender, e) =>
{
networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow));
var failureText = e.Failure?.ToString() ?? "Unknown failure";
consoleMessages.Add(("error", $"Request failed: {e.Method} {e.Url} - {failureText}", DateTime.UtcNow));
};
page.Response += (sender, e) =>
{
// Update the request with response status
var matchingRequests = networkRequests.Where(r => r.url == e.Request.Url && r.status == null).ToList();
if (matchingRequests.Any())
{
var request = matchingRequests.Last();
var index = networkRequests.IndexOf(request);
networkRequests[index] = (request.method, request.url, e.Status, request.resourceType, request.timestamp);
}
};
}
// Inject comprehensive anti-detection scripts (must run before page load)
await page.AddInitScriptAsync(GetAntiDetectionScript());
return (page, consoleMessages, networkRequests);
}
/// <summary>
/// Gets the anti-detection JavaScript script
/// </summary>
private static string GetAntiDetectionScript()
{
return @"
(function() {
'use strict';
// Remove webdriver property completely
Object.defineProperty(navigator, 'webdriver', {
get: () => false,
configurable: true
});
// Delete it from prototype chain
try {
delete navigator.__proto__.webdriver;
} catch (e) {}
// Override chrome object
window.chrome = {
runtime: {},
loadTimes: function() {},
csi: function() {},
app: {}
};
// Override plugins with realistic data
Object.defineProperty(navigator, 'plugins', {
get: () => {
const plugins = [];
for (let i = 0; i < 3; i++) {
plugins.push({
0: { type: 'application/x-google-chrome-pdf', suffixes: 'pdf', description: 'Portable Document Format' },
description: 'Portable Document Format',
filename: 'internal-pdf-viewer',
length: 1,
name: 'Chrome PDF Plugin'
});
}
return plugins;
},
configurable: true
});
// Override languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-AU', 'en', 'en-US'],
configurable: true
});
// Override language
Object.defineProperty(navigator, 'language', {
get: () => 'en-AU',
configurable: true
});
// Override vendor
Object.defineProperty(navigator, 'vendor', {
get: () => 'Google Inc.',
configurable: true
});
// Override platform
Object.defineProperty(navigator, 'platform', {
get: () => 'Win32',
configurable: true
});
// Override hardwareConcurrency
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8,
configurable: true
});
// Override deviceMemory
if ('deviceMemory' in navigator) {
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8,
configurable: true
});
}
// Spoof connection - use named functions to avoid warnings
Object.defineProperty(navigator, 'connection', {
get: () => ({
effectiveType: '4g',
rtt: 50,
downlink: 10,
saveData: false,
onchange: null,
addEventListener: function addEventListener() {},
removeEventListener: function removeEventListener() {},
dispatchEvent: function dispatchEvent() { return true; }
}),
configurable: true
});
// Override permissions API - bind context to avoid illegal invocation
const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
window.navigator.permissions.query = function query(parameters) {
return parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters);
};
// Override getBattery - bind context to avoid illegal invocation
if ('getBattery' in navigator) {
navigator.getBattery = function getBattery() {
return Promise.resolve({
charging: true,
chargingTime: 0,
dischargingTime: Infinity,
level: 1,
onchargingchange: null,
onchargingtimechange: null,
ondischargingtimechange: null,
onlevelchange: null,
addEventListener: function addEventListener() {},
removeEventListener: function removeEventListener() {},
dispatchEvent: function dispatchEvent() { return true; }
});
};
}
// DON'T use Proxy on navigator - it breaks APIs like getGamepads()
// Just hide webdriver property directly
try {
delete navigator.__proto__.webdriver;
} catch (e) {}
// AudioContext fingerprinting
if (window.AudioContext) {
const originalAudioContext = window.AudioContext;
window.AudioContext = function() {
const ctx = new originalAudioContext();
const originalCreateOscillator = ctx.createOscillator;
ctx.createOscillator = function() {
const oscillator = originalCreateOscillator.call(this);
const originalFrequency = Object.getOwnPropertyDescriptor(
AudioParam.prototype, 'value'
).get;
Object.defineProperty(oscillator.frequency, 'value', {
get: originalFrequency,
set: function() {}
});
return oscillator;
};
return ctx;
};
}
})();
";
}
/// <summary>
/// Initializes the browser (pre-warming)
/// </summary>
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
_logger.LogInformation("Pre-warming browser");
try
{
await GetBrowserAsync();
_logger.LogInformation("Browser initialized and ready");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to pre-warm browser, will initialize on first use");
}
}
/// <summary>
/// Gets the semaphore for synchronizing browser operations
/// </summary>
public SemaphoreSlim GetSemaphore() => _semaphore;
public void Dispose()
{
_browser?.DisposeAsync().AsTask().Wait();
_playwright?.Dispose();
_semaphore?.Dispose();
}
}
+141
View File
@@ -0,0 +1,141 @@
using BomLocalService.Utilities;
using Microsoft.Extensions.Hosting;
namespace BomLocalService.Services;
public class CacheCleanupService : BackgroundService
{
private readonly ILogger<CacheCleanupService> _logger;
private readonly string _cacheDirectory;
private readonly int _retentionHours;
private readonly TimeSpan _cleanupInterval;
public CacheCleanupService(ILogger<CacheCleanupService> logger, IConfiguration configuration)
{
_logger = logger;
_cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
_retentionHours = configuration.GetValue<int>("CacheRetentionHours", 24);
var cleanupIntervalHours = configuration.GetValue<int>("CacheCleanup:IntervalHours", 1);
_cleanupInterval = TimeSpan.FromHours(cleanupIntervalHours);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Cache cleanup service started. Retention: {RetentionHours} hours, Interval: {Interval}",
_retentionHours, _cleanupInterval);
while (!stoppingToken.IsCancellationRequested)
{
try
{
await CleanupOldCacheFilesAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during cache cleanup");
}
await Task.Delay(_cleanupInterval, stoppingToken);
}
}
private Task CleanupOldCacheFilesAsync(CancellationToken cancellationToken)
{
if (!Directory.Exists(_cacheDirectory))
{
return Task.CompletedTask;
}
var cutoffTime = DateTime.UtcNow.AddHours(-_retentionHours);
var deletedCount = 0;
var totalSize = 0L;
try
{
// Get all PNG files in cache directory
var pngFiles = Directory.GetFiles(_cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
foreach (var file in pngFiles)
{
if (cancellationToken.IsCancellationRequested)
break;
try
{
var fileInfo = new FileInfo(file);
var fileTime = fileInfo.LastWriteTimeUtc;
// Delete if file is older than retention period
if (fileTime < cutoffTime)
{
var fileSize = fileInfo.Length;
File.Delete(file);
deletedCount++;
totalSize += fileSize;
// Also delete associated metadata file if it exists
var metadataFile = FilePathHelper.GetMetadataFilePath(file);
if (File.Exists(metadataFile))
{
File.Delete(metadataFile);
}
_logger.LogDebug("Deleted old cache file: {File} (age: {Age})",
Path.GetFileName(file), DateTime.UtcNow - fileTime);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete cache file: {File}", file);
}
}
if (deletedCount > 0)
{
var sizeInMB = totalSize / (1024.0 * 1024.0);
_logger.LogInformation("Cache cleanup completed. Deleted {Count} files ({Size:F2} MB)",
deletedCount, sizeInMB);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during cache file enumeration");
}
// Also clean up old debug directories
try
{
var debugDir = Path.Combine(_cacheDirectory, "debug");
if (Directory.Exists(debugDir))
{
var debugDirs = Directory.GetDirectories(debugDir);
foreach (var dir in debugDirs)
{
if (cancellationToken.IsCancellationRequested)
break;
try
{
var dirInfo = new DirectoryInfo(dir);
if (dirInfo.LastWriteTimeUtc < cutoffTime)
{
Directory.Delete(dir, recursive: true);
_logger.LogDebug("Deleted old debug directory: {Dir}", dir);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete debug directory: {Dir}", dir);
}
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error cleaning up debug directories: {Error}", ex.Message);
}
return Task.CompletedTask;
}
}
+210
View File
@@ -0,0 +1,210 @@
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Extensions.Hosting;
namespace BomLocalService.Services;
public class CacheManagementService : BackgroundService
{
private readonly ILogger<CacheManagementService> _logger;
private readonly IBomRadarService _bomRadarService;
private readonly IBrowserService _browserService;
private readonly IConfiguration _configuration;
private readonly TimeSpan _checkInterval;
private readonly HashSet<string> _activeUpdates = new();
private readonly object _lock = new();
public CacheManagementService(
ILogger<CacheManagementService> logger,
IBomRadarService bomRadarService,
IBrowserService browserService,
IConfiguration configuration)
{
_logger = logger;
_bomRadarService = bomRadarService;
_browserService = browserService;
_configuration = configuration;
var checkIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
_checkInterval = TimeSpan.FromMinutes(checkIntervalMinutes);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Cache management service started. Check interval: {Interval}", _checkInterval);
// Wait a bit for the service to fully initialize
var initialDelaySeconds = _configuration.GetValue<int>("CacheManagement:InitialDelaySeconds", 10);
await Task.Delay(TimeSpan.FromSeconds(initialDelaySeconds), stoppingToken);
// Pre-warm the browser before starting cache updates
_logger.LogInformation("Pre-warming browser before cache updates");
try
{
await _browserService.InitializeAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to pre-warm browser, will continue anyway");
}
// Get initial list of locations to manage (could be from config or discovered from cache)
var locationsToManage = GetLocationsToManage();
// Initial cache update on startup
_logger.LogInformation("Performing initial cache update for {Count} locations", locationsToManage.Count);
int initialUpdatesTriggered = 0;
int initialCachesValid = 0;
foreach (var (suburb, state) in locationsToManage)
{
if (stoppingToken.IsCancellationRequested)
break;
var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken);
if (triggered)
{
initialUpdatesTriggered++;
}
else if (isValid)
{
initialCachesValid++;
}
// Stagger updates to avoid overwhelming the system
var updateStaggerSeconds = _configuration.GetValue<int>("CacheManagement:UpdateStaggerSeconds", 2);
await Task.Delay(TimeSpan.FromSeconds(updateStaggerSeconds), stoppingToken);
}
_logger.LogInformation("Initial cache update completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid",
initialUpdatesTriggered, initialCachesValid);
// Periodic cache updates
while (!stoppingToken.IsCancellationRequested)
{
try
{
await Task.Delay(_checkInterval, stoppingToken);
// Refresh list of locations (in case new ones were cached)
locationsToManage = GetLocationsToManage();
_logger.LogInformation("Starting periodic cache check for {Count} locations", locationsToManage.Count);
int updatesTriggered = 0;
int cachesValid = 0;
foreach (var (suburb, state) in locationsToManage)
{
if (stoppingToken.IsCancellationRequested)
break;
var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken);
if (triggered)
{
updatesTriggered++;
}
else if (isValid)
{
cachesValid++;
}
// Small delay between locations
var locationStaggerSeconds = _configuration.GetValue<int>("CacheManagement:LocationStaggerSeconds", 1);
await Task.Delay(TimeSpan.FromSeconds(locationStaggerSeconds), stoppingToken);
}
_logger.LogInformation("Periodic cache check completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid",
updatesTriggered, cachesValid);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during cache management cycle");
}
}
}
private async Task<(bool updateTriggered, bool isValid)> UpdateCacheIfNeededAsync(string suburb, string state, CancellationToken cancellationToken)
{
var locationKey = LocationHelper.GetLocationKey(suburb, state);
// Check if update is already in progress
lock (_lock)
{
if (_activeUpdates.Contains(locationKey))
{
_logger.LogInformation("Update already in progress for {Suburb}, {State}, skipping", suburb, state);
return (false, false);
}
_activeUpdates.Add(locationKey);
}
try
{
var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
if (status.UpdateTriggered)
{
_logger.LogInformation("Cache update triggered for {Suburb}, {State} (expired at {ExpiresAt})",
suburb, state, status.CacheExpiresAt);
return (true, false);
}
else
{
_logger.LogInformation("Cache is valid for {Suburb}, {State} (expires at {ExpiresAt})",
suburb, state, status.CacheExpiresAt);
return (false, true);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking/updating cache for {Suburb}, {State}", suburb, state);
return (false, false);
}
finally
{
lock (_lock)
{
_activeUpdates.Remove(locationKey);
}
}
}
private List<(string suburb, string state)> GetLocationsToManage()
{
var locations = new List<(string, string)>();
try
{
// Get locations from existing cache files
var cacheDirectory = FilePathHelper.GetCacheDirectory(_configuration);
if (Directory.Exists(cacheDirectory))
{
var pngFiles = Directory.GetFiles(cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
var seen = new HashSet<string>();
foreach (var file in pngFiles)
{
var location = LocationHelper.ParseLocationFromFilename(file);
if (location.HasValue)
{
var locationKey = LocationHelper.GetLocationKey(location.Value.suburb, location.Value.state);
if (!seen.Contains(locationKey))
{
seen.Add(locationKey);
locations.Add(location.Value);
}
}
}
}
// Could also read from config if you want to pre-configure locations
// var configuredLocations = _configuration.GetSection("ManagedLocations").Get<List<LocationConfig>>();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error discovering locations from cache");
}
return locations;
}
}
+189
View File
@@ -0,0 +1,189 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using System.Text.Json;
namespace BomLocalService.Services;
public class CacheService : ICacheService
{
private readonly ILogger<CacheService> _logger;
private readonly string _cacheDirectory;
private readonly double _cacheExpirationMinutes;
public CacheService(ILogger<CacheService> logger, IConfiguration configuration)
{
_logger = logger;
_cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
// Ensure cache directory exists
Directory.CreateDirectory(_cacheDirectory);
}
/// <summary>
/// Gets the cached screenshot path and metadata for a location
/// </summary>
public async Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
_logger.LogDebug("Looking for cached files matching pattern: {Pattern} in directory: {Directory}", pattern, _cacheDirectory);
var files = Directory.GetFiles(_cacheDirectory, pattern)
.OrderByDescending(f =>
{
// Try to extract timestamp from filename first
var timestamp = LocationHelper.ParseTimestampFromFilename(f);
if (timestamp.HasValue)
{
return timestamp.Value;
}
// Fallback to file write time
return File.GetLastWriteTime(f);
})
.ToList();
var screenshotPath = files.FirstOrDefault();
if (string.IsNullOrEmpty(screenshotPath) || !File.Exists(screenshotPath))
{
return (null, null);
}
var metadata = await LoadMetadataAsync(screenshotPath, cancellationToken);
return (screenshotPath, metadata);
}
/// <summary>
/// Saves metadata alongside a screenshot
/// </summary>
public async Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default)
{
try
{
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions { WriteIndented = true });
// Use CancellationToken.None for file writes to avoid cancellation issues
// The screenshot is already saved, metadata save is best-effort
await File.WriteAllTextAsync(metadataPath, json, CancellationToken.None);
_logger.LogDebug("Saved metadata to: {Path}", metadataPath);
}
catch (Exception ex)
{
// Log but don't fail - screenshot is already saved
_logger.LogWarning(ex, "Failed to save metadata for screenshot: {Path}", screenshotPath);
}
}
/// <summary>
/// Loads metadata for a screenshot
/// </summary>
public async Task<LastUpdatedInfo?> LoadMetadataAsync(string screenshotPath, CancellationToken cancellationToken = default)
{
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
if (!File.Exists(metadataPath))
{
return null;
}
try
{
var json = await File.ReadAllTextAsync(metadataPath, cancellationToken);
return JsonSerializer.Deserialize<LastUpdatedInfo>(json);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load metadata from: {Path}", metadataPath);
return null;
}
}
/// <summary>
/// Checks if cache is still valid based on observation time
/// </summary>
public bool IsCacheValid(LastUpdatedInfo? metadata)
{
if (metadata == null)
{
return false;
}
// BOM updates observations at :00, :15, :30, :45 (every 15 minutes by default)
// Cache expiration includes buffer (e.g., 15.5 minutes = 15 min + 30 sec buffer)
var nextUpdateTime = metadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
return DateTime.UtcNow < nextUpdateTime;
}
/// <summary>
/// Gets the path to the cached screenshot for a location (simple version, ordered by creation time)
/// </summary>
public Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
var files = Directory.GetFiles(_cacheDirectory, pattern)
.OrderByDescending(f => File.GetCreationTime(f))
.ToList();
return Task.FromResult(files.FirstOrDefault() ?? string.Empty);
}
/// <summary>
/// Deletes all cached files for a location
/// </summary>
public Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default)
{
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
var deleted = false;
try
{
// Delete all PNG files for this location
var pngFiles = Directory.GetFiles(_cacheDirectory, pattern);
foreach (var pngFile in pngFiles)
{
try
{
File.Delete(pngFile);
_logger.LogInformation("Deleted cached screenshot: {File}", pngFile);
deleted = true;
// Also delete associated metadata JSON file
var metadataFile = FilePathHelper.GetMetadataFilePath(pngFile);
if (File.Exists(metadataFile))
{
File.Delete(metadataFile);
_logger.LogInformation("Deleted metadata file: {File}", metadataFile);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete file: {File}", pngFile);
}
}
if (deleted)
{
_logger.LogInformation("Deleted all cached files for location: {Suburb}, {State}", suburb, state);
}
else
{
_logger.LogDebug("No cached files found to delete for location: {Suburb}, {State}", suburb, state);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting cached location: {Suburb}, {State}", suburb, state);
return Task.FromResult(false);
}
return Task.FromResult(deleted);
}
/// <summary>
/// Gets the cache directory path
/// </summary>
public string GetCacheDirectory() => _cacheDirectory;
}
+176
View File
@@ -0,0 +1,176 @@
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Playwright;
namespace BomLocalService.Services;
public class DebugService : IDebugService
{
private readonly ILogger<DebugService> _logger;
private readonly bool _enabled;
private readonly string _debugDirectory;
private readonly int _waitMs;
public DebugService(ILogger<DebugService> logger, IConfiguration configuration)
{
_logger = logger;
_waitMs = configuration.GetValue<int>("Debug:WaitMs", 2000);
// Check all possible ways the config might be set
var debugEnabledEnv = Environment.GetEnvironmentVariable("DEBUG__ENABLED");
var debugEnabledConfig = configuration.GetValue<bool>("Debug:Enabled", false);
var debugEnabledFromConfig = configuration["Debug:Enabled"];
_logger.LogInformation("Debug configuration check - Env var DEBUG__ENABLED: {EnvVar}, Config Debug:Enabled: {ConfigValue}, Config string: {ConfigString}",
debugEnabledEnv ?? "null", debugEnabledConfig, debugEnabledFromConfig ?? "null");
// Try environment variable first, then config
_enabled = !string.IsNullOrEmpty(debugEnabledEnv)
? bool.TryParse(debugEnabledEnv, out var envBool) && envBool
: debugEnabledConfig;
var cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
_debugDirectory = Path.Combine(cacheDirectory, "debug");
if (_enabled)
{
Directory.CreateDirectory(_debugDirectory);
_logger.LogInformation("Debug mode is ENABLED. Debug files will be saved to: {DebugDirectory}", _debugDirectory);
}
else
{
_logger.LogInformation("Debug mode is DISABLED");
}
}
public bool IsEnabled => _enabled;
public string CreateRequestFolder(string requestId)
{
if (!_enabled) return string.Empty;
var requestFolder = Path.Combine(_debugDirectory, requestId);
Directory.CreateDirectory(requestFolder);
_logger.LogDebug("Created debug folder for request: {RequestId} at {Path}", requestId, requestFolder);
return requestFolder;
}
public async Task SaveStepDebugAsync(string requestFolder, int stepNumber, string stepName, IPage page, List<(string type, string text, DateTime timestamp)>? consoleMessages = null, List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, CancellationToken cancellationToken = default)
{
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
try
{
// Wait for page to be fully loaded before taking screenshot
try
{
await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 10000 });
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 5000 });
}
catch
{
// If network idle times out, at least wait for DOM
}
await page.WaitForTimeoutAsync(_waitMs); // Additional wait for dynamic content and animations
var stepFolder = Path.Combine(requestFolder, $"step_{stepNumber:D2}_{stepName}");
Directory.CreateDirectory(stepFolder);
// Save screenshot
var screenshotPath = Path.Combine(stepFolder, "screenshot.png");
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = screenshotPath,
FullPage = true
});
// Save HTML
var htmlContent = await page.ContentAsync();
var htmlPath = Path.Combine(stepFolder, "page.html");
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
// Save console messages if provided
if (consoleMessages != null && consoleMessages.Count > 0)
{
var consolePath = Path.Combine(stepFolder, "console.log");
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
}
// Save network requests summary if provided
if (networkRequests != null && networkRequests.Count > 0)
{
var networkPath = Path.Combine(stepFolder, "network.log");
var networkText = string.Join("\n", networkRequests.Select(req =>
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
));
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
}
_logger.LogDebug("Saved debug files for step {StepNumber} ({StepName}) to {Path}", stepNumber, stepName, stepFolder);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to save debug files for step {StepNumber} ({StepName})", stepNumber, stepName);
}
}
public async Task SaveErrorDebugAsync(string requestFolder, string errorMessage, IPage? page = null,
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
CancellationToken cancellationToken = default)
{
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
try
{
var errorFolder = Path.Combine(requestFolder, "error");
Directory.CreateDirectory(errorFolder);
// Save error message
var errorPath = Path.Combine(errorFolder, "error.txt");
await File.WriteAllTextAsync(errorPath, errorMessage, cancellationToken);
// Save screenshot if page is available
if (page != null)
{
var screenshotPath = Path.Combine(errorFolder, "screenshot.png");
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = screenshotPath,
FullPage = true
});
// Save HTML
var htmlContent = await page.ContentAsync();
var htmlPath = Path.Combine(errorFolder, "page.html");
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
}
// Save console messages if provided
if (consoleMessages != null && consoleMessages.Count > 0)
{
var consolePath = Path.Combine(errorFolder, "console.log");
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
}
// Save network requests summary if provided
if (networkRequests != null && networkRequests.Count > 0)
{
var networkPath = Path.Combine(errorFolder, "network.log");
var networkText = string.Join("\n", networkRequests.Select(req =>
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
));
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
}
_logger.LogDebug("Saved error debug files to {Path}", errorFolder);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to save error debug files");
}
}
}
+60
View File
@@ -0,0 +1,60 @@
using BomLocalService.Models;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Main service interface for BOM radar screenshot operations.
/// Orchestrates cache management, browser automation, and web scraping to provide radar screenshots for Australian locations.
/// </summary>
public interface IBomRadarService
{
/// <summary>
/// Gets a cached radar screenshot for a location.
/// Returns the screenshot response if available in cache, otherwise returns null.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Radar screenshot response with image path and metadata, or null if not cached</returns>
Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Triggers a cache update for a location.
/// If cache is missing or expired, initiates a background update and returns status information.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Status information about the cache update operation</returns>
Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Gets metadata about the cached radar data for a location.
/// Returns observation time, forecast time, weather station, and distance information.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Last updated information, or null if no cached data exists</returns>
Task<LastUpdatedInfo?> GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Gets the file system path to the cached screenshot for a location.
/// Returns empty string if no cached screenshot exists.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>File system path to the cached screenshot, or empty string if not found</returns>
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes all cached files (screenshot and metadata) for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>True if files were deleted, false if no cached files existed</returns>
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
}
+44
View File
@@ -0,0 +1,44 @@
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Service interface for managing Playwright browser instances and automation.
/// Handles browser lifecycle, context creation, and anti-detection measures for web scraping.
/// </summary>
public interface IBrowserService : IDisposable
{
/// <summary>
/// Creates a new browser context with proper configuration for BOM website scraping.
/// Configures viewport, user agent, timezone, and HTTP headers to mimic a real browser.
/// </summary>
/// <returns>A configured browser context ready for page creation</returns>
Task<IBrowserContext> CreateContextAsync();
/// <summary>
/// Creates a new page with anti-detection scripts and optional debug event handlers.
/// Injects comprehensive anti-detection JavaScript to avoid bot detection.
/// If debug mode is enabled, captures console messages and network requests for debugging.
/// </summary>
/// <param name="context">The browser context to create the page in</param>
/// <param name="requestId">Unique request identifier for debug file organization</param>
/// <returns>Tuple containing the page, list of console messages, and list of network requests</returns>
Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync(
IBrowserContext context,
string requestId);
/// <summary>
/// Initializes and pre-warms the browser instance.
/// Launches the Chromium browser to reduce latency on first use.
/// </summary>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task InitializeAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Gets the semaphore used to synchronize browser operations.
/// Ensures only one browser operation runs at a time to prevent resource conflicts.
/// </summary>
/// <returns>The semaphore for synchronizing browser access</returns>
SemaphoreSlim GetSemaphore();
}
+67
View File
@@ -0,0 +1,67 @@
using BomLocalService.Models;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Service interface for managing cached radar screenshot files and metadata.
/// Handles file system operations for storing and retrieving cached BOM radar screenshots.
/// </summary>
public interface ICacheService
{
/// <summary>
/// Gets the cached screenshot file path and associated metadata for a location.
/// Searches for PNG files matching the location pattern and loads the corresponding metadata JSON file.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Tuple containing the screenshot file path and metadata, or (null, null) if not found</returns>
Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
string suburb,
string state,
CancellationToken cancellationToken = default);
/// <summary>
/// Saves metadata JSON file alongside a screenshot file.
/// Creates a JSON file with the same base name as the screenshot but with .json extension.
/// </summary>
/// <param name="screenshotPath">Full path to the screenshot PNG file</param>
/// <param name="metadata">Metadata to save (observation time, forecast time, weather station, distance)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default);
/// <summary>
/// Checks if cached metadata is still valid (not expired).
/// Cache is valid if observation time + expiration buffer (typically 15.5 minutes) is still in the future.
/// BOM updates observations every 15 minutes, so the buffer accounts for timing variations.
/// </summary>
/// <param name="metadata">The metadata to validate, or null</param>
/// <returns>True if metadata exists and cache is still valid, false otherwise</returns>
bool IsCacheValid(LastUpdatedInfo? metadata);
/// <summary>
/// Gets the file system path to the most recent cached screenshot for a location.
/// Returns files ordered by creation time, most recent first.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>File system path to the cached screenshot, or empty string if not found</returns>
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes all cached files (PNG screenshots and JSON metadata) for a location.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>True if any files were deleted, false if no cached files existed</returns>
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
/// <summary>
/// Gets the cache directory path where screenshots and metadata are stored.
/// </summary>
/// <returns>The full path to the cache directory</returns>
string GetCacheDirectory();
}
+72
View File
@@ -0,0 +1,72 @@
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Service interface for debug file generation during web scraping operations.
/// When enabled, saves screenshots, HTML, console logs, and network request information for troubleshooting.
/// </summary>
public interface IDebugService
{
/// <summary>
/// Indicates whether debug mode is enabled.
/// When false, debug methods return immediately without saving files.
/// </summary>
bool IsEnabled { get; }
/// <summary>
/// Creates a folder for storing debug files for a specific request.
/// Folder structure: {CacheDirectory}/debug/{requestId}/
/// </summary>
/// <param name="requestId">Unique identifier for the request (e.g., timestamp + GUID)</param>
/// <returns>Full path to the created debug folder, or empty string if debug is disabled</returns>
string CreateRequestFolder(string requestId);
/// <summary>
/// Saves debug files for a scraping step.
/// Creates a step folder (e.g., "step_01_homepage_loaded") containing:
/// - screenshot.png (full page screenshot)
/// - page.html (complete HTML source)
/// - console.log (browser console messages)
/// - network.log (network request summary)
/// </summary>
/// <param name="requestFolder">The debug folder path for this request (from CreateRequestFolder)</param>
/// <param name="stepNumber">Step number for ordering (e.g., 1, 2, 3)</param>
/// <param name="stepName">Descriptive name for the step (e.g., "homepage_loaded", "search_button_clicked")</param>
/// <param name="page">The Playwright page instance to capture</param>
/// <param name="consoleMessages">Optional list of console messages to save</param>
/// <param name="networkRequests">Optional list of network requests to save</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task SaveStepDebugAsync(
string requestFolder,
int stepNumber,
string stepName,
IPage page,
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Saves debug files when an error occurs during scraping.
/// Creates an "error" subfolder containing:
/// - error.txt (error message and stack trace)
/// - screenshot.png (page state at error time, if page available)
/// - page.html (HTML source at error time, if page available)
/// - console.log (console messages up to error)
/// - network.log (network requests up to error)
/// </summary>
/// <param name="requestFolder">The debug folder path for this request (from CreateRequestFolder)</param>
/// <param name="errorMessage">The error message and/or stack trace to save</param>
/// <param name="page">Optional Playwright page instance to capture error state</param>
/// <param name="consoleMessages">Optional list of console messages to save</param>
/// <param name="networkRequests">Optional list of network requests to save</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
Task SaveErrorDebugAsync(
string requestFolder,
string errorMessage,
IPage? page = null,
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
CancellationToken cancellationToken = default);
}
+34
View File
@@ -0,0 +1,34 @@
using BomLocalService.Models;
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Service interface for scraping radar screenshots from the BOM website.
/// Orchestrates the multi-step process of navigating BOM, finding locations, and capturing radar screenshots.
/// </summary>
public interface IScrapingService
{
/// <summary>
/// Scrapes a radar screenshot from the BOM website for a given location.
/// Performs the complete workflow: navigates to BOM, searches for location, navigates to radar page,
/// waits for map to render, captures screenshot, extracts metadata, and saves to cache.
/// </summary>
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
/// <param name="debugFolder">Folder path for saving debug files (screenshots, HTML, logs) if debug mode is enabled</param>
/// <param name="page">The Playwright page instance to use for scraping</param>
/// <param name="consoleMessages">List to capture console messages from the browser (for debugging)</param>
/// <param name="networkRequests">List to capture network request information (for debugging)</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
/// <returns>Radar screenshot response with image path and metadata</returns>
Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string debugFolder,
IPage page,
List<(string type, string text, DateTime timestamp)> consoleMessages,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
CancellationToken cancellationToken = default);
}
@@ -0,0 +1,29 @@
using BomLocalService.Models;
using Microsoft.Playwright;
namespace BomLocalService.Services.Interfaces;
/// <summary>
/// Service interface for parsing time and metadata information from BOM website content.
/// Extracts observation times, forecast times, weather station names, and distances from BOM weather map pages.
/// </summary>
public interface ITimeParsingService
{
/// <summary>
/// Extracts last updated information from a BOM weather map page.
/// Scrapes the weather metadata section to get observation time, forecast time, weather station, and distance.
/// </summary>
/// <param name="page">The Playwright page instance containing the loaded BOM weather map</param>
/// <returns>Last updated information with parsed times and metadata</returns>
Task<LastUpdatedInfo> ExtractLastUpdatedInfoAsync(IPage page);
/// <summary>
/// Parses last updated text from BOM website to extract structured information.
/// Handles various text formats like "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD".
/// Converts times to UTC using the configured timezone.
/// </summary>
/// <param name="text">The raw text content from the BOM weather metadata section</param>
/// <returns>Parsed last updated information with observation time, forecast time, weather station, and distance</returns>
LastUpdatedInfo ParseLastUpdatedText(string text);
}
+373
View File
@@ -0,0 +1,373 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Playwright;
namespace BomLocalService.Services;
public class ScrapingService : IScrapingService
{
private readonly ILogger<ScrapingService> _logger;
private readonly IBrowserService _browserService;
private readonly ITimeParsingService _timeParsingService;
private readonly ICacheService _cacheService;
private readonly IDebugService _debugService;
private readonly int _dynamicContentWaitMs;
private readonly int _tileRenderWaitMs;
// Selector constants
private static readonly string[] SearchButtonSelectors = new[]
{
"button[data-testid='searchLabel']",
"button[aria-label='Search for a location']",
"button.search-location__trigger-button",
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
"button:has-text('Search for a location')",
"[data-testid='showSearchModal'] button"
};
private static readonly string[] RadarLinkSelectors = new[]
{
"a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
".cta.button a.jump-link:has-text('Rain radar and weather map')",
"a.jump-link:has-text('Rain radar and weather map')",
".cta.button a:has-text('Rain radar and weather map')",
"a.bom-button--secondary:has-text('Rain radar and weather map')",
"a:has-text('Rain radar and weather map')",
"a:has-text('rain radar')",
"a[href*='radar']"
};
public ScrapingService(
ILogger<ScrapingService> logger,
IBrowserService browserService,
ITimeParsingService timeParsingService,
ICacheService cacheService,
IDebugService debugService,
IConfiguration configuration)
{
_logger = logger;
_browserService = browserService;
_timeParsingService = timeParsingService;
_cacheService = cacheService;
_debugService = debugService;
_dynamicContentWaitMs = configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
_tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
}
/// <summary>
/// Scrapes the BOM website to get a radar screenshot for a location
/// </summary>
public async Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
string suburb,
string state,
string debugFolder,
IPage page,
List<(string type, string text, DateTime timestamp)> consoleMessages,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
CancellationToken cancellationToken = default)
{
try
{
// Step 1: Navigate to BOM homepage and wait for search button
_logger.LogInformation("Navigating to BOM homepage");
await page.GotoAsync("https://www.bom.gov.au/", new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30000
});
// Wait for search button to be ready instead of NetworkIdle (faster)
var searchButtonReady = page.Locator("button[data-testid='searchLabel'], button[aria-label='Search for a location'], button.search-location__trigger-button").First;
await searchButtonReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000, State = WaitForSelectorState.Visible });
await _debugService.SaveStepDebugAsync(debugFolder, 1, "homepage_loaded", page, consoleMessages, networkRequests, cancellationToken);
// Step 2: Click "Search for a location" button to open search UI
_logger.LogInformation("Clicking 'Search for a location' button");
var searchButton = await SelectorHelper.FindLocatorBySelectorsAsync(page, SearchButtonSelectors, _logger);
if (searchButton == null)
{
var errorMsg = "Could not find 'Search for a location' button on BOM homepage.";
await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
throw new Exception(errorMsg);
}
await searchButton.ClickAsync();
// Wait for search input to appear
var searchInputReady = page.Locator("#search-enter-keyword").First;
await searchInputReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000, State = WaitForSelectorState.Visible });
await _debugService.SaveStepDebugAsync(debugFolder, 2, "search_button_clicked", page, consoleMessages, networkRequests, cancellationToken);
// Step 3: Find and fill search box with suburb name
_logger.LogInformation("Searching for suburb: {Suburb}", suburb);
var searchInput = page.Locator("#search-enter-keyword").First;
await searchInput.FillAsync(suburb);
// Step 4: Wait for autocomplete suggestions to appear
_logger.LogInformation("Waiting for autocomplete suggestions");
await page.WaitForFunctionAsync(@"() => {
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
return results.length > 0 && results.some(r => r.offsetParent !== null);
}", new PageWaitForFunctionOptions { Timeout = 10000 });
await _debugService.SaveStepDebugAsync(debugFolder, 3, "search_input_filled", page, consoleMessages, networkRequests, cancellationToken);
// Step 5: Find the matching search result based on suburb and state
_logger.LogInformation("Looking for matching search result for {Suburb}, {State}", suburb, state);
var suburbLower = suburb.ToLower().Trim();
var stateLower = state.ToLower().Trim();
// Get the actual count from the summary element
var summaryText = await page.Locator("#location-results-title, [data-testid='location-results-title']").First.TextContentAsync();
int? actualCount = null;
if (!string.IsNullOrEmpty(summaryText))
{
// Parse "3 of 3 location results" or similar
var countMatch = System.Text.RegularExpressions.Regex.Match(summaryText, @"(\d+)\s+of\s+(\d+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
if (countMatch.Success && countMatch.Groups.Count >= 3)
{
if (int.TryParse(countMatch.Groups[2].Value, out var total))
{
actualCount = total;
}
}
}
// Fetch all result data in one JavaScript evaluation - extract location name and state separately
List<(string name, string desc, string fullText)> results = new();
try
{
var resultData = await page.EvaluateAsync<string[][]>(@"() => {
// Scope to the location results list - find the ul with aria-labelledby pointing to location-results-title
const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
if (!resultsList) {
console.log('Location results list not found');
return [];
}
const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
console.log('Found', results.length, 'location results');
return results.map((r, index) => {
// Query from the li element - elements are nested inside <a> tag
const nameEl = r.querySelector('[data-testid=""location-name""]');
const descEl = r.querySelector('.bom-linklist-item__desc');
const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
const fullText = (r.textContent || r.innerText || '').trim();
console.log('Result', index, ':', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
return [name, desc, fullText];
});
}");
// Convert to structured data
results = resultData.Select(arr => (
name: arr.Length > 0 ? arr[0] : "",
desc: arr.Length > 1 ? arr[1] : "",
fullText: arr.Length > 2 ? arr[2] : ""
)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content");
// Fallback to simple text extraction
var resultTexts = await page.EvaluateAsync<string[]>(@"() => {
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
return results.map(r => r.textContent || '');
}");
results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList();
}
if (actualCount.HasValue)
{
_logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim());
// Only use the first N results that match the actual count
if (results.Count > actualCount.Value)
{
results = results.Take(actualCount.Value).ToList();
}
}
else
{
_logger.LogInformation("Found {Count} search results", results.Count);
}
int? matchingIndex = null;
for (int i = 0; i < results.Count; i++)
{
var (name, desc, fullText) = results[i];
var nameLower = name.ToLower().Trim();
var descLower = desc.ToLower().Trim();
var fullTextLower = fullText.ToLower();
_logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'",
i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText);
// Check if suburb name matches (from location-name element or fallback to fullText)
var matchesSuburb = false;
if (!string.IsNullOrEmpty(name))
{
matchesSuburb = nameLower == suburbLower || nameLower.Contains(suburbLower);
}
else
{
// Fallback to fullText if name extraction failed
matchesSuburb = fullTextLower.Contains(suburbLower);
}
// Check if state matches (from description which contains "Queensland 4300" or "New South Wales 2469")
var matchesState = false;
if (!string.IsNullOrEmpty(desc))
{
matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower);
}
// Always check fullText for state as fallback
if (!matchesState)
{
matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower);
}
_logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (suburb='{Suburb}' vs name='{Name}'), matchesState={MatchesState} (state='{State}' vs desc='{Desc}')",
i, matchesSuburb, suburbLower, nameLower, matchesState, stateLower, descLower);
if (matchesSuburb && matchesState)
{
matchingIndex = i;
_logger.LogInformation("Found matching result: {Name} - {Desc}", name, desc);
break;
}
}
// Click the matching result (or first if no match)
// Scope to the location results list
var resultsList = page.Locator("ul[aria-labelledby='location-results-title']");
var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']");
var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First;
if (!matchingIndex.HasValue)
{
_logger.LogInformation("No exact match found, using first result");
}
await resultToClick.ClickAsync();
await _debugService.SaveStepDebugAsync(debugFolder, 4, "search_result_selected", page, consoleMessages, networkRequests, cancellationToken);
// Wait for forecast page to load
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
await page.WaitForTimeoutAsync(_dynamicContentWaitMs); // Brief wait for dynamic content
await _debugService.SaveStepDebugAsync(debugFolder, 5, "forecast_page_loaded", page, consoleMessages, networkRequests, cancellationToken);
// Step 6: Find and click "Rain radar and weather map" link
_logger.LogInformation("Looking for 'Rain radar and weather map' link");
var radarLink = await SelectorHelper.FindLocatorBySelectorsAsync(page, RadarLinkSelectors, _logger);
if (radarLink == null)
{
var errorMsg = $"Could not find 'Rain radar and weather map' link for {suburb}, {state}";
await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
throw new Exception(errorMsg);
}
await radarLink.ClickAsync();
await _debugService.SaveStepDebugAsync(debugFolder, 6, "radar_link_clicked", page, consoleMessages, networkRequests, cancellationToken);
// Step 7: Wait for weather map page to load and map to fully render
_logger.LogInformation("Waiting for weather map page to load");
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
// Wait for the map canvas element to appear with proper dimensions
_logger.LogInformation("Waiting for map canvas element to render");
var mapCanvas = page.Locator(".esri-view-surface canvas").First;
await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 });
// Wait for canvas to have valid dimensions
await page.WaitForFunctionAsync(@"() => {
const canvas = document.querySelector('.esri-view-surface canvas');
return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
}", new PageWaitForFunctionOptions { Timeout = 15000 });
_logger.LogInformation("Map canvas is ready - waiting for map to render");
// Wait for Esri map view to be ready
try
{
await page.WaitForFunctionAsync(@"() => {
try {
const elements = document.querySelectorAll('.esri-view');
for (let el of elements) {
if (el.__view && el.__view.ready) {
return true;
}
}
} catch(e) {}
return false;
}", new PageWaitForFunctionOptions { Timeout = 30000 });
_logger.LogInformation("Esri map view is ready");
}
catch
{
_logger.LogInformation("Esri view ready check timed out, continuing with fixed wait");
}
// Additional wait for tiles to render
await page.WaitForTimeoutAsync(_tileRenderWaitMs);
await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken);
// Step 8: Extract last updated information
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page);
// Step 9: Find the map container and take screenshot of just the map area
_logger.LogInformation("Taking screenshot of map area");
var mapContainer = page.Locator(".esri-view-surface").First;
await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
// Ensure the map container is visible and has dimensions before taking screenshot
await page.WaitForFunctionAsync(@"() => {
const container = document.querySelector('.esri-view-surface');
return container && container.offsetWidth > 0 && container.offsetHeight > 0;
}", new PageWaitForFunctionOptions { Timeout = 10000 });
// Get bounding box of map container
var boundingBox = await mapContainer.BoundingBoxAsync();
if (boundingBox == null)
{
throw new Exception("Could not determine map container bounds");
}
// Generate cache filename based on suburb, state and timestamp
var locationKey = LocationHelper.GetLocationKey(suburb, state);
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
var cacheFileName = $"{safeLocationKey}_{timestamp}.png";
var cacheFilePath = Path.Combine(_cacheService.GetCacheDirectory(), cacheFileName);
// Take screenshot of the map area only
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = cacheFilePath,
Clip = new Clip
{
X = boundingBox.X,
Y = boundingBox.Y,
Width = boundingBox.Width,
Height = boundingBox.Height
}
});
_logger.LogInformation("Screenshot saved to: {Path}", cacheFilePath);
// Save metadata alongside the screenshot
await _cacheService.SaveMetadataAsync(cacheFilePath, lastUpdatedInfo, cancellationToken);
return ResponseBuilder.CreateRadarScreenshotResponse(cacheFilePath, lastUpdatedInfo);
}
catch (Exception ex)
{
// Save error debug info if debug is enabled
await _debugService.SaveErrorDebugAsync(debugFolder, $"Exception: {ex.Message}\n\nStackTrace:\n{ex.StackTrace}", page, consoleMessages, networkRequests, cancellationToken);
_logger.LogError(ex, "Error during radar screenshot capture");
throw;
}
}
}
+254
View File
@@ -0,0 +1,254 @@
using BomLocalService.Models;
using BomLocalService.Services.Interfaces;
using Microsoft.Playwright;
using System.Globalization;
using System.Text.RegularExpressions;
namespace BomLocalService.Services;
public class TimeParsingService : ITimeParsingService
{
private readonly ILogger<TimeParsingService> _logger;
private readonly TimeZoneInfo _timeZoneInfo;
public TimeParsingService(ILogger<TimeParsingService> logger, IConfiguration configuration)
{
_logger = logger;
var timezone = configuration.GetValue<string>("Timezone") ?? "Australia/Brisbane";
try
{
_timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
_logger.LogInformation("Using timezone: {Timezone} ({DisplayName})", timezone, _timeZoneInfo.DisplayName);
}
catch (TimeZoneNotFoundException)
{
_logger.LogWarning("Timezone {Timezone} not found, defaulting to Australia/Brisbane", timezone);
_timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
}
}
/// <summary>
/// Extracts last updated information from the weather map page
/// </summary>
public async Task<LastUpdatedInfo> ExtractLastUpdatedInfoAsync(IPage page)
{
try
{
// Look for the weather metadata section using the correct selector
// HTML structure: <section data-testid="weatherMetadata"> with aria-label="Last updated"
var metadataSection = page.Locator("section[data-testid='weatherMetadata'], section[aria-label='Last updated']").First;
// Extract text from each div separately to preserve structure
// Structure: <div>Observations: X minutes ago, 9:40 pm AEST</div><div>at Gympie...</div><div>Forecast: ...</div>
var lastUpdatedText = await page.EvaluateAsync<string>(@"() => {
const section = document.querySelector('section[data-testid=""weatherMetadata""]') ||
document.querySelector('section[aria-label=""Last updated""]');
if (!section) return null;
// Get all divs and join with newlines to preserve structure
const divs = section.querySelectorAll('div');
return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
}");
if (string.IsNullOrEmpty(lastUpdatedText))
{
// Fallback: get all text content
lastUpdatedText = await metadataSection.TextContentAsync();
}
return ParseLastUpdatedText(lastUpdatedText ?? string.Empty);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to extract last updated information");
return new LastUpdatedInfo
{
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
}
}
/// <summary>
/// Parses last updated text to extract observation time, forecast time, weather station, and distance
/// </summary>
public LastUpdatedInfo ParseLastUpdatedText(string text)
{
var info = new LastUpdatedInfo();
_logger.LogInformation("Parsing last updated text: {Text}", text);
// Try to parse "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD"
// And "Forecast: 41 minutes ago, 7:50 pm AEST"
// Prefer parsing the actual time string over "minutes ago" for accuracy
// Parse observation time - improved regex to better capture time and timezone
// Stop timezone capture before "at" (e.g., "AEST at Gympie" or "AESTat Gympie" should capture just "AEST")
// Pattern: "Observations: X minutes ago, 9:40 pm AESTat Gympie..." (note: sometimes no space between AEST and "at")
// Capture timezone as 3-4 uppercase letters (AEST, AEDT, etc.) optionally followed by "at"
var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
_logger.LogDebug("Observation regex match result: Success={Success}, Groups.Count={Count}", observationMatch.Success, observationMatch.Groups.Count);
if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
{
var timeStr = observationMatch.Groups[1].Value.Trim();
var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null;
// Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST")
if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2)
{
timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2);
}
_logger.LogDebug("Extracted observation time string: '{TimeStr}', timezone: '{TzStr}'", timeStr, timezoneStr ?? "null");
if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
{
info.ObservationTime = observationTime;
_logger.LogInformation("Successfully parsed observation time: {Time} UTC (from '{TimeStr}' {TzStr})",
observationTime, timeStr, timezoneStr ?? "default timezone");
}
else
{
// Fallback to "minutes ago" if time parsing fails
_logger.LogWarning("Failed to parse observation time from '{TimeStr}', falling back to 'minutes ago'", timeStr);
var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
{
info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
_logger.LogInformation("Used 'minutes ago' fallback: {MinutesAgo} minutes ago = {Time} UTC",
minutesAgo, info.ObservationTime);
}
}
}
// Parse forecast time
var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase);
if (forecastMatch.Success && forecastMatch.Groups[1].Success)
{
var timeStr = forecastMatch.Groups[1].Value.Trim();
var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null;
if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
{
info.ForecastTime = forecastTime;
}
else
{
// Fallback to "minutes ago" if time parsing fails
var minutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
{
info.ForecastTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
}
}
}
// Extract weather station name
var stationMatch = Regex.Match(text, @"at\s+([^,]+)\s+weather\s+station", RegexOptions.IgnoreCase);
if (stationMatch.Success)
{
info.WeatherStation = stationMatch.Groups[1].Value.Trim();
}
// Extract distance
var distanceMatch = Regex.Match(text, @"(\d+)\s*km\s+from", RegexOptions.IgnoreCase);
if (distanceMatch.Success)
{
info.Distance = $"{distanceMatch.Groups[1].Value} km";
}
return info;
}
/// <summary>
/// Tries to parse a time string with optional timezone to UTC
/// </summary>
private bool TryParseTimeString(string timeStr, string? timezoneStr, out DateTime utcTime)
{
utcTime = DateTime.UtcNow;
try
{
// Parse time formats like "8:20 pm", "8:20pm", "20:20"
var timeFormats = new[] { "h:mm tt", "h:mmtt", "HH:mm", "H:mm tt", "H:mmtt", "H:mm" };
DateTime localTime = default;
// Try parsing with various formats
bool parsed = false;
foreach (var format in timeFormats)
{
if (DateTime.TryParseExact(timeStr, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsedTime))
{
localTime = parsedTime;
parsed = true;
break;
}
}
if (!parsed)
{
// Fallback to general parsing
if (!DateTime.TryParse(timeStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out localTime))
{
return false;
}
}
// Determine the timezone - use configured timezone if not specified or if it's AEST/AEDT
TimeZoneInfo tz = _timeZoneInfo;
if (!string.IsNullOrEmpty(timezoneStr))
{
// AEST/AEDT are typically the same as Australia/Brisbane or Australia/Sydney
// Both Brisbane and Sydney use AEST/AEDT
if (timezoneStr.Contains("AEST", StringComparison.OrdinalIgnoreCase) ||
timezoneStr.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
{
tz = _timeZoneInfo; // Use configured timezone
}
else
{
// Try to find the timezone, fallback to configured
try
{
tz = TimeZoneInfo.FindSystemTimeZoneById(timezoneStr);
}
catch
{
tz = _timeZoneInfo;
}
}
}
// Get current time in the target timezone to determine the date
var nowInTz = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz);
var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay);
// If the parsed time is in the future (e.g., it's 9pm but we parsed 8pm and it's actually yesterday),
// or if it's more than 12 hours in the past, assume it's from yesterday
if (dateTimeInTz > nowInTz)
{
// Time is in the future, so it must be from yesterday
dateTimeInTz = dateTimeInTz.AddDays(-1);
}
else if ((nowInTz - dateTimeInTz).TotalHours > 12)
{
// More than 12 hours ago, likely from yesterday
dateTimeInTz = dateTimeInTz.AddDays(-1);
}
// Convert to UTC
// Note: Brisbane (Australia/Brisbane) is always UTC+10 (AEST), no daylight saving
utcTime = TimeZoneInfo.ConvertTimeToUtc(dateTimeInTz, tz);
_logger.LogInformation("Parsed time string '{TimeStr}' with timezone '{TzStr}' as {DateTimeInTz} local ({UtcTime} UTC)",
timeStr, timezoneStr ?? "default", dateTimeInTz, utcTime);
return true;
}
catch
{
return false;
}
}
}
+40
View File
@@ -0,0 +1,40 @@
namespace BomLocalService.Utilities;
public static class FilePathHelper
{
/// <summary>
/// Gets the cache directory path from configuration or returns default
/// </summary>
public static string GetCacheDirectory(IConfiguration configuration)
{
return configuration.GetValue<string>("CacheDirectory")
?? Path.Combine(AppContext.BaseDirectory, "cache");
}
/// <summary>
/// Gets the metadata file path for a given screenshot path
/// </summary>
public static string GetMetadataFilePath(string screenshotPath)
{
return Path.ChangeExtension(screenshotPath, ".json");
}
/// <summary>
/// Gets the cache file pattern for a location key (e.g., "Pomona_QLD_*.png")
/// </summary>
public static string GetCacheFilePattern(string locationKey)
{
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
return $"{safeLocationKey}_*.png";
}
/// <summary>
/// Gets the cache file pattern for suburb and state
/// </summary>
public static string GetCacheFilePattern(string suburb, string state)
{
var locationKey = LocationHelper.GetLocationKey(suburb, state);
return GetCacheFilePattern(locationKey);
}
}
+63
View File
@@ -0,0 +1,63 @@
namespace BomLocalService.Utilities;
public static class LocationHelper
{
/// <summary>
/// Creates a location key from suburb and state (e.g., "Pomona_QLD")
/// </summary>
public static string GetLocationKey(string suburb, string state)
{
return $"{suburb}_{state}";
}
/// <summary>
/// Sanitizes a filename by removing invalid characters and replacing spaces/commas
/// </summary>
public static string SanitizeFileName(string fileName)
{
var invalidChars = Path.GetInvalidFileNameChars();
return string.Join("_", fileName.Split(invalidChars, StringSplitOptions.RemoveEmptyEntries))
.Replace(" ", "_")
.Replace(",", "");
}
/// <summary>
/// Parses suburb and state from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png)
/// Returns null if parsing fails
/// </summary>
public static (string suburb, string state)? ParseLocationFromFilename(string fileName)
{
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
var parts = fileNameWithoutExtension.Split('_');
if (parts.Length >= 2)
{
return (parts[0], parts[1]);
}
return null;
}
/// <summary>
/// Parses timestamp from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png)
/// Returns null if parsing fails
/// </summary>
public static DateTime? ParseTimestampFromFilename(string fileName)
{
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
var parts = fileNameWithoutExtension.Split('_');
if (parts.Length >= 3)
{
// Last two parts should be date and time
var dateTimeStr = $"{parts[^2]}_{parts[^1]}";
if (DateTime.TryParseExact(dateTimeStr, "yyyyMMdd_HHmmss", null, System.Globalization.DateTimeStyles.None, out var fileTime))
{
return fileTime;
}
}
return null;
}
}
+40
View File
@@ -0,0 +1,40 @@
using BomLocalService.Models;
namespace BomLocalService.Utilities;
public static class ResponseBuilder
{
/// <summary>
/// Creates a RadarScreenshotResponse from a file path and metadata
/// </summary>
public static RadarScreenshotResponse CreateRadarScreenshotResponse(
string imagePath,
LastUpdatedInfo? metadata = null)
{
var lastWriteTime = File.Exists(imagePath)
? File.GetLastWriteTime(imagePath)
: DateTime.UtcNow;
if (metadata == null)
{
return new RadarScreenshotResponse
{
ImagePath = imagePath,
LastUpdated = lastWriteTime,
ObservationTime = DateTime.UtcNow,
ForecastTime = DateTime.UtcNow
};
}
return new RadarScreenshotResponse
{
ImagePath = imagePath,
LastUpdated = lastWriteTime,
ObservationTime = metadata.ObservationTime,
ForecastTime = metadata.ForecastTime,
WeatherStation = metadata.WeatherStation,
Distance = metadata.Distance
};
}
}
+37
View File
@@ -0,0 +1,37 @@
using Microsoft.Playwright;
namespace BomLocalService.Utilities;
public static class SelectorHelper
{
/// <summary>
/// Tries to find a locator by trying multiple selectors in order
/// Returns the first visible locator found, or null if none found
/// </summary>
public static async Task<ILocator?> FindLocatorBySelectorsAsync(
IPage page,
string[] selectors,
ILogger? logger = null)
{
foreach (var selector in selectors)
{
try
{
var locator = page.Locator(selector).First;
if (await locator.IsVisibleAsync())
{
logger?.LogInformation("Found element with selector: {Selector}", selector);
return locator;
}
}
catch
{
// Continue to next selector
continue;
}
}
return null;
}
}
+45
View File
@@ -0,0 +1,45 @@
namespace BomLocalService.Utilities;
public static class StateAbbreviationHelper
{
private static readonly Dictionary<string, string[]> StateMappings = new(StringComparer.OrdinalIgnoreCase)
{
{ "qld", new[] { "queensland" } },
{ "nsw", new[] { "new south wales" } },
{ "vic", new[] { "victoria" } },
{ "sa", new[] { "south australia" } },
{ "wa", new[] { "western australia" } },
{ "tas", new[] { "tasmania" } },
{ "nt", new[] { "northern territory" } },
{ "act", new[] { "australian capital territory" } }
};
/// <summary>
/// Checks if the text matches the state abbreviation or full state name
/// </summary>
public static bool MatchesState(string text, string stateAbbreviation)
{
if (string.IsNullOrWhiteSpace(stateAbbreviation))
{
return true; // No state specified, match any
}
var textLower = text.ToLower();
var stateLower = stateAbbreviation.ToLower().Trim();
// Direct match
if (textLower.Contains(stateLower))
{
return true;
}
// Check abbreviation mappings
if (StateMappings.TryGetValue(stateLower, out var fullNames))
{
return fullNames.Any(fullName => textLower.Contains(fullName));
}
return false;
}
}
+24
View File
@@ -0,0 +1,24 @@
namespace BomLocalService.Utilities;
public static class ValidationHelper
{
/// <summary>
/// Validates suburb and state parameters
/// Returns error message if validation fails, null if valid
/// </summary>
public static string? ValidateLocation(string suburb, string state)
{
if (string.IsNullOrWhiteSpace(suburb))
{
return "Suburb is required";
}
if (string.IsNullOrWhiteSpace(state))
{
return "State is required";
}
return null;
}
}
+30
View File
@@ -0,0 +1,30 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*",
"CacheDirectory": "./cache",
"CacheRetentionHours": 24,
"CacheExpirationMinutes": 12.5,
"CacheManagement": {
"CheckIntervalMinutes": 5,
"InitialDelaySeconds": 10,
"UpdateStaggerSeconds": 2,
"LocationStaggerSeconds": 1
},
"CacheCleanup": {
"IntervalHours": 1
},
"Screenshot": {
"DynamicContentWaitMs": 2000,
"TileRenderWaitMs": 5000
},
"Debug": {
"Enabled": false,
"WaitMs": 2000
},
"Timezone": "Australia/Brisbane"
}
+45
View File
@@ -0,0 +1,45 @@
services:
bom-local-service:
build:
context: .
dockerfile: Dockerfile
container_name: bom-local-service
ports:
- "${HOST_PORT:-8082}:${CONTAINER_PORT:-8080}"
volumes:
- ./cache:/app/cache
environment:
# ASP.NET Core configuration
# Default to Production for security (use Development only for local debugging)
- ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Production}
- ASPNETCORE_URLS=http://+:${CONTAINER_PORT:-8080}
# Virtual display for non-headless browser
- DISPLAY=:99
# Application configuration (override appsettings.json)
# Use double underscore (__) for nested config keys
- CACHEDIRECTORY=${CACHEDIRECTORY:-/app/cache}
- CACHERETENTIONHOURS=${CACHERETENTIONHOURS:-24}
- CACHEEXPIRATIONMINUTES=${CACHEEXPIRATIONMINUTES:-12.5}
- CACHEMANAGEMENT__CHECKINTERVALMINUTES=${CACHEMANAGEMENT__CHECKINTERVALMINUTES:-5}
- CACHEMANAGEMENT__INITIALDELAYSECONDS=${CACHEMANAGEMENT__INITIALDELAYSECONDS:-10}
- CACHEMANAGEMENT__UPDATESTAGGERSECONDS=${CACHEMANAGEMENT__UPDATESTAGGERSECONDS:-2}
- CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS=${CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS:-1}
- CACHECLEANUP__INTERVALHOURS=${CACHECLEANUP__INTERVALHOURS:-1}
- SCREENSHOT__DYNAMICCONTENTWAITMS=${SCREENSHOT__DYNAMICCONTENTWAITMS:-2000}
- SCREENSHOT__TILERENDERWAITMS=${SCREENSHOT__TILERENDERWAITMS:-5000}
- DEBUG__ENABLED=${DEBUG__ENABLED:-false}
- DEBUG__WAITMS=${DEBUG__WAITMS:-2000}
- TIMEZONE=${TIMEZONE:-Australia/Brisbane}
shm_size: '1gb'
ipc: host
restart: unless-stopped
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]